diff options
175 files changed, 4045 insertions, 1587 deletions
diff --git a/SConstruct b/SConstruct index 102717fe5a8..f72671f57e8 100644 --- a/SConstruct +++ b/SConstruct @@ -2924,6 +2924,63 @@ def doConfigure(myenv): # ask each module to configure itself and the build environment. moduleconfig.configure_modules(mongo_modules, conf) + if env['TARGET_ARCH'] == "ppc64le": + # This checks for an altivec optimization we use in full text search. + # Different versions of gcc appear to put output bytes in different + # parts of the output vector produced by vec_vbpermq. This configure + # check looks to see which format the compiler produces. + # + # NOTE: This breaks cross compiles, as it relies on checking runtime functionality for the + # environment we're in. A flag to choose the index, or the possibility that we don't have + # multiple versions to support (after a compiler upgrade) could solve the problem if we + # eventually need them. + def CheckAltivecVbpermqOutput(context, index): + test_body = """ + #include <altivec.h> + #include <cstring> + #include <cstdint> + #include <cstdlib> + + int main() {{ + using Native = __vector signed char; + const size_t size = sizeof(Native); + const Native bits = {{ 120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 0 }}; + + uint8_t inputBuf[size]; + std::memset(inputBuf, 0xFF, sizeof(inputBuf)); + + for (size_t offset = 0; offset <= size; offset++) {{ + Native vec = vec_vsx_ld(0, reinterpret_cast<const Native*>(inputBuf)); + + uint64_t mask = vec_extract(vec_vbpermq(vec, bits), {0}); + + size_t initialZeros = (mask == 0 ? size : __builtin_ctzll(mask)); + if (initialZeros != offset) {{ + return 1; + }} + + if (offset < size) {{ + inputBuf[offset] = 0; // Add an initial 0 for the next loop. + }} + }} + + return 0; + }} + """.format(index) + + context.Message('Checking for vec_vbperm output in index {0}... '.format(index)) + ret = context.TryRun(textwrap.dedent(test_body), ".cpp") + context.Result(ret[0]) + return ret[0] + + conf.AddTest('CheckAltivecVbpermqOutput', CheckAltivecVbpermqOutput) + + outputIndex = next((idx for idx in [0,1] if conf.CheckAltivecVbpermqOutput(idx)), None) + if outputIndex is not None: + conf.env.SetConfigHeaderDefine("MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX", outputIndex) + else: + myenv.ConfError("Running on ppc64le, but can't find a correct vec_vbpermq output index. Compiler or platform not supported") + return conf.Finish() env = doConfigure( env ) diff --git a/buildscripts/packager.py b/buildscripts/packager.py index dcc95726398..d46d7e193c3 100755 --- a/buildscripts/packager.py +++ b/buildscripts/packager.py @@ -258,9 +258,7 @@ class Distro(object): else: raise Exception("unsupported build_os: %s" % build_os) elif self.n == 'debian': - if build_os == 'debian71': - return 'wheezy' - elif build_os == 'debian81': + if build_os == 'debian81': return 'jessie' else: raise Exception("unsupported build_os: %s" % build_os) @@ -277,7 +275,7 @@ class Distro(object): def build_os(self, arch): """Return the build os label in the binary package to download (e.g. "rhel55" for redhat, - "ubuntu1204" for ubuntu, "debian71" for debian, "suse11" for suse, etc.)""" + "ubuntu1204" for ubuntu, "debian81" for debian, "suse11" for suse, etc.)""" # Community builds only support amd64 if arch not in ['x86_64', 'ppc64le', 's390x', 'arm64']: raise Exception("BUG: unsupported architecture (%s)" % arch) @@ -291,7 +289,7 @@ class Distro(object): elif self.n == 'ubuntu': return [ "ubuntu1204", "ubuntu1404", "ubuntu1604", ] elif self.n == 'debian': - return [ "debian71", "debian81" ] + return [ "debian81" ] else: raise Exception("BUG: unsupported platform?") @@ -471,12 +469,8 @@ def make_deb(distro, build_os, arch, spec, srcdir): sdir=setupdir(distro, build_os, arch, spec) if re.search("debian", distro.name()): os.unlink(sdir+"debian/mongod.upstart") - if build_os == "debian71": - os.link(sdir+"debian/init.d", sdir+"debian/%s%s-server.mongod.init" % (distro.pkgbase(), suffix)) - os.unlink(sdir+"debian/mongod.service") - else: - os.link(sdir+"debian/mongod.service", sdir+"debian/%s%s-server.mongod.service" % (distro.pkgbase(), suffix)) - os.unlink(sdir+"debian/init.d") + os.link(sdir+"debian/mongod.service", sdir+"debian/%s%s-server.mongod.service" % (distro.pkgbase(), suffix)) + os.unlink(sdir+"debian/init.d") elif re.search("ubuntu", distro.name()): os.unlink(sdir+"debian/init.d") if build_os in ("ubuntu1204", "ubuntu1404", "ubuntu1410"): diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index 7aa13277384..cee9ce4e391 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -54,10 +54,18 @@ def _execute_suite(suite, logging_config): logger.info("Skipping %ss, no tests to run", group.test_kind) continue + archive = None + if resmokelib.config.ARCHIVE_FILE: + archive = resmokelib.utils.archival.Archival( + archival_json_file=resmokelib.config.ARCHIVE_FILE, + limit_size_mb=resmokelib.config.ARCHIVE_LIMIT_MB, + limit_files=resmokelib.config.ARCHIVE_LIMIT_TESTS, + logger=logger) group_config = suite.get_executor_config().get(group.test_kind, {}) executor = resmokelib.testing.executor.TestGroupExecutor(logger, group, logging_config, + archive_instance=archive, **group_config) try: @@ -73,6 +81,9 @@ def _execute_suite(suite, logging_config): group.test_kind, suite.get_name()) suite.return_code = 2 return False + finally: + if archive: + archive.exit() def _log_summary(logger, suites, time_taken): diff --git a/buildscripts/resmokeconfig/suites/aggregation.yml b/buildscripts/resmokeconfig/suites/aggregation.yml index c7a8c6e64be..0a775a2d93c 100644 --- a/buildscripts/resmokeconfig/suites/aggregation.yml +++ b/buildscripts/resmokeconfig/suites/aggregation.yml @@ -8,6 +8,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/aggregation_ese.yml b/buildscripts/resmokeconfig/suites/aggregation_ese.yml index 395a8a37898..a2d260d91d1 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_ese.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_ese.yml @@ -16,6 +16,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/aggregation_facet_unwind_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_facet_unwind_passthrough.yml index 40d248aa22d..476ba72b05d 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_facet_unwind_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_facet_unwind_passthrough.yml @@ -8,6 +8,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml index 72b838b3c07..5eea3a717bb 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml @@ -31,6 +31,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml index 1cb834a63eb..c5b76c31ef5 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml @@ -29,6 +29,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/bulk_gle_passthrough.yml b/buildscripts/resmokeconfig/suites/bulk_gle_passthrough.yml index 8e88985f671..309dd0364ea 100644 --- a/buildscripts/resmokeconfig/suites/bulk_gle_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/bulk_gle_passthrough.yml @@ -7,6 +7,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: legacy diff --git a/buildscripts/resmokeconfig/suites/concurrency.yml b/buildscripts/resmokeconfig/suites/concurrency.yml index e585902274d..4af6a5e31ee 100644 --- a/buildscripts/resmokeconfig/suites/concurrency.yml +++ b/buildscripts/resmokeconfig/suites/concurrency.yml @@ -12,6 +12,10 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections + tests: true config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_replication.yml index 8c9d272caea..9a575e63356 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication.yml @@ -6,6 +6,8 @@ selector: # Concurrency tests that run against a replica set start one themselves. executor: js_test: + archive: + tests: true config: shell_options: nodb: '' diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml index b5bd8b7169e..0e219f2f26c 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml @@ -6,6 +6,8 @@ selector: # Concurrency tests that run against a sharded cluster start one themselves. executor: js_test: + archive: + tests: true config: shell_options: nodb: '' diff --git a/buildscripts/resmokeconfig/suites/core.yml b/buildscripts/resmokeconfig/suites/core.yml index 04a6f97cb7d..9ba9ee4b742 100644 --- a/buildscripts/resmokeconfig/suites/core.yml +++ b/buildscripts/resmokeconfig/suites/core.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/core_auth.yml b/buildscripts/resmokeconfig/suites/core_auth.yml index 72c6c716927..cca107d5afa 100644 --- a/buildscripts/resmokeconfig/suites/core_auth.yml +++ b/buildscripts/resmokeconfig/suites/core_auth.yml @@ -18,6 +18,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/core_compression.yml b/buildscripts/resmokeconfig/suites/core_compression.yml index d09716735da..32e3d381bbc 100644 --- a/buildscripts/resmokeconfig/suites/core_compression.yml +++ b/buildscripts/resmokeconfig/suites/core_compression.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/core_ese.yml b/buildscripts/resmokeconfig/suites/core_ese.yml index 74ab7c9b195..86aa646bdb7 100644 --- a/buildscripts/resmokeconfig/suites/core_ese.yml +++ b/buildscripts/resmokeconfig/suites/core_ese.yml @@ -9,6 +9,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/core_minimum_batch_size.yml b/buildscripts/resmokeconfig/suites/core_minimum_batch_size.yml index 085c32c8c29..3b5a179914b 100644 --- a/buildscripts/resmokeconfig/suites/core_minimum_batch_size.yml +++ b/buildscripts/resmokeconfig/suites/core_minimum_batch_size.yml @@ -13,6 +13,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/core_op_query.yml b/buildscripts/resmokeconfig/suites/core_op_query.yml index 5ec056a57ff..45e57b65376 100644 --- a/buildscripts/resmokeconfig/suites/core_op_query.yml +++ b/buildscripts/resmokeconfig/suites/core_op_query.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: rpcProtocols: opQueryOnly diff --git a/buildscripts/resmokeconfig/suites/decimal.yml b/buildscripts/resmokeconfig/suites/decimal.yml index 62cb11ecab2..6c934554e42 100644 --- a/buildscripts/resmokeconfig/suites/decimal.yml +++ b/buildscripts/resmokeconfig/suites/decimal.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/disk.yml b/buildscripts/resmokeconfig/suites/disk.yml index fd8a5fd54b1..845ce95ea8f 100644 --- a/buildscripts/resmokeconfig/suites/disk.yml +++ b/buildscripts/resmokeconfig/suites/disk.yml @@ -7,6 +7,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/dur_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/dur_jscore_passthrough.yml index 347d23de967..f973f95586c 100644 --- a/buildscripts/resmokeconfig/suites/dur_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/dur_jscore_passthrough.yml @@ -8,6 +8,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/httpinterface.yml b/buildscripts/resmokeconfig/suites/httpinterface.yml index 1f15a873236..c74801ccdf6 100644 --- a/buildscripts/resmokeconfig/suites/httpinterface.yml +++ b/buildscripts/resmokeconfig/suites/httpinterface.yml @@ -7,6 +7,9 @@ selector: - jstests/httpinterface/httpinterface.js executor: js_test: + archive: + hooks: + - ValidateCollections hooks: - class: ValidateCollections fixture: diff --git a/buildscripts/resmokeconfig/suites/integration_tests_replset.yml b/buildscripts/resmokeconfig/suites/integration_tests_replset.yml index 1345b567eeb..f4d3e6e312e 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_replset.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_replset.yml @@ -4,6 +4,11 @@ selector: executor: cpp_integration_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: {} hooks: # The CheckReplDBHash hook waits until all operations have replicated to and have been applied diff --git a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml index a0e8c16be9b..444c66aa30d 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml @@ -6,6 +6,10 @@ selector: executor: cpp_integration_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: {} hooks: - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml b/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml index c594fc6fae1..127a7535568 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml @@ -4,6 +4,9 @@ selector: executor: cpp_integration_test: + archive: + hooks: + - ValidateCollections config: {} hooks: - class: ValidateCollections diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz.yml b/buildscripts/resmokeconfig/suites/jstestfuzz.yml index 872c2a1f229..591255038f3 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml index a482038899f..f32a24e4ee8 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml @@ -5,6 +5,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml index 034f254494f..d38814a0008 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - BackgroundInitialSync config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_resync.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_resync.yml index 1038b99a10c..973914a3c01 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_resync.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_resync.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - BackgroundInitialSync config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml index 9a8a02db6b9..bf2f84fe373 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml @@ -5,6 +5,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/master_slave_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/master_slave_jscore_passthrough.yml index a46c0cf29ed..1de4c693009 100644 --- a/buildscripts/resmokeconfig/suites/master_slave_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/master_slave_jscore_passthrough.yml @@ -10,6 +10,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/mmap.yml b/buildscripts/resmokeconfig/suites/mmap.yml index 6fc951c25cb..0aefad2ee53 100644 --- a/buildscripts/resmokeconfig/suites/mmap.yml +++ b/buildscripts/resmokeconfig/suites/mmap.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml index 2afbe6ae044..8012bfaf155 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml @@ -9,6 +9,10 @@ selector: # noPassthrough tests start their own mongod's. executor: js_test: + archive: + tests: + - jstests/noPassthrough/backup*.js + - jstests/noPassthrough/wt_unclean_shutdown.js config: shell_options: nodb: '' diff --git a/buildscripts/resmokeconfig/suites/no_passthrough_with_mongod.yml b/buildscripts/resmokeconfig/suites/no_passthrough_with_mongod.yml index fa8e62813d1..4e8bfc22bab 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough_with_mongod.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough_with_mongod.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/parallel.yml b/buildscripts/resmokeconfig/suites/parallel.yml index 7f42d19c18c..9611ec55c48 100644 --- a/buildscripts/resmokeconfig/suites/parallel.yml +++ b/buildscripts/resmokeconfig/suites/parallel.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml index 22e84bc9c63..d469ff85f75 100644 --- a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml @@ -28,6 +28,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml index 114ebcab372..034c0812e90 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml @@ -73,6 +73,9 @@ selector: run_hook_interval: &run_hook_interval 20 executor: js_test: + archive: + hooks: + - BackgroundInitialSync config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_initsync_static_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_initsync_static_jscore_passthrough.yml index c15b23046a9..8fce4506393 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_initsync_static_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_initsync_static_jscore_passthrough.yml @@ -12,6 +12,9 @@ selector: run_hook_interval: &run_hook_interval 20 executor: js_test: + archive: + hooks: + - IntermediateInitialSync config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml index f487c1aa113..60da231d3bc 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml @@ -11,6 +11,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_kill_secondaries_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_kill_secondaries_jscore_passthrough.yml index a4b6c5a5feb..e62727bb606 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_kill_secondaries_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_kill_secondaries_jscore_passthrough.yml @@ -15,6 +15,9 @@ selector: executor: js_test: + archive: + hooks: + - PeriodicKillSecondaries config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml index 1e51713a475..c7d605d1184 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml @@ -73,6 +73,9 @@ selector: run_hook_interval: &run_hook_interval 20 executor: js_test: + archive: + hooks: + - BackgroundInitialSync config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_resync_static_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_resync_static_jscore_passthrough.yml index dc2059879d3..bd26f1c23f8 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_resync_static_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_resync_static_jscore_passthrough.yml @@ -12,6 +12,9 @@ selector: run_hook_interval: &run_hook_interval 20 executor: js_test: + archive: + hooks: + - IntermediateInitialSync config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/rlp.yml b/buildscripts/resmokeconfig/suites/rlp.yml index ef0dfeda45a..16413f1d4a8 100644 --- a/buildscripts/resmokeconfig/suites/rlp.yml +++ b/buildscripts/resmokeconfig/suites/rlp.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/serial_run.yml b/buildscripts/resmokeconfig/suites/serial_run.yml index 4ae010c7ec8..c25e61a980a 100644 --- a/buildscripts/resmokeconfig/suites/serial_run.yml +++ b/buildscripts/resmokeconfig/suites/serial_run.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml index 96674cfc20c..104707d1ac8 100644 --- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml @@ -69,6 +69,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml index 8879e39f143..5f646fd7653 100644 --- a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml @@ -19,6 +19,9 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml index 496bd265a58..d2cb6ed55f1 100644 --- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml @@ -51,6 +51,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/slow1.yml b/buildscripts/resmokeconfig/suites/slow1.yml index 10c399ec9e5..8a764c39d94 100644 --- a/buildscripts/resmokeconfig/suites/slow1.yml +++ b/buildscripts/resmokeconfig/suites/slow1.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml index 388046d2782..b19989e3656 100644 --- a/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml @@ -46,6 +46,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index b75753398a5..8f636ddbadd 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -33,14 +33,20 @@ MONGO_RUNNER_SUBDIR = "mongorunner" # Names below correspond to how they are specified via the command line or in the options YAML file. DEFAULTS = { + "archiveFile": None, + "archiveLimitMb": 5000, + "archiveLimitTests": 10, "basePort": 20000, "buildloggerUrl": "https://logkeeper.mongodb.org", "continueOnFailure": False, "dbpathPrefix": None, "dbtest": None, + "distroId": None, "dryRun": None, "excludeWithAllTags": None, "excludeWithAnyTags": None, + "executionNumber": 0, + "gitRevision": None, "includeWithAllTags": None, "includeWithAnyTags": None, "jobs": 1, @@ -51,6 +57,7 @@ DEFAULTS = { "mongosSetParameters": None, "nojournal": False, "numClientsPerFixture": 1, + "projectName": "mongodb-mongo-master", "repeat": 1, "reportFile": None, "seed": long(time.time() * 256), # Taken from random.py code in Python 2.7. @@ -59,6 +66,9 @@ DEFAULTS = { "shuffle": False, "storageEngine": None, "storageEngineCacheSizeGB": None, + "taskId": None, + "taskName": None, + "variantName": None, "wiredTigerCollectionConfigString": None, "wiredTigerEngineConfigString": None, "wiredTigerIndexConfigString": None @@ -69,6 +79,15 @@ DEFAULTS = { # Variables that are set by the user at the command line or with --options. ## +# The name of the archive JSON file used to associate S3 archives to an Evergreen task. +ARCHIVE_FILE = None + +# The limit size of all archive files for an Evergreen task. +ARCHIVE_LIMIT_MB = None + +# The limit number of tests to archive for an Evergreen task. +ARCHIVE_LIMIT_TESTS = None + # The starting port number to use for mongod and mongos processes spawned by resmoke.py and the # mongo shell. BASE_PORT = None @@ -87,6 +106,28 @@ DBTEST_EXECUTABLE = None # actually running them). DRY_RUN = None +# The identifier for the Evergreen distro that resmoke.py is being run on. +EVERGREEN_DISTRO_ID = None + +# The number of the Evergreen execution that resmoke.py is being run on. +EVERGREEN_EXECUTION = None + +# The name of the Evergreen project that resmoke.py is being run on. +EVERGREEN_PROJECT_NAME = None + +# The git revision of the Evergreen task that resmoke.py is being run on. +EVERGREEN_REVISION = None + +# The identifier for the Evergreen task that resmoke.py is being run under. If set, then the +# Evergreen task id value will be transmitted to logkeeper when creating builds and tests. +EVERGREEN_TASK_ID = None + +# The name of the Evergreen task that resmoke.py is being run for. +EVERGREEN_TASK_NAME = None + +# The name of the Evergreen build variant that resmoke.py is being run on. +EVERGREEN_VARIANT_NAME = None + # If set, then any jstests that have all of the specified tags will be excluded from the suite(s). EXCLUDE_WITH_ALL_TAGS = None @@ -177,6 +218,9 @@ WT_INDEX_CONFIG = None # Internally used configuration options that aren't exposed to the user ## +# S3 Bucket to upload archive files. +ARCHIVE_BUCKET = "mongodatafiles" + # Default sort order for test execution. Will only be changed if --suites wasn't specified. ORDER_TESTS_BY_NAME = True diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 870e0e9145c..59b4585a273 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -18,14 +18,20 @@ from .. import resmokeconfig # Mapping of the attribute of the parsed arguments (dest) to its key as it appears in the options # YAML configuration file. Most should only be converting from snake_case to camelCase. DEST_TO_CONFIG = { + "archive_file": "archiveFile", + "archive_limit_mb": "archiveLimitMb", + "archive_limit_tests": "archiveLimitTests", "base_port": "basePort", "buildlogger_url": "buildloggerUrl", "continue_on_failure": "continueOnFailure", "dbpath_prefix": "dbpathPrefix", "dbtest_executable": "dbtest", + "distro_id": "distroId", "dry_run": "dryRun", "exclude_with_all_tags": "excludeWithAllTags", "exclude_with_any_tags": "excludeWithAnyTags", + "execution_number": "executionNumber", + "git_revision": "gitRevision", "include_with_all_tags": "includeWithAllTags", "include_with_any_tags": "includeWithAnyTags", "jobs": "jobs", @@ -37,6 +43,7 @@ DEST_TO_CONFIG = { "no_journal": "nojournal", "num_clients_per_fixture": "numClientsPerFixture", "prealloc_journal": "preallocJournal", + "project_name": "projectName", "repeat": "repeat", "report_file": "reportFile", "seed": "seed", @@ -45,6 +52,9 @@ DEST_TO_CONFIG = { "shuffle": "shuffle", "storage_engine": "storageEngine", "storage_engine_cache_size": "storageEngineCacheSizeGB", + "task_id": "taskId", + "task_name": "taskName", + "variant_name": "variantName", "wt_coll_config": "wiredTigerCollectionConfigString", "wt_engine_config": "wiredTigerEngineConfigString", "wt_index_config": "wiredTigerIndexConfigString" @@ -79,6 +89,23 @@ def parse_command_line(): parser.add_option("--options", dest="options_file", metavar="OPTIONS", help="A YAML file that specifies global options to resmoke.py.") + parser.add_option("--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE", + help=("Sets the archive file name for the Evergreen task running the tests." + " The archive file is JSON format containing a list of tests that were" + " successfully archived to S3. If unspecified, no data files from tests" + " will be archived in S3. Tests can be designated for archival in the" + " task suite configuration file.")) + + parser.add_option("--archiveLimitMb", type="int", dest="archive_limit_mb", + metavar="ARCHIVE_LIMIT_MB", + help=("Sets the limit (in MB) for archived files to S3. A value of 0" + " indicates there is no limit.")) + + parser.add_option("--archiveLimitTests", type="int", dest="archive_limit_tests", + metavar="ARCHIVE_LIMIT_TESTS", + help=("Sets the maximum number of tests to archive to S3. A value" + " of 0 indicates there is no limit.")) + parser.add_option("--basePort", dest="base_port", metavar="PORT", help=("The starting port number to use for mongod and mongos processes" " spawned by resmoke.py or the tests themselves. Each fixture and Job" @@ -209,6 +236,42 @@ def parse_command_line(): parser.add_option("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG", help="Set the WiredTiger index configuration setting for all mongod's.") + + evergreen_options = optparse.OptionGroup( + parser, title="Evergreen options", + description=("Options used to propagate information about the Evergreen task running this" + " script.")) + parser.add_option_group(evergreen_options) + + evergreen_options.add_option("--buildId", dest="build_id", metavar="BUILD_ID", + help="Sets the build ID of the task.") + + evergreen_options.add_option("--distroId", dest="distro_id", metavar="DISTRO_ID", + help=("Sets the identifier for the Evergreen distro running the" + " tests.")) + + evergreen_options.add_option("--executionNumber", type="int", dest="execution_number", + metavar="EXECUTION_NUMBER", + help=("Sets the number for the Evergreen execution running the" + " tests.")) + + evergreen_options.add_option("--gitRevision", dest="git_revision", metavar="GIT_REVISION", + help=("Sets the git revision for the Evergreen task running the" + " tests.")) + + evergreen_options.add_option("--projectName", dest="project_name", metavar="PROJECT_NAME", + help=("Sets the name of the Evergreen project running the tests.")) + + evergreen_options.add_option("--taskName", dest="task_name", metavar="TASK_NAME", + help="Sets the name of the Evergreen task running the tests.") + + evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID", + help="Sets the Id of the Evergreen task running the tests.") + + evergreen_options.add_option("--variantName", dest="variant_name", metavar="VARIANT_NAME", + help=("Sets the name of the Evergreen build variant running the" + " tests.")) + parser.set_defaults(executor_file="with_server", logger_file="console", dry_run="off", @@ -237,11 +300,21 @@ def update_config_vars(values): if values[dest] is not None: config[config_var] = values[dest] + _config.ARCHIVE_FILE = config.pop("archiveFile") + _config.ARCHIVE_LIMIT_MB = config.pop("archiveLimitMb") + _config.ARCHIVE_LIMIT_TESTS = config.pop("archiveLimitTests") _config.BASE_PORT = int(config.pop("basePort")) _config.BUILDLOGGER_URL = config.pop("buildloggerUrl") _config.DBPATH_PREFIX = _expand_user(config.pop("dbpathPrefix")) _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest")) _config.DRY_RUN = config.pop("dryRun") + _config.EVERGREEN_DISTRO_ID = config.pop("distroId") + _config.EVERGREEN_EXECUTION = config.pop("executionNumber") + _config.EVERGREEN_PROJECT_NAME = config.pop("projectName") + _config.EVERGREEN_REVISION = config.pop("gitRevision") + _config.EVERGREEN_TASK_ID = config.pop("taskId") + _config.EVERGREEN_TASK_NAME = config.pop("taskName") + _config.EVERGREEN_VARIANT_NAME = config.pop("variantName") _config.EXCLUDE_WITH_ALL_TAGS = config.pop("excludeWithAllTags") _config.EXCLUDE_WITH_ANY_TAGS = config.pop("excludeWithAnyTags") _config.FAIL_FAST = not config.pop("continueOnFailure") diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 438a06b9c8e..e88eed6bc02 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -8,6 +8,7 @@ import threading import time from . import fixtures +from . import hook_test_archival as archival from . import hooks as _hooks from . import job as _job from . import report as _report @@ -35,7 +36,9 @@ class TestGroupExecutor(object): logging_config, config=None, fixture=None, - hooks=None): + hooks=None, + archive_instance=None, + archive=None): """ Initializes the TestGroupExecutor with the test group to run. """ @@ -49,6 +52,11 @@ class TestGroupExecutor(object): self.hooks_config = utils.default_if_none(hooks, []) self.test_config = utils.default_if_none(config, {}) + self.archival = None + if archive_instance: + self.archival = archival.HookTestArchival(test_group, self.hooks_config, + archive_instance, archive) + self._test_group = test_group self._using_buildlogger = logging.config.using_buildlogger(logging_config) @@ -299,7 +307,7 @@ class TestGroupExecutor(object): build_id=build_id, build_config=build_config) - return _job.Job(logger, fixture, hooks, report) + return _job.Job(logger, fixture, hooks, report, self.archival) def _make_test_queue(self): """ diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index f2cb1b9c8b8..5da95175008 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -4,12 +4,16 @@ Interface of the different fixtures for executing JSTests against. from __future__ import absolute_import +import os.path import time import pymongo +import pymongo.errors +from ... import config from ... import errors from ... import logging +from ... import utils class Fixture(object): @@ -17,7 +21,7 @@ class Fixture(object): Base class for all fixtures. """ - def __init__(self, logger, job_num): + def __init__(self, logger, job_num, dbpath_prefix=None): """ Initializes the fixture with a logger instance. """ @@ -32,9 +36,12 @@ class Fixture(object): self.logger = logger self.job_num = job_num - self.port = None # Port that the mongo shell should connect to. + dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) + dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) + self._dbpath_prefix = os.path.join(dbpath_prefix, "job{}".format(self.job_num)) + def setup(self): """ Creates the fixture. @@ -79,6 +86,9 @@ class Fixture(object): """ return True + def get_dbpath_prefix(self): + return self._dbpath_prefix + def get_internal_connection_string(self): """ Returns the connection string for this fixture. This is NOT a diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py index fb444cfe097..f8bf4417451 100644 --- a/buildscripts/resmokelib/testing/fixtures/masterslave.py +++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py @@ -31,7 +31,7 @@ class MasterSlaveFixture(interface.ReplFixture): dbpath_prefix=None, preserve_dbpath=False): - interface.ReplFixture.__init__(self, logger, job_num) + interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) if "dbpath" in mongod_options: raise ValueError("Cannot specify mongod_options.dbpath") @@ -42,12 +42,7 @@ class MasterSlaveFixture(interface.ReplFixture): self.slave_options = utils.default_if_none(slave_options, {}) self.preserve_dbpath = preserve_dbpath - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self._dbpath_prefix = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.master = None self.slave = None diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index 0ec3de30280..b185ee7f8f2 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -40,7 +40,7 @@ class ReplicaSetFixture(interface.ReplFixture): voting_secondaries=False, use_replica_set_connection_string=False): - interface.ReplFixture.__init__(self, logger, job_num) + interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) self.mongod_executable = mongod_executable self.mongod_options = utils.default_if_none(mongod_options, {}) @@ -59,12 +59,7 @@ class ReplicaSetFixture(interface.ReplFixture): if "dbpath" in self.mongod_options: self._dbpath_prefix = self.mongod_options.pop("dbpath") else: - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self._dbpath_prefix = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.nodes = [] self.replset_name = None diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index 2e2db535d6d..9a90c5b31e1 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -47,7 +47,7 @@ class ShardedClusterFixture(interface.Fixture): the mongod and mongos processes. """ - interface.Fixture.__init__(self, logger, job_num) + interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) if "dbpath" in mongod_options: raise ValueError("Cannot specify mongod_options.dbpath") @@ -62,12 +62,7 @@ class ShardedClusterFixture(interface.Fixture): self.enable_sharding = utils.default_if_none(enable_sharding, []) self.auth_options = auth_options - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self._dbpath_prefix = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.configsvr = None self.mongos = None @@ -108,11 +103,7 @@ class ShardedClusterFixture(interface.Fixture): self.port = self.mongos.port client = utils.new_mongo_client(port=self.port) - if self.auth_options is not None: - auth_db = client[self.auth_options["authenticationDatabase"]] - auth_db.authenticate(self.auth_options["username"], - password=self.auth_options["password"], - mechanism=self.auth_options["authenticationMechanism"]) + self._auth_to_db(client) # Inform mongos about each of the shards for shard in self.shards: @@ -123,6 +114,20 @@ class ShardedClusterFixture(interface.Fixture): self.logger.info("Enabling sharding for '%s' database...", db_name) client.admin.command({"enablesharding": db_name}) + def _auth_to_db(self, client): + """Authenticate client for the 'authenticationDatabase'.""" + if self.auth_options is not None: + auth_db = client[self.auth_options["authenticationDatabase"]] + auth_db.authenticate(self.auth_options["username"], + password=self.auth_options["password"], + mechanism=self.auth_options["authenticationMechanism"]) + + def _stop_balancer(self, timeout_ms=60000): + """Stop the balancer.""" + client = utils.new_mongo_client(port=self.port) + self._auth_to_db(client) + client.admin.command({"balancerStop": 1}, maxTimeMS=timeout_ms) + def _do_teardown(self): """ Shuts down the sharded cluster. @@ -134,6 +139,8 @@ class ShardedClusterFixture(interface.Fixture): self.logger.info( "Sharded cluster was expected to be running in _do_teardown(), but wasn't.") + self._stop_balancer() + if self.configsvr is not None: if running_at_start: self.logger.info("Stopping config server...") @@ -154,6 +161,8 @@ class ShardedClusterFixture(interface.Fixture): if running_at_start: self.logger.info("Stopping shards...") + + for shard in self.shards: success = shard.teardown() and success if running_at_start: diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index bc69775c285..d462685a827 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -6,7 +6,6 @@ from __future__ import absolute_import import os import os.path -import shutil import time import pymongo @@ -34,7 +33,7 @@ class MongoDFixture(interface.Fixture): dbpath_prefix=None, preserve_dbpath=False): - interface.Fixture.__init__(self, logger, job_num) + interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) if "dbpath" in mongod_options and dbpath_prefix is not None: raise ValueError("Cannot specify both mongod_options.dbpath and dbpath_prefix") @@ -48,19 +47,16 @@ class MongoDFixture(interface.Fixture): # The dbpath in mongod_options takes precedence over other settings to make it easier for # users to specify a dbpath containing data to test against. if "dbpath" not in self.mongod_options: - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self.mongod_options["dbpath"] = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self.mongod_options["dbpath"] = os.path.join( + self._dbpath_prefix, config.FIXTURE_SUBDIR) self._dbpath = self.mongod_options["dbpath"] self.mongod = None def setup(self): - if not self.preserve_dbpath: - shutil.rmtree(self._dbpath, ignore_errors=True) + """Set up the mongod.""" + if not self.preserve_dbpath and os.path.lexists(self._dbpath): + utils.rmtree(self._dbpath, ignore_errors=False) try: os.makedirs(self._dbpath) @@ -145,6 +141,10 @@ class MongoDFixture(interface.Fixture): def is_running(self): return self.mongod is not None and self.mongod.poll() is None + def get_dbpath_prefix(self): + """ Returns the _dbpath, as this is the root of the data directory. """ + return self._dbpath + def get_internal_connection_string(self): if self.mongod is None: raise ValueError("Must call setup() before calling get_internal_connection_string()") diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py new file mode 100644 index 00000000000..bde562cf617 --- /dev/null +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -0,0 +1,116 @@ +""" +Enables supports for archiving tests or hooks. +""" + +from __future__ import absolute_import + +import os +import threading + +from .. import config +from .. import utils +from ..utils import globstar + + +class HookTestArchival(object): + """ + Archives hooks and tests to S3. + """ + + def __init__(self, suite, hooks, archive_instance, archive_config): + self.archive_instance = archive_instance + archive_config = utils.default_if_none(archive_config, {}) + + self.on_success = archive_config.get("on_success", False) + + self.tests = [] + if "tests" in archive_config: + # 'tests' is either a list of tests to archive or a bool (archive all if True). + if not isinstance(archive_config["tests"], bool): + for test in archive_config["tests"]: + self.tests += globstar.glob(test) + elif archive_config["tests"]: + self.tests = suite.tests + + self.hooks = [] + if "hooks" in archive_config: + # 'hooks' is either a list of hooks to archive or a bool (archive all if True). + if not isinstance(archive_config["hooks"], bool): + self.hooks = archive_config["hooks"] + elif archive_config["hooks"]: + for hook in hooks: + self.hooks.append(hook["class"]) + + self._tests_repeat = {} + self._lock = threading.Lock() + + def _should_archive(self, success): + """ Return True if failed test or 'on_success' is True. """ + return not success or self.on_success + + def _archive_hook(self, logger, hook, test, success): + """ Helper to archive hooks. """ + hook_match = hook.__class__.__name__ in self.hooks + if not hook_match or not self._should_archive(success): + return + + test_name = "{}:{}".format(test.short_name(), hook.__class__.__name__) + self._archive_hook_or_test(logger, test_name, test) + + def _archive_test(self, logger, test, success): + """ Helper to archive tests. """ + test_name = test.test_name + test_match = False + for arch_test in self.tests: + # Ensure that the test_name is in the same format as the arch_test. + if os.path.normpath(test_name) == os.path.normpath(arch_test): + test_match = True + break + if not test_match or not self._should_archive(success): + return + + self._archive_hook_or_test(logger, test_name, test) + + def archive(self, logger, test, success, hook=None): + """ Archives data files for hooks or tests. """ + if not config.ARCHIVE_FILE or not self.archive_instance: + return + if hook: + self._archive_hook(logger, hook, test, success) + else: + self._archive_test(logger, test, success) + + def _archive_hook_or_test(self, logger, test_name, test): + """ Trigger archive of data files for a test or hook. """ + + with self._lock: + # Test repeat number is how many times the particular test has been archived. + if test_name not in self._tests_repeat: + self._tests_repeat[test_name] = 0 + else: + self._tests_repeat[test_name] += 1 + # Normalize test path from a test or hook name. + test_path = \ + test_name.replace("/", "_").replace("\\", "_").replace(".", "_").replace(":", "_") + file_name = "mongo-data-{}-{}-{}-{}.tgz".format( + config.EVERGREEN_TASK_ID, + test_path, + config.EVERGREEN_EXECUTION, + self._tests_repeat[test_name]) + # Retrieve root directory for all dbPaths from fixture. + input_files = test.fixture.get_dbpath_prefix() + s3_bucket = config.ARCHIVE_BUCKET + s3_path = "{}/{}/{}/datafiles/{}".format( + config.EVERGREEN_PROJECT_NAME, + config.EVERGREEN_VARIANT_NAME, + config.EVERGREEN_REVISION, + file_name) + display_name = "Data files {} - Execution {} Repetition {}".format( + test_name, + config.EVERGREEN_EXECUTION, + self._tests_repeat[test_name]) + logger.info("Archiving data files for test %s from %s", test_name, input_files) + status, message = self.archive_instance.archive_files_to_s3( + display_name, input_files, s3_bucket, s3_path) + if status: + logger.warning("Archive failed for %s: %s", test_name, message) diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py index 4d17152b17d..c8be906dd3a 100644 --- a/buildscripts/resmokelib/testing/job.py +++ b/buildscripts/resmokelib/testing/job.py @@ -18,7 +18,7 @@ class Job(object): Runs tests from a queue. """ - def __init__(self, logger, fixture, hooks, report): + def __init__(self, logger, fixture, hooks, report, archival): """ Initializes the job with the specified fixture and custom behaviors. @@ -28,6 +28,7 @@ class Job(object): self.fixture = fixture self.hooks = hooks self.report = report + self.archival = archival def __call__(self, queue, interrupt_flag, teardown_flag=None): """ @@ -98,20 +99,38 @@ class Job(object): self._run_hooks_before_tests(test) test(self.report) - if config.FAIL_FAST and not self.report.wasSuccessful(): - self.logger.info("%s failed, so stopping..." % (test.shortDescription())) - raise errors.StopExecution("%s failed" % (test.shortDescription())) - - if not self.fixture.is_running(): - self.logger.error("%s marked as a failure because the fixture crashed during the test.", - test.shortDescription()) - self.report.setFailure(test, return_code=2) - # Always fail fast if the fixture fails. - raise errors.StopExecution("%s not running after %s" % - (self.fixture, test.shortDescription())) + try: + if config.FAIL_FAST and not self.report.wasSuccessful(): + self.logger.info("%s failed, so stopping..." % (test.shortDescription())) + raise errors.StopExecution("%s failed" % (test.shortDescription())) + + if not self.fixture.is_running(): + self.logger.error( + "%s marked as a failure because the fixture crashed during the test.", + test.shortDescription()) + self.report.setFailure(test, return_code=2) + # Always fail fast if the fixture fails. + raise errors.StopExecution("%s not running after %s" % + (self.fixture, test.shortDescription())) + + finally: + success = self.report.find_test_info(test).status == "pass" + self.archival.archive(self.logger, test, success) + if self.archival: + self.archival.archive(self.logger, test, success) self._run_hooks_after_tests(test) + def _run_hook(self, hook, hook_function, test): + """Provide helper to run hook and archival.""" + try: + success = False + hook_function(test, self.report) + success = True + finally: + if self.archival: + self.archival.archive(self.logger, test, success, hook=hook) + def _run_hooks_before_tests(self, test): """ Runs the before_test method on each of the hooks. @@ -122,7 +141,7 @@ class Job(object): try: for hook in self.hooks: - hook.before_test(test, self.report) + self._run_hook(hook, hook.before_test, test) except errors.StopExecution: raise @@ -156,7 +175,7 @@ class Job(object): """ try: for hook in self.hooks: - hook.after_test(test, self.report) + self._run_hook(hook, hook.after_test, test) except errors.StopExecution: raise diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index b9cccccc14f..e3c68393a4d 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -147,7 +147,7 @@ class TestReport(unittest.TestResult): unittest.TestResult.stopTest(self, test) with self._lock: - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.end_time = time.time() time_taken = test_info.end_time - test_info.start_time @@ -177,7 +177,7 @@ class TestReport(unittest.TestResult): with self._lock: self.num_errored += 1 - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.status = "error" test_info.return_code = test.return_code @@ -187,7 +187,7 @@ class TestReport(unittest.TestResult): """ with self._lock: - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) if test_info.end_time is None: raise ValueError("stopTest was not called on %s" % (test.basename())) @@ -211,7 +211,7 @@ class TestReport(unittest.TestResult): with self._lock: self.num_failed += 1 - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.status = "fail" test_info.return_code = test.return_code @@ -221,7 +221,7 @@ class TestReport(unittest.TestResult): """ with self._lock: - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) if test_info.end_time is None: raise ValueError("stopTest was not called on %s" % (test.basename())) @@ -244,7 +244,7 @@ class TestReport(unittest.TestResult): with self._lock: self.num_succeeded += 1 - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.status = "pass" test_info.return_code = test.return_code @@ -352,7 +352,7 @@ class TestReport(unittest.TestResult): # protecting it with the lock. self.__original_loggers = {} - def _find_test_info(self, test): + def find_test_info(self, test): """ Returns the status and timing information associated with 'test'. diff --git a/buildscripts/resmokelib/testing/testcases.py b/buildscripts/resmokelib/testing/testcases.py index 21d35215a29..78110ff680d 100644 --- a/buildscripts/resmokelib/testing/testcases.py +++ b/buildscripts/resmokelib/testing/testcases.py @@ -6,7 +6,6 @@ from __future__ import absolute_import import os import os.path -import shutil import threading import unittest @@ -236,7 +235,7 @@ class DBTestCase(TestCase): dbpath = os.path.join(dbpath_prefix, "job%d" % (self.fixture.job_num), "unittest") self.dbtest_options["dbpath"] = dbpath - shutil.rmtree(dbpath, ignore_errors=True) + utils.rmtree(dbpath, ignore_errors=True) try: os.makedirs(dbpath) @@ -352,7 +351,7 @@ class JSTestCase(TestCase): global_vars["TestData"] = test_data self.shell_options["global_vars"] = global_vars - shutil.rmtree(data_dir, ignore_errors=True) + utils.rmtree(data_dir, ignore_errors=True) self.num_clients = num_clients @@ -364,14 +363,19 @@ class JSTestCase(TestCase): process_kwargs = self.shell_options.get("process_kwargs", {}).copy() - if "KRB5_CONFIG" in process_kwargs and "KRB5CCNAME" not in process_kwargs: + if process_kwargs \ + and "env_vars" in process_kwargs \ + and "KRB5_CONFIG" in process_kwargs["env_vars"] \ + and "KRB5CCNAME" not in process_kwargs["env_vars"]: # Use a job-specific credential cache for JavaScript tests involving Kerberos. krb5_dir = os.path.join(data_dir, "krb5") + try: os.makedirs(krb5_dir) except os.error: pass - process_kwargs["KRB5CCNAME"] = "DIR:" + os.path.join(krb5_dir, ".") + + process_kwargs["env_vars"]["KRB5CCNAME"] = "DIR:" + krb5_dir self.shell_options["process_kwargs"] = process_kwargs diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py index df387cc3323..ff37fbb954a 100644 --- a/buildscripts/resmokelib/utils/__init__.py +++ b/buildscripts/resmokelib/utils/__init__.py @@ -5,15 +5,41 @@ Helper functions. from __future__ import absolute_import import os.path +import shutil +import sys import pymongo import yaml +from . import archival + def default_if_none(value, default): return value if value is not None else default +def rmtree(path, **kwargs): + """Wrap shutil.rmtreee. + + Use a UTF-8 unicode path if Windows. + See https://bugs.python.org/issue24672, where shutil.rmtree can fail with UTF-8. + Use a bytes path to rmtree, otherwise. + See https://github.com/pypa/setuptools/issues/706. + """ + if is_windows(): + if not isinstance(path, unicode): + path = unicode(path, "utf-8") + else: + if isinstance(path, unicode): + path = path.encode("utf-8") + shutil.rmtree(path, **kwargs) + + +def is_windows(): + """Return True if Windows.""" + return sys.platform.startswith("win32") or sys.platform.startswith("cygwin") + + def is_string_list(lst): """ Returns true if 'lst' is a list of strings, and false otherwise. diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py new file mode 100644 index 00000000000..1002f2ce902 --- /dev/null +++ b/buildscripts/resmokelib/utils/archival.py @@ -0,0 +1,331 @@ +""" +Archival utility. +""" + +from __future__ import absolute_import + +import Queue +import collections +import json +import math +import os +import sys +import tarfile +import tempfile +import threading +import time + +_IS_WINDOWS = sys.platform == "win32" or sys.platform == "cygwin" + +if _IS_WINDOWS: + import ctypes + +UploadArgs = collections.namedtuple( + "UploadArgs", + ["archival_file", + "display_name", + "local_file", + "content_type", + "s3_bucket", + "s3_path", + "delete_file"]) + +ArchiveArgs = collections.namedtuple( + "ArchiveArgs", ["archival_file", "display_name", "remote_file"]) + + +def file_list_size(files): + """ Return size (in bytes) of all 'files' and their subdirectories. """ + if isinstance(files, str): + files = [files] + file_bytes = 0 + for ifile in files: + if not os.path.exists(ifile): + pass + elif os.path.isdir(ifile): + file_bytes += directory_size(ifile) + else: + file_bytes += os.path.getsize(ifile) + return file_bytes + + +def directory_size(directory): + """ Return size (in bytes) of files in 'directory' tree. """ + dir_bytes = 0 + for root_dir, _, files in os.walk(unicode(directory)): + for name in files: + full_name = os.path.join(root_dir, name) + try: + dir_bytes += os.path.getsize(full_name) + except OSError: + # A file might be deleted while we are looping through the os.walk() result. + pass + return dir_bytes + + +def free_space(path): + """ Return file system free space (in bytes) for 'path'. """ + if _IS_WINDOWS: + dirname = os.path.dirname(path) + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW( + ctypes.c_wchar_p(dirname), None, None, ctypes.pointer(free_bytes)) + return free_bytes.value + else: + stat = os.statvfs(path) + return stat.f_bavail * stat.f_bsize + + +def remove_file(file_name): + """ Attempts to remove file. Returns status and message. """ + try: + # File descriptors, on Windows, are inherited by all subprocesses and file removal may fail + # because the file is still open. + # See https://www.python.org/dev/peps/pep-0446/#issues-with-inheritable-file-descriptors + os.remove(file_name) + status = 0 + message = "Successfully deleted file {}".format(file_name) + except Exception as err: + status = 1 + message = "Error deleting file {}: {}".format(file_name, err) + return status, message + + +class Archival(object): + """ Class to support file archival to S3.""" + + def __init__(self, + logger, + archival_json_file="archive.json", + limit_size_mb=0, + limit_files=0, + s3_client=None): + """ Archival init method. """ + + self.archival_json_file = archival_json_file + self.limit_size_mb = limit_size_mb + self.limit_files = limit_files + self.size_mb = 0 + self.num_files = 0 + self.archive_time = 0 + self.logger = logger + + # Lock to control access from multiple threads. + self._lock = threading.Lock() + + # Start the worker thread to update the 'archival_json_file'. + self._archive_file_queue = Queue.Queue() + self._archive_file_worker = threading.Thread( + target=self._update_archive_file_wkr, + args=(self._archive_file_queue, logger), + name="archive_file_worker") + self._archive_file_worker.setDaemon(True) + self._archive_file_worker.start() + if not s3_client: + self.s3_client = self._get_s3_client() + else: + self.s3_client = s3_client + + # Start the worker thread which uploads the archive. + self._upload_queue = Queue.Queue() + self._upload_worker = threading.Thread( + target=self._upload_to_s3_wkr, + args=(self._upload_queue, self._archive_file_queue, logger, self.s3_client), + name="upload_worker") + self._upload_worker.setDaemon(True) + self._upload_worker.start() + + @staticmethod + def _get_s3_client(): + # Since boto3 is a 3rd party module, we import locally. + import boto3 + return boto3.client("s3") + + def archive_files_to_s3(self, display_name, input_files, s3_bucket, s3_path): + """ + Archive 'input_files' to 's3_bucket' and 's3_path'. + + Archive is not done if user specified limits are reached. The size limit is + enforced after it has been exceeded, since it can only be calculated after the + tar/gzip has been done. + + Returns status and message, where message contains information if status is non-0. + """ + + start_time = time.time() + with self._lock: + if not input_files: + status = 1 + message = "No input_files specified" + elif self.limit_size_mb and self.size_mb >= self.limit_size_mb: + status = 1 + message = "Files not archived, {}MB size limit reached".format(self.limit_size_mb) + elif self.limit_files and self.num_files >= self.limit_files: + status = 1 + message = "Files not archived, {} file limit reached".format(self.limit_files) + else: + status, message, file_size_mb = self._archive_files( + display_name, + input_files, + s3_bucket, + s3_path) + + if status == 0: + self.num_files += 1 + self.size_mb += file_size_mb + self.archive_time += time.time() - start_time + + return status, message + + @staticmethod + def _update_archive_file_wkr(queue, logger): + """ Worker thread: Update the archival JSON file from 'queue'. """ + archival_json = [] + while True: + archive_args = queue.get() + # Exit worker thread when sentinel is received. + if archive_args is None: + queue.task_done() + break + archival_record = { + "name": archive_args.display_name, + "link": archive_args.remote_file, + "visibility": "private" + } + logger.debug( + "Updating archive file %s with %s", archive_args.archival_file, archival_record) + archival_json.append(archival_record) + with open(archive_args.archival_file, "w") as archival_fh: + json.dump(archival_json, archival_fh) + queue.task_done() + + @staticmethod + def _upload_to_s3_wkr(queue, archive_file_queue, logger, s3_client): + """" Worker thread: Upload to S3 from 'queue', dispatch to 'archive_file_queue'. """ + while True: + upload_args = queue.get() + # Exit worker thread when sentinel is received. + if upload_args is None: + queue.task_done() + archive_file_queue.put(None) + break + extra_args = {"ContentType": upload_args.content_type, "ACL": "public-read"} + logger.debug("Uploading to S3 %s to bucket %s path %s", + upload_args.local_file, + upload_args.s3_bucket, + upload_args.s3_path) + upload_completed = False + try: + s3_client.upload_file(upload_args.local_file, + upload_args.s3_bucket, + upload_args.s3_path, + ExtraArgs=extra_args) + upload_completed = True + logger.debug("Upload to S3 completed for %s to bucket %s path %s", + upload_args.local_file, + upload_args.s3_bucket, + upload_args.s3_path) + except Exception as err: + logger.exception("Upload to S3 error %s", err) + + if upload_args.delete_file: + status, message = remove_file(upload_args.local_file) + if status: + logger.error("Upload to S3 delete file error %s", message) + + remote_file = "https://s3.amazonaws.com/{}/{}".format( + upload_args.s3_bucket, upload_args.s3_path) + if upload_completed: + archive_file_queue.put(ArchiveArgs( + upload_args.archival_file, upload_args.display_name, remote_file)) + + queue.task_done() + + def _archive_files(self, display_name, input_files, s3_bucket, s3_path): + """ + Gather 'input_files' into a single tar/gzip and archive to 's3_path'. + + The caller waits until the list of files has been tar/gzipped to a temporary file. + The S3 upload and subsequent update to 'archival_json_file' will be done asynchronosly. + + Returns status, message and size_mb of archive. + """ + + # Parameter 'input_files' can either be a string or list of strings. + if isinstance(input_files, str): + input_files = [input_files] + + message = "Tar/gzip {} files: {}".format(display_name, input_files) + status = 0 + size_mb = 0 + + # Tar/gzip to a temporary file. + _, temp_file = tempfile.mkstemp(suffix=".tgz") + + # Check if there is sufficient space for the temporary tgz file. + if file_list_size(input_files) > free_space(temp_file): + status, message = remove_file(temp_file) + if status: + self.logger.warning("Removing tarfile due to insufficient space - %s", message) + return 1, "Insufficient space for {}".format(message), 0 + + try: + with tarfile.open(temp_file, "w:gz") as tar_handle: + for input_file in input_files: + try: + tar_handle.add(input_file) + except (IOError, OSError, tarfile.TarError) as err: + message = "{}; Unable to add {} to archive file: {}".format( + message, input_file, err) + except (IOError, OSError, tarfile.TarError) as err: + status, message = remove_file(temp_file) + if status: + self.logger.warning("Removing tarfile due to creation failure - %s", message) + return 1, str(err), 0 + + # Round up the size of the archive. + size_mb = int(math.ceil(float(file_list_size(temp_file)) / (1024 * 1024))) + self._upload_queue.put(UploadArgs( + self.archival_json_file, + display_name, + temp_file, + "application/x-gzip", + s3_bucket, + s3_path, + True)) + + return status, message, size_mb + + def check_thread(self, thread, expected_alive): + if thread.isAlive() and not expected_alive: + self.logger.warning( + "The %s thread did not complete, some files might not have been uploaded" + " to S3 or archived to %s.", thread.name, self.archival_json_file) + elif not thread.isAlive() and expected_alive: + self.logger.warning( + "The %s thread is no longer running, some files might not have been uploaded" + " to S3 or archived to %s.", thread.name, self.archival_json_file) + + def exit(self, timeout=30): + """ Waits for worker threads to finish. """ + # Put sentinel on upload queue to trigger worker thread exit. + self._upload_queue.put(None) + self.check_thread(self._upload_worker, True) + self.check_thread(self._archive_file_worker, True) + self._upload_worker.join(timeout=timeout) + self.check_thread(self._upload_worker, False) + + # Archive file worker thread exit should be triggered by upload thread worker. + self._archive_file_worker.join(timeout=timeout) + self.check_thread(self._archive_file_worker, False) + + self.logger.info("Total tar/gzip archive time is %0.2f seconds, for %d file(s) %d MB", + self.archive_time, self.num_files, self.size_mb) + + def files_archived_num(self): + """ Returns the number of the archived files. """ + return self.num_files + + def files_archived_size_mb(self): + """ Returns the size of the archived files. """ + return self.size_mb diff --git a/buildscripts/tests/resmokelib/test_archival.py b/buildscripts/tests/resmokelib/test_archival.py new file mode 100644 index 00000000000..bc36cdb4832 --- /dev/null +++ b/buildscripts/tests/resmokelib/test_archival.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python + +""" Unit tests for archival. """ + +import logging +import os +import random +import shutil +import sys +import tempfile +import unittest + + +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.getcwd()) +from buildscripts.resmokelib.utils import archival + +_BUCKET = "mongodatafiles" + + +def create_random_file(file_name, num_chars_mb): + """ Creates file with random characters, which will have minimal compression. """ + with open(file_name, "wb") as fileh: + for _ in xrange(num_chars_mb * 1024 * 1024): + fileh.write(chr(random.randint(0, 255))) + + +class MockS3Client(object): + """ Class to mock the S3 client. """ + + def __init__(self, logger): + self.logger = logger + self.logger.info("MockS3Client init") + return + + def upload_file(self, *args, **kwargs): + self.logger.info("MockS3Client upload_file %s %s", args, kwargs) + return + + def delete_object(self, *args, **kwargs): + self.logger.info("MockS3Client delete_object %s %s", args, kwargs) + return + + +class ArchivalTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + logging.basicConfig() + cls.logger = logging.getLogger() + cls.logger.setLevel(logging.INFO) + cls.bucket = _BUCKET + cls.temp_dir = tempfile.mkdtemp() + cls.remote_paths = [] + + # We can use the AWS S3 calls by setting the environment variable MOCK_CLIENT to 0. + mock_client = os.environ.get("MOCK_CLIENT", "1") != "0" + if mock_client: + cls.s3_client = MockS3Client(cls.logger) + else: + cls.s3_client = archival.Archival._get_s3_client() + cls.archive = cls.create_archival() + + @classmethod + def tearDownClass(cls): + cls.archive.exit() + cls.logger.info("Cleaning temp directory %s", cls.temp_dir) + shutil.rmtree(cls.temp_dir, ignore_errors=True) + for path in cls.remote_paths: + cls.delete_s3_file(path) + + @classmethod + def create_archival(cls): + return archival.Archival(cls.logger, s3_client=cls.s3_client) + + @classmethod + def delete_s3_file(cls, path): + cls.logger.info("Cleaning S3 bucket %s path %s", cls.bucket, path) + cls.s3_client.delete_object(Bucket=cls.bucket, Key=path) + + def s3_path(self, path, add_remote_path=True): + if add_remote_path: + self.remote_paths.append(path) + return path + + +class ArchivalFileTests(ArchivalTestCase): + def test_nofile(self): + # Invalid input_files + display_name = "Unittest invalid file" + input_files = "no_file" + s3_path = self.s3_path("unittest/no_file.tgz", False) + self.assertRaises( + OSError, + lambda: self.archive.archive_files_to_s3( + display_name, input_files, self.bucket, s3_path)) + + # Invalid input_files in a list + input_files = ["no_file", "no_file2"] + s3_path = self.s3_path("unittest/no_files.tgz", False) + self.assertRaises( + OSError, + lambda: self.archive.archive_files_to_s3( + display_name, input_files, self.bucket, s3_path)) + + # No files + display_name = "Unittest no files" + s3_path = self.s3_path("unittest/no_files.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, [], self.bucket, s3_path) + self.assertEqual(1, status, message) + + def test_files(self): + # Valid files + display_name = "Unittest valid file" + temp_file = tempfile.mkstemp(dir=self.temp_dir)[1] + s3_path = self.s3_path("unittest/valid_file.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + + # 2 valid files + display_name = "Unittest 2 valid files" + temp_file2 = tempfile.mkstemp(dir=self.temp_dir)[1] + s3_path = self.s3_path("unittest/2valid_files.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, [temp_file, temp_file2], self.bucket, s3_path) + self.assertEqual(0, status, message) + + def test_empty_directory(self): + # Valid directory + display_name = "Unittest valid directory no files" + temp_dir = tempfile.mkdtemp(dir=self.temp_dir) + s3_path = self.s3_path("unittest/valid_directory.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_dir, self.bucket, s3_path) + self.assertEqual(0, status, message) + + display_name = "Unittest valid directories no files" + temp_dir2 = tempfile.mkdtemp(dir=self.temp_dir) + s3_path = self.s3_path("unittest/valid_directories.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, [temp_dir, temp_dir2], self.bucket, s3_path) + self.assertEqual(0, status, message) + + def test_directory(self): + display_name = "Unittest directory with files" + temp_dir = tempfile.mkdtemp(dir=self.temp_dir) + s3_path = self.s3_path("unittest/directory_with_files.tgz") + # Create 10 empty files + for _ in xrange(10): + tempfile.mkstemp(dir=temp_dir) + status, message = self.archive.archive_files_to_s3( + display_name, temp_dir, self.bucket, s3_path) + self.assertEqual(0, status, message) + + display_name = "Unittest 2 valid directory files" + temp_dir2 = tempfile.mkdtemp(dir=self.temp_dir) + s3_path = self.s3_path("unittest/directories_with_files.tgz") + # Create 10 empty files + for _ in xrange(10): + tempfile.mkstemp(dir=temp_dir2) + status, message = self.archive.archive_files_to_s3( + display_name, [temp_dir, temp_dir2], self.bucket, s3_path) + self.assertEqual(0, status, message) + + +class ArchivalLimitSizeTests(ArchivalTestCase): + @classmethod + def create_archival(cls): + return archival.Archival(cls.logger, limit_size_mb=5, s3_client=cls.s3_client) + + def test_limit_size(self): + + # Files within limit size + display_name = "Unittest under limit size" + temp_file = tempfile.mkstemp(dir=self.temp_dir)[1] + create_random_file(temp_file, 3) + s3_path = self.s3_path("unittest/valid_limit_size.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + # Note the size limit is enforced after the file uploaded. Subsequent + # uploads will not be permitted, once the limit has been reached. + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + + # Files beyond limit size + display_name = "Unittest over limit size" + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(1, status, message) + + +class ArchivalLimitFileTests(ArchivalTestCase): + @classmethod + def create_archival(cls): + return archival.Archival(cls.logger, limit_files=3, s3_client=cls.s3_client) + + def test_limit_file(self): + + # Files within limit number + display_name = "Unittest under limit number" + temp_file = tempfile.mkstemp(dir=self.temp_dir)[1] + s3_path = self.s3_path("unittest/valid_limit_number.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + + # Files beyond limit number + display_name = "Unittest over limit number" + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(1, status, message) + + +class ArchivalLimitTests(ArchivalTestCase): + @classmethod + def create_archival(cls): + return archival.Archival( + cls.logger, limit_size_mb=3, limit_files=3, s3_client=cls.s3_client) + + def test_limits(self): + + # Files within limits + display_name = "Unittest under limits" + temp_file = tempfile.mkstemp(dir=self.temp_dir)[1] + create_random_file(temp_file, 1) + s3_path = self.s3_path("unittest/valid_limits.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + + # Files beyond limits + display_name = "Unittest over limits" + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(1, status, message) + + +if __name__ == "__main__": + unittest.main() diff --git a/buildscripts/tests/resmokelib/utils/__init__.py b/buildscripts/tests/resmokelib/utils/__init__.py new file mode 100644 index 00000000000..4b7a2bb941b --- /dev/null +++ b/buildscripts/tests/resmokelib/utils/__init__.py @@ -0,0 +1 @@ +"""Empty.""" diff --git a/buildscripts/tests/resmokelib/utils/test_rmtree.py b/buildscripts/tests/resmokelib/utils/test_rmtree.py new file mode 100644 index 00000000000..1908395766c --- /dev/null +++ b/buildscripts/tests/resmokelib/utils/test_rmtree.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +""" Unit tests for utils.rmtree. """ + +from __future__ import absolute_import +from __future__ import print_function + +import os +import shutil +import sys +import tempfile +import unittest + +from buildscripts.resmokelib import utils + +# pylint: disable=missing-docstring,protected-access + + +def rmtree(dir_root): + """Invoke utils.rmtree(dir_root) and return True if removed.""" + utils.rmtree(dir_root) + return not os.path.exists(dir_root) + + +def create_file(path): + """Create file named 'path'.""" + with open(path, "w") as fh: + fh.write("") + + +def ascii_filesystemencoding(): + """Return True if the file system encoding is type ASCII. + + See https://www.iana.org/assignments/character-sets/character-sets.xhtml. + """ + encoding = sys.getfilesystemencoding() + return encoding.startswith("ANSI_X3.4") or encoding == "US-ASCII" + + +def string_for_ascii_filesystem_encoding(path): + """Return encoded string type for unicode on ASCII file system encoding. + + Some file system encodings are set to ASCII if LANG=C or LC_ALL=C is specified. + """ + if ascii_filesystemencoding() and isinstance(path, unicode): + return path.encode("utf-8") + return path + + +class RmtreeTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp_dir_root = tempfile.mkdtemp() + cls.cwd = os.getcwd() + + @classmethod + def tearDownClass(cls): + os.chdir(cls.cwd) + shutil.rmtree(cls.temp_dir_root, ignore_errors=True) + + def do_test(self, name): + pass + + def test_ascii(self): + # ASCII name + self.do_test("ascii") + + def test_unicode(self): + # Unicode name + self.do_test(u"unicode") + + def test_greek(self): + # Name with Greek + self.do_test(string_for_ascii_filesystem_encoding(u"ελληνικά")) + + def test_japanese(self): + # Name with Japanese + self.do_test(string_for_ascii_filesystem_encoding(u"会社案")) + + +class RmtreeFileTests(RmtreeTestCase): + def do_test(self, file_name): # pylint: disable=arguments-differ + """Execute file test.""" + temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root) + os.chdir(temp_dir) + create_file(file_name) + os.chdir(self.temp_dir_root) + self.assertTrue(rmtree(temp_dir)) + + +class RmtreeDirectoryTests(RmtreeTestCase): + def do_test(self, dir_name): # pylint: disable=arguments-differ + """Execute directory test.""" + os.chdir(self.temp_dir_root) + os.mkdir(dir_name) + self.assertTrue(rmtree(dir_name)) + + +class RmtreeDirectoryWithNonAsciiTests(RmtreeTestCase): + def do_test(self, name): + """Execute directory with non-ASCII file test.""" + os.chdir(self.temp_dir_root) + os.mkdir(name) + os.chdir(name) + create_file(name) + os.chdir(self.temp_dir_root) + self.assertTrue(rmtree(name)) + + +class ShutilWindowsRmtreeFileTests(RmtreeFileTests): + def do_test(self, file_name): + """Execute file test that are known to fail in shutil.rmtree.""" + if not utils.is_windows(): + print("Skipping ShutilWindowsRmtreeFileTests on non-Windows platforms") + return + temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root) + os.chdir(temp_dir) + create_file(file_name) + os.chdir(self.temp_dir_root) + with self.assertRaises(WindowsError): # pylint: disable=undefined-variable + shutil.rmtree(temp_dir) + + def test_ascii(self): + pass + + def test_unicode(self): + pass diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 53df64a5c49..5992bb0f57b 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -147,9 +147,7 @@ variables: # create separate symbol files is commented out. - &debug_symbol_variants - amazon - - debian71 - debian81 - - enterprise-debian71-64 - enterprise-debian81-64 - enterprise-linux-64-amazon-ami - enterprise-osx-108 @@ -180,7 +178,6 @@ variables: - linux-64-debug - linux-64-duroff - linux-64-ephemeralForTest - - linux-64-lsm - linux-64-repeated-execution - osx-108 - osx-108-debug @@ -344,7 +341,7 @@ functions: sleep 60 done - "setup credentials" : &setup_credentials + "set up credentials" : &set_up_credentials command: shell.exec params: working_dir: src @@ -358,6 +355,53 @@ functions: build_phase='${task_name}_${execution}' END_OF_CREDS + "set up remote credentials": &set_up_remote_credentials + command: shell.exec + params: + silent: true + script: | + set -o errexit + + # Since the macros 'private_key_remote' and 'private_key_file' are not always defined + # we default to /dev/null to avoid syntax errors of an empty expansion. + if [ ! -z "${private_key_remote}" ] && [ ! -z "${private_key_file}" ] ; then + mkdir -p ~/.ssh + echo -n "${private_key_remote}" > ${private_key_file|/dev/null} + chmod 0600 ${private_key_file|/dev/null} + fi + + if [ ! -d ~.aws ]; then + mkdir -p ~/.aws + fi + + # If ${aws_profile_remote} is not specified then the config & credentials are + # stored in the 'default' profile. + aws_profile="${aws_profile_remote|default}" + + # The profile in the config file is specified as [profile <profile>], except + # for [default], see http://boto3.readthedocs.io/en/latest/guide/configuration.html + if [ $aws_profile = "default" ] ; then + aws_profile_config="[default]" + else + aws_profile_config="[profile $aws_profile]" + fi + cat <<EOF >> ~/.aws/config + $aws_profile_config + region = us-east-1 + EOF + + # The profile in the credentials file is specified as [<profile>]. + cat <<EOF >> ~/.aws/credentials + [$aws_profile] + aws_access_key_id = ${aws_key_remote} + aws_secret_access_key = ${aws_secret_remote} + EOF + + cat <<EOF > ~/.boto + [Boto] + https_validate_certificates = False + EOF + "upload debugsymbols" : &upload_debugsymbols command: s3.put params: @@ -430,7 +474,7 @@ functions: - *extract_binaries - *check_binary_version - *get_buildnumber - - *setup_credentials + - *set_up_credentials - *run_diskstats - *monitor_process_threads @@ -438,115 +482,138 @@ functions: command: shell.exec type: test params: - working_dir: src script: | # exit immediately if virtualenv is not found set -o errexit set -o verbose - if [ ${build_variant} == enterprise-suse12-64 ]; then - virtualenv ./venv - else - virtualenv --system-site-packages ./venv + python_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2}) + if [ "Windows_NT" = "$OS" ]; then + python_loc=$(cygpath -w $python_loc) fi + # Set up virtualenv in ${workdir} + virtualenv --python "$python_loc" --system-site-packages "${workdir}/venv" "run tests" : - command: shell.exec - type: test - params: - working_dir: src - script: | - set -o errexit - set -o verbose + - command: expansions.update + params: + updates: + - key: aws_key_remote + value: ${mongodatafiles_aws_key} + - key: aws_profile_remote + value: mongodata_aws + - key: aws_secret_remote + value: ${mongodatafiles_aws_secret} + - *set_up_remote_credentials + - command: shell.exec + type: test + params: + working_dir: src + script: | + set -o errexit + set -o verbose - # Set the TMPDIR environment variable to be a directory in the task's working - # directory so that temporary files created by processes spawned by resmoke.py get - # cleaned up after the task completes. This also ensures the spawned processes - # aren't impacted by limited space in the mount point for the /tmp directory. - export TMPDIR="${workdir}/tmp" - mkdir -p $TMPDIR + # Set the TMPDIR environment variable to be a directory in the task's working + # directory so that temporary files created by processes spawned by resmoke.py get + # cleaned up after the task completes. This also ensures the spawned processes + # aren't impacted by limited space in the mount point for the /tmp directory. + export TMPDIR="${workdir}/tmp" + mkdir -p $TMPDIR - # check if virtualenv is set up - if [ -d "venv" ]; then - if [ "Windows_NT" = "$OS" ]; then - . ./venv/Scripts/activate - else - . ./venv/bin/activate + ${activate_virtualenv} + pip install boto3==1.5.27 + + if [ -f /proc/self/coredump_filter ]; then + # Set the shell process (and its children processes) to dump ELF headers (bit 4), + # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). + echo 0x13 > /proc/self/coredump_filter + + if [ -f /sbin/sysctl ]; then + # Check that the core pattern is set explicitly on our distro image instead + # of being the OS's default value. This ensures that coredump names are consistent + # across distros and can be picked up by Evergreen. + core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") + if [ "$core_pattern" = "dump_%e.%p.core" ]; then + echo "Enabling coredumps" + ulimit -c unlimited + fi + fi fi - fi - if [ -f /proc/self/coredump_filter ]; then - # Set the shell process (and its children processes) to dump ELF headers (bit 4), - # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). - echo 0x13 > /proc/self/coredump_filter - - if [ -f /sbin/sysctl ]; then - # Check that the core pattern is set explicitly on our distro image instead - # of being the OS's default value. This ensures that coredump names are consistent - # across distros and can be picked up by Evergreen. - core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") - if [ "$core_pattern" = "dump_%e.%p.core" ]; then - echo "Enabling coredumps" - ulimit -c unlimited + extra_args="" + if [ ${run_multiple_jobs|false} = true ]; then + processor_architecture=$(uname -m) + num_jobs_available=${num_jobs_available|1} + # Reduce the number of available jobs by half when running any sharding*, replica_sets* + # and select jsCore passthrough tasks on Windows and ARM to avoid overwhelming test hosts. + if [ "Windows_NT" = "$OS" ] || [ "aarch64" = $processor_architecture ]; then + case "${task_name}" in + replica_sets_initsync_jscore_passthrough \ + |replica_sets_initsync_static_jscore_passthrough \ + |replica_sets_resync_static_jscore_passthrough \ + |replica_sets* \ + |sharding* \ + ) + num_jobs_available=$((${num_jobs_available|2} / 2)) + echo "Reducing jobs from ${num_jobs_available|1} to $num_jobs_available" + ;; + esac fi - fi - fi - extra_args="" - if [ ${run_multiple_jobs|false} = true ]; then - processor_architecture=$(uname -m) - num_jobs_available=${num_jobs_available|1} - # Reduce the number of available jobs by half when running any sharding*, replica_sets* - # and select jsCore passthrough tasks on Windows and ARM to avoid overwhelming test hosts. - if [ "Windows_NT" = "$OS" ] || [ "aarch64" = $processor_architecture ]; then - case "${task_name}" in - replica_sets_initsync_jscore_passthrough \ - |replica_sets_initsync_static_jscore_passthrough \ - |replica_sets_resync_static_jscore_passthrough \ - |replica_sets* \ - |sharding* \ - ) - num_jobs_available=$((${num_jobs_available|2} / 2)) - echo "Reducing jobs from ${num_jobs_available|1} to $num_jobs_available" - ;; - esac + if [ ${max_jobs|0} -gt 0 ] && [ ${max_jobs|0} -lt $num_jobs_available ]; then + extra_args="$extra_args --jobs=${max_jobs}" + else + extra_args="$extra_args --jobs=$num_jobs_available" + fi fi - if [ ${max_jobs|0} -gt 0 ] && [ ${max_jobs|0} -lt $num_jobs_available ]; then - extra_args="$extra_args --jobs=${max_jobs}" - else - extra_args="$extra_args --jobs=$num_jobs_available" + if [ ${should_shuffle|true} = true ]; then + extra_args="$extra_args --shuffle" fi - fi - if [ ${should_shuffle|true} = true ]; then - extra_args="$extra_args --shuffle" - fi - - if [ ${continue_on_failure|true} = true ]; then - extra_args="$extra_args --continueOnFailure" - fi + if [ ${continue_on_failure|true} = true ]; then + extra_args="$extra_args --continueOnFailure" + fi - # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed. - # resmoke will assign to the appropriate parameter on storage engines that support it. - set +o errexit - echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB - if [ $? -eq 1 ]; then - extra_args="$extra_args --storageEngineCacheSizeGB=1" - fi - set -o errexit + # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed. + # resmoke will assign to the appropriate parameter on storage engines that support it. + set +o errexit + echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB + if [ $? -eq 1 ]; then + extra_args="$extra_args --storageEngineCacheSizeGB=1" + fi + set -o errexit - # Combine the exclusion tags from both the variant and the task. - extra_args="$extra_args --excludeWithAnyTags=${task_excluded_tags},${variant_excluded_flags}" + # Combine the exclusion tags from both the variant and the task. + extra_args="$extra_args --excludeWithAnyTags=${task_excluded_tags},${variant_excluded_flags}" - path_value="$PATH" - if [ ${variant_path_suffix} ]; then - path_value="$path_value:${variant_path_suffix}" - fi - if [ ${task_path_suffix} ]; then - path_value="$path_value:${task_path_suffix}" - fi + path_value="$PATH" + if [ ${variant_path_suffix} ]; then + path_value="$path_value:${variant_path_suffix}" + fi + if [ ${task_path_suffix} ]; then + path_value="$path_value:${task_path_suffix}" + fi - ${resmoke_wrapper} PATH="$path_value" ${san_symbolizer} ${lang_environment} ${san_options} ${rlp_environment} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py ${resmoke_args} $extra_args ${test_flags} --log=buildlogger --reportFile=report.json + ${resmoke_wrapper} \ + PATH="$path_value" \ + AWS_PROFILE=${aws_profile_remote} \ + ${san_symbolizer} \ + ${lang_environment} \ + ${san_options} \ + ${rlp_environment} \ + $python buildscripts/resmoke.py \ + ${resmoke_args} \ + $extra_args ${test_flags} \ + --log=buildlogger \ + --taskId=${task_id} \ + --taskName=${task_name} \ + --executionNumber=${execution} \ + --projectName=${project} \ + --variantName=${build_variant} \ + --distroId=${distro_id} \ + --gitRevision=${revision} \ + --archiveFile=archive.json \ + --reportFile=report.json "do jepsen setup" : @@ -812,28 +879,6 @@ functions: kill_process "$process" done - "set up kitchen credentials": - command: shell.exec - params: - silent: true - script: | - set -o errexit - - mkdir -p ~/.ssh ~/.aws - echo -n "${kitchen_private_key}" > ~/.ssh/kitchen.pem - chmod 0600 ~/.ssh/kitchen.pem - - cat <<EOF > ~/.aws/config - [default] - region = us-east-1 - EOF - - cat <<EOF > ~/.aws/credentials - [default] - aws_access_key_id = ${kitchen_aws_key} - aws_secret_access_key = ${kitchen_aws_secret} - EOF - "run kitchen": command: shell.exec type: test @@ -843,13 +888,6 @@ functions: script: | set -o errexit - # Override the aws credentials with the kitchen specific credentials - cat <<EOF > ~/.aws/credentials - [default] - aws_access_key_id = ${kitchen_aws_key} - aws_secret_access_key = ${kitchen_aws_secret} - EOF - export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}.tgz" export KITCHEN_SECURITY_GROUP="${kitchen_security_group}" export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}" @@ -875,9 +913,28 @@ pre: - command: shell.track - func: "kill processes" - func: "cleanup environment" + - func: "set up virtualenv" - command: expansions.update params: updates: + - key: activate_virtualenv + value: | + # check if virtualenv is set up + if [ -d "${workdir}/venv" ]; then + if [ "Windows_NT" = "$OS" ]; then + # Need to quote the path on Windows to preserve the separator. + . "${workdir}/venv/Scripts/activate" 2> /tmp/activate_error.log + else + . ${workdir}/venv/bin/activate 2> /tmp/activate_error.log + fi + if [ $? -ne 0 ]; then + echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)" + fi + python=python + else + python=${python|/opt/mongodbtoolchain/v2/bin/python2} + fi + echo "python set to $(which $python)" - key: set_sudo value: | set -o | grep errexit | grep on @@ -917,6 +974,11 @@ post: - command: attach.results params: file_location: src/report.json + - command: attach.artifacts + params: + ignore_artifacts_for_spawn: false + files: + - src/archive.json - func: "kill processes" # Print out any Out of Memory killed process messages. - command: shell.exec @@ -944,32 +1006,6 @@ post: echo "No OOM (Out of memory) killed processes detected" fi - # Gather and archive FTDC data. - - command: shell.exec - params: - working_dir: src - script: | - # Using shell and tar to recurse properly to all possible diagnostic.data subdirectories. - # The archive.targz_pack command is not being used here because the command's glob support - # did not allow us to gather all directories. - if [ -d /data/db ]; then - file_list=$(cd /data/db && find . -type d -name diagnostic.data) - if [ -n "$file_list" ]; then - ${tar|tar} cvzf diagnostic-data.tgz -C /data/db $file_list - fi - fi - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/diagnostic-data.tgz - remote_file: mongodb-mongo-v3.4/${build_variant}/${revision}/ftdc/mongo-diagnostic-data-${task_id}-${execution}.tgz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: FTDC Diagnostic Data - Execution ${execution} - optional: true - # Process and save coverage data. - command: shell.exec params: @@ -1225,7 +1261,7 @@ tasks: params: directory: src - func: "get buildnumber" - - func: "setup credentials" + - func: "set up credentials" - func: "build new tools" # noop if ${newtools} is not "true" - command: shell.exec params: @@ -1376,7 +1412,7 @@ tasks: params: directory: src - func: "get buildnumber" - - func: "setup credentials" + - func: "set up credentials" - func: "build new tools" # noop if ${newtools} is not "true" - *generate_compile_expansions # Then we load the generated version data into the agent so we can use it in task definitions. @@ -1435,15 +1471,11 @@ tasks: enterprise: ${enterprise_rev} - command: shell.exec - func: "do setup" - - func: "set up virtualenv" - command: shell.exec params: working_dir: burn_in_tests_clonedir script: | set -o errexit - # Create a symbolic link to the venv in the src directory so activate_virtualenv can use it - # if it exists. - ln -s ../src/venv venv ${activate_virtualenv} set -o verbose # If this is a scheduled build, we check for changes against the last scheduled commit. @@ -1454,7 +1486,7 @@ tasks: # list of dbtest suites. cp ../src/dbtest${exe} . # Capture a list of new and modified tests. - ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=${build_variant} --testListOutfile=jstests/new_tests.json --noExec $burn_in_args + $python buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=${build_variant} --testListOutfile=jstests/new_tests.json --noExec $burn_in_args # Copy the results to the src dir. cp jstests/new_tests.json ../src/jstests/new_tests.json - func: "do multiversion setup" @@ -1895,7 +1927,6 @@ tasks: name: ese_WT commands: - func: "do setup" - - func: "set up virtualenv" - command: shell.exec type: test params: @@ -1904,11 +1935,7 @@ tasks: set -o errexit set -o verbose - if [ "Windows_NT" = "$OS" ]; then - . ./venv/Scripts/activate - else - . ./venv/bin/activate - fi + ${activate_virtualenv} # we should go back to using pip when it works on SLES easy_install --upgrade `cat src/mongo/db/modules/enterprise/jstests/encryptdb/libs/requirements.txt` - func: "run tests" @@ -3033,7 +3060,12 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - func: "set up kitchen credentials" + - func: "set up remote credentials" + vars: + private_key_file: ~/.ssh/kitchen.pem + private_key_remote: ${kitchen_private_key} + aws_key_remote: ${kitchen_aws_key} + aws_secret_remote: ${kitchen_aws_secret} - func: "run kitchen" - name: push @@ -3105,7 +3137,7 @@ tasks: /usr/local/bin/notary-client.py --key-name "server-3.4" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} mongodb-win32-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${ext|tar.gz} if [ "${has_packages|}" = "true" ]; then - CURATOR_RELEASE="ea8d75dcc1a587111e7418e2428fb67e267af9fe" + CURATOR_RELEASE=${curator_release|"ea8d75dcc1a587111e7418e2428fb67e267af9fe"} curl -L -O http://boxes.10gen.com/build/curator/curator-dist-rhel70-$CURATOR_RELEASE.tar.gz tar -zxvf curator-dist-rhel70-$CURATOR_RELEASE.tar.gz ./curator repo --config ./etc/repo_config.yaml --distro ${packager_distro} --edition ${repo_edition} --version ${version} --arch ${packager_arch} --packages repo @@ -3859,65 +3891,6 @@ buildvariants: - name: sharding_jscore_passthrough - name: tool -- name: linux-64-lsm - stepback: false - display_name: Linux LSM - run_on: - - rhel62-small - batchtime: 1440 # 1 day - expansions: - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy - num_jobs_available: $(grep -c ^processor /proc/cpuinfo) - test_flags: --wiredTigerCollectionConfig=type=lsm --wiredTigerIndexConfig=type=lsm - use_scons_cache: true - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' - tooltags: "" - build_mongoreplay: true - tasks: - - name: compile - distros: - - rhel62-large - - name: compile_all - distros: - - rhel62-large - - name: aggregation_WT - - name: auth_WT - - name: dbtest_WT - - name: gle_auth_WT - - name: gle_auth_write_cmd_WT - - name: gle_auth_basics_passthrough_WT - - name: gle_auth_basics_passthrough_write_cmd_WT - - name: sharding_gle_auth_basics_passthrough_WT - - name: sharding_gle_auth_basics_passthrough_write_cmd_WT - - name: jsCore_WT - - name: jsCore_compatibility_WT - - name: jstestfuzz_WT - - name: jstestfuzz_concurrent_WT - - name: jstestfuzz_concurrent_replication_WT - - name: jstestfuzz_concurrent_sharded_WT - - name: jstestfuzz_replication_WT - - name: jstestfuzz_sharded_WT - - name: noPassthrough_WT - - name: noPassthroughWithMongod_WT - - name: bulk_gle_passthrough_WT - - name: parallel_WT - - name: parallel_compatibility_WT - - name: concurrency_WT - - name: concurrency_replication_WT - - name: concurrency_sharded_WT - - name: concurrency_simultaneous_WT - - name: replica_sets_WT - - name: replica_sets_jscore_passthrough_WT - - name: master_slave_WT - - name: master_slave_jscore_passthrough_WT - - name: sharding_WT - - name: slow1_WT - - name: serial_run_WT - - name: sharded_collections_jscore_passthrough_WT - - name: sharding_jscore_passthrough_WT - - name: sharding_jscore_passthrough_wire_ops_WT - - name: tool_WT - - name: ubuntu1404 display_name: SSL Ubuntu 14.04 run_on: @@ -5467,6 +5440,7 @@ buildvariants: push_arch: x86_64 compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.8 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include num_jobs_available: 1 + variant_excluded_flags: requires_ssl_mongo_tools gorootvars: 'PATH="/usr/local/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/usr/local/go1.8/go CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.8 CGO_LDFLAGS=-mmacosx-version-min=10.8' tooltags: -tags 'ssl openssl_pre_1.0' build_mongoreplay: true @@ -5611,6 +5585,7 @@ buildvariants: push_arch: x86_64-enterprise compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.8 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include num_jobs_available: 1 + variant_excluded_flags: requires_ssl_mongo_tools gorootvars: 'PATH="/usr/local/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/usr/local/go1.8/go CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.8 CGO_LDFLAGS=-mmacosx-version-min=10.8' tooltags: -tags 'sasl ssl openssl_pre_1.0' build_mongoreplay: true @@ -5678,6 +5653,7 @@ buildvariants: - name: compile requires: - name: burn_in_tests + - name: lint distros: - rhel62-large - name: compile_all @@ -7152,66 +7128,6 @@ buildvariants: # Debian buildvariants # ########################################### -- name: enterprise-debian71-64 - display_name: Enterprise Debian 7.1 - modules: - - enterprise - run_on: - - debian71-test - batchtime: 1440 # 1 day - expansions: - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: x86_64-enterprise-debian71 - compile_flags: --ssl MONGO_DISTMOD=debian71 --release -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy - num_jobs_available: $(grep -c ^processor /proc/cpuinfo) - has_packages: true - packager_script: packager-enterprise.py - packager_arch: x86_64 - packager_distro: debian71 - repo_edition: enterprise - use_scons_cache: true - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' - tooltags: -tags 'sasl ssl' - build_mongoreplay: true - tasks: - - name: compile - distros: - - debian71-build - - name: compile_all - distros: - - debian71-build - - name: audit - - name: audit_WT - - name: dbtest - - name: dbtest_WT - - name: ese_WT - - name: jsCore - - name: jsCore_auth - - name: jstestfuzz - - name: jstestfuzz_WT - - name: jstestfuzz_concurrent_WT - - name: jstestfuzz_concurrent_replication_WT - - name: jstestfuzz_concurrent_sharded_WT - - name: jstestfuzz_replication - - name: jstestfuzz_replication_WT - - name: jstestfuzz_sharded - - name: jstestfuzz_sharded_WT - - name: mmap - - name: replica_sets_auth - - name: sasl - - name: sharding_auth - - name: snmp - - name: snmp_WT - - name: ssl - - name: sslSpecial - - name: package - distros: - - ubuntu1604-packer - - name: push - - - name: enterprise-debian81-64 display_name: Enterprise Debian 8.1 modules: @@ -7272,129 +7188,6 @@ buildvariants: - name: push -- name: debian71 - display_name: SSL Debian 7.1 - run_on: - - debian71-test - batchtime: 1440 # 1 day - expansions: - push_path: linux - push_bucket: downloads.mongodb.org - push_name: linux - push_arch: x86_64-debian71 - compile_flags: --ssl MONGO_DISTMOD=debian71 -j$(grep -c ^processor /proc/cpuinfo) --release CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy - multiversion_platform_arch: "debian71" - multiversion_edition: "targeted" - num_jobs_available: $(grep -c ^processor /proc/cpuinfo) - has_packages: true - packager_script: packager.py - packager_arch: x86_64 - packager_distro: debian71 - repo_edition: org - use_scons_cache: true - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' - tooltags: -tags 'ssl' - build_mongoreplay: true - tasks: - - name: compile - distros: - - debian71-build - - name: compile_all - distros: - - debian71-build - - name: aggregation - - name: aggregation_WT - - name: aggregation_auth - - name: auth - - name: auth_WT - - name: dbtest - - name: dbtest_WT - - name: disk - - name: durability - - name: dur_jscore_passthrough - - name: failpoints - - name: failpoints_auth - - name: gle_auth - - name: gle_auth_WT - - name: gle_auth_write_cmd - - name: gle_auth_write_cmd_WT - - name: gle_auth_basics_passthrough - - name: gle_auth_basics_passthrough_WT - - name: gle_auth_basics_passthrough_write_cmd - - name: gle_auth_basics_passthrough_write_cmd_WT - - name: sharding_gle_auth_basics_passthrough - - name: sharding_gle_auth_basics_passthrough_WT - - name: sharding_gle_auth_basics_passthrough_write_cmd - - name: sharding_gle_auth_basics_passthrough_write_cmd_WT - - name: jsCore - - name: jsCore_WT - - name: jsCore_auth - - name: jsCore_compatibility - - name: jsCore_compatibility_WT - - name: jstestfuzz - - name: jstestfuzz_WT - - name: jstestfuzz_concurrent_WT - - name: jstestfuzz_concurrent_replication_WT - - name: jstestfuzz_concurrent_sharded_WT - - name: jstestfuzz_replication - - name: jstestfuzz_replication_WT - - name: jstestfuzz_sharded - - name: jstestfuzz_sharded_WT - - name: mmap - - name: mongosTest - - name: multiversion - - name: multiversion_WT - - name: noPassthrough - - name: noPassthrough_WT - - name: noPassthroughWithMongod - - name: noPassthroughWithMongod_WT - - name: bulk_gle_passthrough - - name: bulk_gle_passthrough_WT - - name: parallel - - name: parallel_WT - - name: parallel_compatibility - - name: parallel_compatibility_WT - - name: concurrency - - name: concurrency_WT - - name: concurrency_replication - - name: concurrency_replication_WT - - name: concurrency_sharded - - name: concurrency_sharded_WT - - name: concurrency_simultaneous - - name: concurrency_simultaneous_WT - - name: replica_sets - - name: replica_sets_WT - - name: replica_sets_auth - - name: replica_sets_jscore_passthrough - - name: replica_sets_jscore_passthrough_WT - - name: master_slave - - name: master_slave_WT - - name: master_slave_auth - - name: master_slave_jscore_passthrough - - name: master_slave_jscore_passthrough_WT - - name: sharding - - name: sharding_WT - - name: sharding_auth - - name: sharding_last_stable_mongos_and_mixed_shards - - name: slow1 - - name: slow1_WT - - name: serial_run - - name: serial_run_WT - - name: sharded_collections_jscore_passthrough - - name: sharded_collections_jscore_passthrough_WT - - name: sharding_jscore_passthrough - - name: sharding_jscore_passthrough_WT - - name: sharding_jscore_passthrough_wire_ops_WT - - name: ssl - - name: sslSpecial - - name: tool - - name: tool_WT - - name: package - distros: - - ubuntu1604-packer - - name: push - - - name: debian81 display_name: SSL Debian 8.1 run_on: @@ -8221,141 +8014,6 @@ buildvariants: distros: - ubuntu1604-build -- name: enterprise-debug-rhel-62-cxx14 - display_name: ~ Enterprise RHEL 6.2 C++14 DEBUG - modules: - - enterprise - run_on: - - rhel62-small - batchtime: 1440 # 1 day - expansions: - rlp_environment: MONGOD_UNITTEST_RLP_LANGUAGE_TEST_BTROOT=/opt/basis - compile_flags: --allocator=system --dbg=on --opt=on --ssl -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy CPPPATH="/opt/basis/rlp/rlp/include /opt/basis/rlp/utilities/include" --use-basis-tech-rosette-linguistics-platform=on --cxx-std=14 - multiversion_platform_arch: "rhel62" - multiversion_edition: "targeted" - num_jobs_available: $(grep -c ^processor /proc/cpuinfo) - use_scons_cache: true - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' - tooltags: -tags 'sasl ssl' - build_mongoreplay: true - tasks: - - name: compile - distros: - - rhel62-large - - name: compile_all - distros: - - rhel62-large - - name: aggregation - - name: aggregation_WT - - name: aggregation_WT_ese - - name: aggregation_auth - - name: aggregation_read_concern_majority_passthrough_WT - - name: aggregation_sharded_collections_passthrough_WT - - name: audit - - name: audit_WT - - name: auth - - name: auth_WT - - name: auth_audit - - name: auth_audit_WT - - name: bulk_gle_passthrough - - name: bulk_gle_passthrough_WT - - name: concurrency - - name: concurrency_WT - - name: concurrency_replication - - name: concurrency_replication_WT - - name: concurrency_sharded - - name: concurrency_sharded_WT - - name: concurrency_simultaneous - - name: concurrency_simultaneous_WT - - name: dbtest - - name: dbtest_WT - - name: disk - - name: dur_jscore_passthrough - - name: durability - - name: ese_WT - - name: failpoints - - name: failpoints_auth - - name: gle_auth - - name: gle_auth_WT - - name: gle_auth_basics_passthrough - - name: gle_auth_basics_passthrough_WT - - name: gle_auth_basics_passthrough_write_cmd - - name: gle_auth_basics_passthrough_write_cmd_WT - - name: gle_auth_write_cmd - - name: gle_auth_write_cmd_WT - - name: httpinterface - - name: integration_tests_replset - - name: integration_tests_standalone - - name: jsCore - - name: jsCore_WT - - name: jsCore_WT_ese - - name: jsCore_auth - - name: jsCore_compatibility - - name: jsCore_compatibility_WT - - name: jsCore_minimum_batch_size_WT - - name: jsCore_op_query - - name: jsCore_op_query_WT - - name: mmap - - name: mongosTest - - name: multiversion - - name: multiversion_WT - - name: noPassthrough - - name: noPassthroughWithMongod - - name: noPassthroughWithMongod_WT - - name: noPassthrough_WT - - name: parallel - - name: parallel_WT - - name: parallel_compatibility - - name: parallel_compatibility_WT - - name: read_concern_majority_passthrough_WT - - name: read_only - - name: read_only_WT - - name: read_only_sharded - - name: read_only_sharded_WT - - name: replica_sets - - name: replica_sets_WT - - name: replica_sets_WT_ese - - name: replica_sets_auth - - name: replica_sets_pv0 - - name: replica_sets_jscore_passthrough - - name: replica_sets_jscore_passthrough_WT - - name: master_slave - - name: master_slave_WT - - name: master_slave_auth - - name: master_slave_jscore_passthrough - - name: master_slave_jscore_passthrough_WT - - name: rlp - - name: rlp_WT - - name: sasl - - name: sharded_collections_jscore_passthrough - - name: sharded_collections_jscore_passthrough_WT - - name: sharding - - name: sharding_WT - - name: sharding_WT_ese - - name: sharding_auth - - name: sharding_auth_audit_WT - - name: sharding_gle_auth_basics_passthrough - - name: sharding_gle_auth_basics_passthrough_WT - - name: sharding_gle_auth_basics_passthrough_write_cmd - - name: sharding_gle_auth_basics_passthrough_write_cmd_WT - - name: sharding_jscore_passthrough - - name: sharding_jscore_passthrough_WT - - name: sharding_jscore_passthrough_wire_ops_WT - - name: sharding_last_stable_mongos_and_mixed_shards - - name: sharding_op_query_WT - - name: sharding_csrs_continuous_config_stepdown_WT - - name: slow1 - - name: slow1_WT - - name: serial_run - - name: serial_run_WT - - name: snmp - - name: snmp_WT - - name: ssl - - name: sslSpecial - - name: tool - - name: tool_WT - - name: write_concern_majority_passthrough_WT - - name: initsync-3dot2-rhel-62 display_name: "~ Initial Sync 3.2 Enterprise RHEL 6.2" modules: diff --git a/etc/longevity.yml b/etc/longevity.yml index 9cdf9502fa7..c045bc83f62 100644 --- a/etc/longevity.yml +++ b/etc/longevity.yml @@ -175,19 +175,7 @@ functions: cat perf.json | egrep "name|ops_per_sec" "analyze": - - command: json.get_history - params: - task: ${task_name} - file: "work/history.json" - name: "perf" - - command: json.get_history - params: - tags: true - task: ${task_name} - file: "work/tags.json" - name: "perf" - command: shell.exec - # post_run_check.py and longevity_override.json for DSI tests are part of dsi repo type : test params: working_dir: work @@ -197,7 +185,7 @@ functions: TAG="3.2.1-Baseline" PROJECT="mongo-longevity" OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/longevity_override.json" - python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id $PROJECT --task_name ${task_name} --variant ${build_variant} + python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} --refTag $TAG --overrideFile $OVERRIDEFILE --project_id $PROJECT --task_name ${task_name} --variant ${build_variant} tasks: - name: compile diff --git a/etc/system_perf.yml b/etc/system_perf.yml index b278ca3b75a..930ced96aba 100644 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -16,7 +16,7 @@ post: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: work/dsi-artifacts.tgz - remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.${ext|tgz} + remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.${ext|tgz} bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} @@ -26,7 +26,7 @@ post: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/workloads/workloads/jsdoc/jsdocs-redirect.html - remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/workloads-${task_name}-${build_id}.html + remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/workloads-${task_name}-${build_id}.html bucket: mciuploads permissions: public-read content_type: text/html @@ -60,6 +60,85 @@ post: fi functions: + "compile mongodb": + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + # We get the raw version string (r1.2.3-45-gabcdef) from git + MONGO_VERSION=$(git describe) + # If we're going to compile the upstream wtdevelop repository for wiredtiger, add + # that githash to version string. + if [ "${wtdevelop|}" = "-wtdevelop" ]; then + WT_VERSION=$(cd src/third_party/wtdevelop; git describe | cut -c 9-) + MONGO_VERSION="$MONGO_VERSION-wtdevelop-$WT_VERSION" + fi + # If this is a patch build, we add the patch version id to the version string so we know + # this build was a patch, and which evergreen task it came from + if [ "${is_patch|}" = "true" ]; then + MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" + fi + + # This script converts the generated version string into a sanitized version string for + # use by scons and uploading artifacts as well as information about for the scons cache. + echo $MONGO_VERSION | USE_SCONS_CACHE=${use_scons_cache|false} ${python|python} buildscripts/generate_compile_expansions.py | tee compile_expansions.yml + # Then we load the generated version data into the agent so we can use it in task definitions + - command: expansions.update + params: + file: src/compile_expansions.yml + - command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + ${python|python} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} mongo${extension} mongod${extension} mongos${extension} MONGO_VERSION=${version} + mkdir -p mongodb/bin + mkdir -p mongodb/jstests/hooks + mv mongo${extension|} mongodb/bin + mv mongod${extension|} mongodb/bin + mv mongos${extension|} mongodb/bin + if [ -d jstests/hooks ] + then + echo "Fetching JS test DB correctness checks from directory jstests" + cp -a jstests/* mongodb/jstests + echo "Now adding our own special run_validate_collections.js wrapper" + mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js + cat << EOF > mongodb/jstests/hooks/run_validate_collections.js + print("NOTE: run_validate_collections.js will skip the oplog!"); + TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; + load('jstests/hooks/run_validate_collections.actual.js'); + EOF + fi + tar czf mongodb${wtdevelop|}.tar.gz mongodb + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/mongodb${wtdevelop|}.tar.gz + remote_file: ${project_dir}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: mongodb${wtdevelop|}.tar.gz + + # NOTE: Unlike evergreen.yml, there's no conditional here. If called, this is never a noop! + "use WiredTiger develop" : + command: shell.exec + params: + working_dir: src + script: | + set -o errexit + set -o verbose + cd src/third_party + for wtdir in api dist examples ext lang src test tools ; do + rm -rf wiredtiger/$wtdir + mv wtdevelop/$wtdir wiredtiger/ + done + "prepare environment": - command: shell.exec params: @@ -85,7 +164,7 @@ functions: storageEngine: ${storageEngine} test_control: ${test} production: true - mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz" + mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project_dir}/${version_id}/${revision}/${platform}/mongodb${wtdevelop|}-${version_id}.tar.gz" workloads_dir: ../src/workloads/workloads ycsb_dir: ../src/YCSB/YCSB EOF @@ -101,7 +180,7 @@ functions: version_id: ${version_id} workdir: ${workdir} revision: ${revision} - project: ${project} + project_dir: ${project_dir} branch_name: ${branch_name} # sys-perf expansions @@ -111,6 +190,7 @@ functions: dsi_rev: ${dsi_rev} workloads_rev: ${workloads_rev} EOF + - command: shell.exec params: silent: true @@ -127,6 +207,7 @@ functions: aws_secret_key: "${terraform_secret}" perf_jira_user: "${perf_jira_user}" perf_jira_pw: "${perf_jira_pw}" + dsi_analysis_atlas_pw: "${dsi_analysis_atlas_pw}" EOF chmod 400 runtime_secret.yml - command: shell.exec @@ -136,10 +217,7 @@ functions: # configure environment, has private information, no logging script: | set -e - virtualenv ./venv - source ./venv/bin/activate - pip install -r ../src/dsi/dsi/requirements.txt - python ../src/dsi/dsi/bin/bootstrap.py + ../src/dsi/dsi/run-dsi python ../src/dsi/dsi/bin/bootstrap.py - command: shell.exec params: script: | @@ -157,10 +235,9 @@ functions: set -e set -v source ./dsienv.sh - source ./venv/bin/activate - infrastructure_provisioning.py - workload_setup.py - mongodb_setup.py + ../src/dsi/dsi/run-dsi infrastructure_provisioning.py + ../src/dsi/dsi/run-dsi workload_setup.py + ../src/dsi/dsi/run-dsi mongodb_setup.py "run test": - command: shell.exec @@ -171,51 +248,28 @@ functions: set -e set -v source ./dsienv.sh - source ./venv/bin/activate - test_control.py + ../src/dsi/dsi/run-dsi test_control.py - command: "json.send" params: name: "perf" file: "work/perf.json" "analyze": - - command: json.get_history - params: - task: ${task_name} - file: "work/history.json" - name: "perf" - - command: json.get_history - params: - tags: true - task: ${task_name} - file: "work/tags.json" - name: "perf" - command: shell.exec - # generate dashboard data - type : test params: working_dir: work - silent: true script: | - set -o errexit - TAGS="3.2.17-Baseline 3.4.9-Baseline" - OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json" - python -u ../src/dsi/dsi/analysis/dashboard_gen.py --rev ${revision} -f history.json -t tags.json --refTag $TAGS --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} --jira-user ${perf_jira_user} --jira-password ${perf_jira_pw} || true - - command: "json.send" - params: - name: "dashboard" - file: "work/dashboard.json" + ../src/dsi/dsi/run-dsi detect-changes - command: shell.exec - # post_run_check.py and override.json for DSI tests are part of dsi repo type : test params: working_dir: work script: | set -o errexit set -o verbose - TAG="3.2.17-Baseline" + TAG="3.2.20-Baseline" OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json" - python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} + python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} ####################################### # Tasks # @@ -228,76 +282,13 @@ tasks: - command: git.get_project params: directory: src - - command: shell.exec - params: - working_dir: src - script: | - set -o errexit - set -o verbose - - # We get the raw version string (r1.2.3-45-gabcdef) from git - MONGO_VERSION=$(git describe) - # If this is a patch build, we add the patch version id to the version string so we know - # this build was a patch, and which evergreen task it came from - if [ "${is_patch|}" = "true" ]; then - MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" - fi - - # This script converts the generated version string into a sanitized version string for - # use by scons and uploading artifacts as well as information about for the scons cache. - echo $MONGO_VERSION | USE_SCONS_CACHE=${use_scons_cache|false} ${python|python} buildscripts/generate_compile_expansions.py | tee compile_expansions.yml - # Then we load the generated version data into the agent so we can use it in task definitions - - command: expansions.update - params: - file: src/compile_expansions.yml - - command: shell.exec - params: - working_dir: src - script: | - set -o errexit - set -o verbose - ${python|python} ./buildscripts/scons.py ${compile_flags|} ${scons_cache_args|} mongo${extension} mongod${extension} mongos${extension} MONGO_VERSION=${version} - mkdir -p mongodb/bin - mkdir -p mongodb/jstests/hooks - mv mongo${extension|} mongodb/bin - mv mongod${extension|} mongodb/bin - mv mongos${extension|} mongodb/bin - if [ -d jstests/hooks ] - then - echo "Fetching JS test DB correctness checks from directory jstests" - cp -a jstests/* mongodb/jstests - - echo "Now adding our own special run_validate_collections.js wrapper" - mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js + - func: "compile mongodb" - cat << EOF > mongodb/jstests/hooks/run_validate_collections.js - print("NOTE: run_validate_collections.js will skip the oplog!"); - TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; - load('jstests/hooks/run_validate_collections.actual.js'); - EOF - fi - tar cf mongodb.tar mongodb - gzip mongodb.tar - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/mongodb.tar.gz - remote_file: ${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: mongodb.tar.gz - -- name: industry_benchmarks_WT - depends_on: - - name: compile - variant: linux-standalone +- name: industry_benchmarks priority: 5 commands: - func: "prepare environment" vars: - storageEngine: "wiredTiger" test: "ycsb" - func: "deploy cluster" - func: "run test" @@ -305,47 +296,11 @@ tasks: vars: script_flags: --ycsb-throughput-analysis reports -- name: industry_benchmarks_MMAPv1 +- name: industry_benchmarks_wmajority priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "mmapv1" - test: "ycsb" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - vars: - script_flags: --ycsb-throughput-analysis reports - -- name: industry_benchmarks_wmajority_WT - priority: 5 - depends_on: - - name: compile - variant: linux-standalone - commands: - - func: "prepare environment" - vars: - storageEngine: "wiredTiger" - test: "ycsb-wmajority" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - vars: - script_flags: --ycsb-throughput-analysis reports - -- name: industry_benchmarks_wmajority_MMAPv1 - priority: 5 - depends_on: - - name: compile - variant: linux-standalone - commands: - - func: "prepare environment" - vars: - storageEngine: "mmapv1" test: "ycsb-wmajority" - func: "deploy cluster" - func: "run test" @@ -353,57 +308,41 @@ tasks: vars: script_flags: --ycsb-throughput-analysis reports -- name: crud_workloads_WT +- name: crud_workloads priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "wiredTiger" test: "crud_workloads" - func: "deploy cluster" - func: "run test" - func: "analyze" -- name: mixed_workloads_WT +- name: mixed_workloads priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "wiredTiger" test: "mixed_workloads" - func: "deploy cluster" - func: "run test" - func: "analyze" -- name: misc_workloads_WT +- name: misc_workloads priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "wiredTiger" test: "misc_workloads" - func: "deploy cluster" - func: "run test" - func: "analyze" -- name: map_reduce_workloads_WT +- name: map_reduce_workloads priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "wiredTiger" test: "map_reduce_workloads" - func: "deploy cluster" - func: "run test" @@ -411,188 +350,58 @@ tasks: - name: smoke_test priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "wiredTiger" test: "short" - func: "deploy cluster" - func: "run test" - func: "analyze" -- name: crud_workloads_MMAPv1 - priority: 5 - depends_on: - - name: compile - variant: linux-standalone - commands: - - func: "prepare environment" - vars: - storageEngine: "mmapv1" - test: "crud_workloads" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - -- name: mixed_workloads_MMAPv1 +- name: non_sharded_workloads priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "mmapv1" - test: "mixed_workloads" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - -- name: misc_workloads_MMAPv1 - priority: 5 - depends_on: - - name: compile - variant: linux-standalone - commands: - - func: "prepare environment" - vars: - storageEngine: "mmapv1" - test: "misc_workloads" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - -- name: map_reduce_workloads_MMAPv1 - priority: 5 - depends_on: - - name: compile - variant: linux-standalone - commands: - - func: "prepare environment" - vars: - storageEngine: "mmapv1" - test: "map_reduce_workloads" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - -- name: non_sharded_workloads_WT - priority: 5 - depends_on: - - name: compile - variant: linux-standalone - commands: - - func: "prepare environment" - vars: - storageEngine: "wiredTiger" - test: "non_sharded" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - -- name: non_sharded_workloads_MMAPv1 - priority: 5 - depends_on: - - name: compile - variant: linux-standalone - commands: - - func: "prepare environment" - vars: - storageEngine: "mmapv1" test: "non_sharded" - func: "deploy cluster" - func: "run test" - func: "analyze" - -- name: mongos_workloads_WT +- name: mongos_workloads priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "wiredTiger" test: "mongos" - func: "deploy cluster" - func: "run test" - func: "analyze" -- name: mongos_workloads_MMAPv1 +- name: move_chunk_workloads priority: 5 - depends_on: - - name: compile - variant: linux-standalone - exec_timeout_secs: 32400 # 9 hours - commands: - - func: "prepare environment" - vars: - storageEngine: "mmapv1" - test: "mongos" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - -- name: move_chunk_workloads_WT - priority: 5 - depends_on: - - name: compile - variant: linux-standalone - commands: - - func: "prepare environment" - vars: - storageEngine: "wiredTiger" - test: "move_chunk" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - -- name: move_chunk_workloads_MMAPv1 - priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "mmapv1" test: "move_chunk" - func: "deploy cluster" - func: "run test" - func: "analyze" -- name: secondary_performance_WT +- name: move_chunk_waiting_workloads priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "wiredTiger" - # Unfortunately the dash/underscore style is different for mongodb_setup and test_control - test: "secondary_performance" - setup: "secondary-performance" + test: "move_chunk_waiting" - func: "deploy cluster" - func: "run test" - func: "analyze" -- name: secondary_performance_MMAPv1 +- name: secondary_performance priority: 5 - depends_on: - - name: compile - variant: linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "mmapv1" # Unfortunately the dash/underscore style is different for mongodb_setup and test_control test: "secondary_performance" setup: "secondary-performance" @@ -600,44 +409,22 @@ tasks: - func: "run test" - func: "analyze" -- name: initialsync_WT - priority: 5 - depends_on: - - name: compile - variant: linux-standalone - commands: - - func: "prepare environment" - vars: - storageEngine: "wiredTiger" - test: "initialsync" - - func: "deploy cluster" - - func: "run test" - - func: "analyze" - -- name: initialsync_MMAPv1 +- name: initialsync priority: 5 - depends_on: - - name: compile - variant : linux-standalone commands: - func: "prepare environment" vars: - storageEngine: "mmapv1" test: "initialsync" - func: "deploy cluster" - func: "run test" - func: "analyze" -- name: initialsync-logkeeper_WT +- name: initialsync-logkeeper priority: 5 - depends_on: - - name: compile - variant: linux-standalone exec_timeout_secs: 216000 # 2.5 days commands: - func: "prepare environment" vars: - storageEngine: "wiredTiger" test: "initialsync-logkeeper" - func: "deploy cluster" - func: "run test" @@ -661,10 +448,33 @@ modules: ####################################### -# Linux Buildvariants # +# Buildvariants # ####################################### - buildvariants: + +# We are explicitly tracking the rhel70 variant compile options from evergreen.yml. If we can get +# proper artifacts directly from that project, we should do that and remove these tasks. +- name: compile-rhel70 + display_name: Compile on rhel70 + batchtime: 1440 # 24 hours + modules: + - wtdevelop + expansions: + compile_flags: --ssl MONGO_DISTMOD=rhel70 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars + gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH" + platform: linux + project_dir: &project_dir dsi + tooltags: "" + use_scons_cache: true + run_on: + - "rhel70" + tasks: + - name: compile + + +####################################### +# Linux Buildvariants # +####################################### - name: linux-1-node-replSet display_name: Linux 1-Node ReplSet batchtime: 10080 # 7 days @@ -676,55 +486,46 @@ buildvariants: setup: single-replica cluster: single platform: linux - use_scons_cache: true - project: &project dsi-v3.4 + project_dir: *project_dir + storageEngine: wiredTiger run_on: - "rhel70-perf-single" - tasks: - - name: industry_benchmarks_WT - - name: crud_workloads_WT - - name: mixed_workloads_WT - - name: misc_workloads_WT - - name: map_reduce_workloads_WT + depends_on: + - name: compile + variant: compile-rhel70 + tasks: &1nodetasks + - name: industry_benchmarks + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads - name: smoke_test - - name: industry_benchmarks_MMAPv1 - - name: crud_workloads_MMAPv1 - - name: mixed_workloads_MMAPv1 - - name: misc_workloads_MMAPv1 - - name: map_reduce_workloads_MMAPv1 - - name: non_sharded_workloads_WT - - name: non_sharded_workloads_MMAPv1 + - name: non_sharded_workloads + - name: linux-standalone display_name: Linux Standalone batchtime: 10080 # 7 days modules: *modules expansions: - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy setup: standalone cluster: single platform: linux - use_scons_cache: true - project: *project + project_dir: *project_dir + storageEngine: wiredTiger run_on: - "rhel70-perf-single" - tasks: - - name: compile - distros: - - rhel70 - - name: industry_benchmarks_WT - - name: crud_workloads_WT - - name: mixed_workloads_WT - - name: misc_workloads_WT - - name: map_reduce_workloads_WT + depends_on: + - name: compile + variant: compile-rhel70 + tasks: &standalonetasks + - name: industry_benchmarks + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads - name: smoke_test - - name: industry_benchmarks_MMAPv1 - - name: crud_workloads_MMAPv1 - - name: mixed_workloads_MMAPv1 - - name: misc_workloads_MMAPv1 - - name: map_reduce_workloads_MMAPv1 - - name: non_sharded_workloads_WT - - name: non_sharded_workloads_MMAPv1 + - name: non_sharded_workloads - name: linux-3-shard display_name: Linux 3-Shard Cluster @@ -736,27 +537,23 @@ buildvariants: cluster: shard platform: linux use_scons_cache: true - project: *project + project_dir: *project_dir + storageEngine: wiredTiger run_on: - "rhel70-perf-shard" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - - name: industry_benchmarks_WT - - name: crud_workloads_WT - - name: mixed_workloads_WT - - name: misc_workloads_WT - - name: map_reduce_workloads_WT + - name: industry_benchmarks + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads - name: smoke_test - - name: industry_benchmarks_MMAPv1 - - name: industry_benchmarks_wmajority_WT - - name: industry_benchmarks_wmajority_MMAPv1 - - name: crud_workloads_MMAPv1 - - name: mixed_workloads_MMAPv1 - - name: misc_workloads_MMAPv1 - - name: map_reduce_workloads_MMAPv1 - - name: mongos_workloads_WT - - name: mongos_workloads_MMAPv1 - - name: move_chunk_workloads_WT - - name: move_chunk_workloads_MMAPv1 + - name: industry_benchmarks_wmajority + - name: mongos_workloads + - name: move_chunk_workloads - name: linux-3-node-replSet display_name: Linux 3-Node ReplSet @@ -767,27 +564,23 @@ buildvariants: setup: replica cluster: replica platform: linux - project: *project + project_dir: *project_dir + storageEngine: wiredTiger run_on: - "rhel70-perf-replset" - tasks: - - name: industry_benchmarks_WT - - name: crud_workloads_WT - - name: mixed_workloads_WT - - name: misc_workloads_WT - - name: map_reduce_workloads_WT + depends_on: + - name: compile + variant: compile-rhel70 + tasks: &3nodetasks + - name: industry_benchmarks + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads - name: smoke_test - - name: industry_benchmarks_MMAPv1 - - name: industry_benchmarks_wmajority_WT - - name: industry_benchmarks_wmajority_MMAPv1 - - name: crud_workloads_MMAPv1 - - name: mixed_workloads_MMAPv1 - - name: misc_workloads_MMAPv1 - - name: map_reduce_workloads_MMAPv1 - - name: secondary_performance_WT - - name: secondary_performance_MMAPv1 - - name: non_sharded_workloads_WT - - name: non_sharded_workloads_MMAPv1 + - name: industry_benchmarks_wmajority + - name: secondary_performance + - name: non_sharded_workloads - name: linux-3-node-replSet-initialsync display_name: Linux 3-Node ReplSet Initial Sync @@ -799,12 +592,15 @@ buildvariants: cluster: replica platform: linux use_scons_cache: true - project: *project + project_dir: *project_dir + storageEngine: wiredTiger run_on: - "rhel70-perf-replset" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - - name: initialsync_WT - - name: initialsync_MMAPv1 + - name: initialsync - name: linux-replSet-initialsync-logkeeper display_name: Linux ReplSet Initial Sync LogKeeper @@ -816,8 +612,133 @@ buildvariants: cluster: initialsync-logkeeper platform: linux use_scons_cache: true - project: *project + storageEngine: wiredTiger + project_dir: *project_dir run_on: - "rhel70-perf-initialsync-logkeeper" + depends_on: + - name: compile + variant: compile-rhel70 + tasks: + - name: initialsync-logkeeper + +####################################### +# MMAP Buildvariants # +####################################### +- name: mmap-1-node-replSet + display_name: MMAP 1-Node ReplSet + batchtime: 10080 # 7 days + modules: *modules + expansions: + setup: single-replica + cluster: single + platform: linux + storageEngine: "mmapv1" + project_dir: *project_dir + run_on: + - "rhel70-perf-single" + depends_on: + - name: compile + variant: compile-rhel70 + tasks: + - name: smoke_test + - name: industry_benchmarks + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: non_sharded_workloads + +- name: mmap-standalone + display_name: MMAP Standalone + batchtime: 10080 # 7 days + modules: *modules + expansions: + setup: standalone + cluster: single + platform: linux + storageEngine: "mmapv1" + project_dir: *project_dir + run_on: + - "rhel70-perf-single" + depends_on: + - name: compile + variant: compile-rhel70 + tasks: + - name: smoke_test + - name: industry_benchmarks + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: non_sharded_workloads + +- name: mmap-3-shard + display_name: MMAP 3-Shard Cluster + batchtime: 10080 # 7 days + modules: *modules + expansions: + setup: shard + cluster: shard + platform: linux + project_dir: *project_dir + storageEngine: "mmapv1" + run_on: + - "rhel70-perf-shard" + depends_on: + - name: compile + variant: compile-rhel70 + tasks: + - name: smoke_test + - name: industry_benchmarks + - name: industry_benchmarks_wmajority + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: mongos_workloads + - name: move_chunk_workloads + +- name: mmap-3-node-replSet + display_name: MMAP 3-Node ReplSet + batchtime: 10080 # 7 days + modules: *modules + expansions: + setup: replica + cluster: replica + platform: linux + storageEngine: "mmapv1" + project_dir: *project_dir + run_on: + - "rhel70-perf-replset" + depends_on: + - name: compile + variant: compile-rhel70 + tasks: + - name: smoke_test + - name: industry_benchmarks + - name: industry_benchmarks_wmajority + - name: crud_workloads + - name: mixed_workloads + - name: misc_workloads + - name: map_reduce_workloads + - name: secondary_performance + - name: non_sharded_workloads + +- name: mmap-3-node-replSet-initialsync + display_name: MMAP 3-Node ReplSet Initial Sync + batchtime: 10080 # 7 days + modules: *modules + expansions: + setup: replica-2node + cluster: replica + platform: linux + storageEngine: "mmapv1" + project_dir: *project_dir + run_on: + - "rhel70-perf-replset" + depends_on: + - name: compile + variant: compile-rhel70 tasks: - - name: initialsync-logkeeper_WT + - name: initialsync diff --git a/jstests/concurrency/fsm_libs/cluster.js b/jstests/concurrency/fsm_libs/cluster.js index f3e2429f898..ee6d561a368 100644 --- a/jstests/concurrency/fsm_libs/cluster.js +++ b/jstests/concurrency/fsm_libs/cluster.js @@ -161,13 +161,14 @@ var Cluster = function(options) { "Both 'masterSlave' and 'sharded.enabled' cannot" + "be true"); } - function makeReplSetTestConfig(numReplSetNodes) { + function makeReplSetTestConfig(numReplSetNodes, firstNodeOnlyVote = true) { const REPL_SET_VOTING_LIMIT = 7; + var firstNodeNotVoting = firstNodeOnlyVote ? 1 : REPL_SET_VOTING_LIMIT; // Workaround for SERVER-26893 to specify when numReplSetNodes > REPL_SET_VOTING_LIMIT. var rstConfig = []; for (var i = 0; i < numReplSetNodes; i++) { rstConfig[i] = {}; - if (i >= REPL_SET_VOTING_LIMIT) { + if (i >= firstNodeNotVoting) { rstConfig[i].rsConfig = {priority: 0, votes: 0}; } } @@ -223,11 +224,11 @@ var Cluster = function(options) { conn = st.s; // mongos - this.teardown = function teardown() { + this.teardown = function teardown(opts) { options.teardownFunctions.mongod.forEach(this.executeOnMongodNodes); options.teardownFunctions.mongos.forEach(this.executeOnMongosNodes); - st.stop(); + st.stop(opts); }; // Save all mongos and mongod connections @@ -273,10 +274,9 @@ var Cluster = function(options) { conn = rst.getPrimary(); replSets = [rst]; - this.teardown = function teardown() { + this.teardown = function teardown(opts) { options.teardownFunctions.mongod.forEach(this.executeOnMongodNodes); - - rst.stopSet(); + rst.stopSet(undefined, undefined, opts); }; this._addReplicaSetConns(rst); diff --git a/jstests/concurrency/fsm_libs/runner.js b/jstests/concurrency/fsm_libs/runner.js index ec629245dad..f5d35f1dbf6 100644 --- a/jstests/concurrency/fsm_libs/runner.js +++ b/jstests/concurrency/fsm_libs/runner.js @@ -647,6 +647,7 @@ var runner = (function() { var bgCleanup = []; var errors = []; var configServerData = []; + let activeException = false; try { prepareCollections(bgWorkloads, bgContext, cluster, clusterOptions, executionOptions); @@ -705,6 +706,9 @@ var runner = (function() { dbHashBlacklist, configServerData); }); + } catch (err) { + activeException = true; + throw err; } finally { // Set a flag so background threads know to terminate. bgThreadMgr.markAllForTermination(); @@ -732,8 +736,14 @@ var runner = (function() { } throwError(workloadErrors); + } catch (err) { + activeException = true; + throw err; } finally { - cluster.teardown(); + // We preserve the data files when an FSM workload failed so that they can later be + // archived to S3. + const opts = activeException ? {noCleanData: true} : {}; + cluster.teardown(opts); } } } diff --git a/jstests/core/max_time_ms.js b/jstests/core/max_time_ms.js index efe3dabcc6d..e50e70560f6 100644 --- a/jstests/core/max_time_ms.js +++ b/jstests/core/max_time_ms.js @@ -51,23 +51,23 @@ assert.doesNotThrow(function() { // // Simple positive test for getmore: // - Issue a find() that returns 2 batches: a fast batch, then a slow batch. -// - The find() has a 1-second time limit; the first batch should run "instantly", but the second +// - The find() has a 4-second time limit; the first batch should run "instantly", but the second // batch takes ~15 seconds, so the getmore should be aborted. // t.drop(); -t.insert([{}, {}, {}]); // fast batch -t.insert([{slow: true}, {slow: true}, {slow: true}]); // slow batch +t.insert([{_id: 0}, {_id: 1}, {_id: 2}]); // fast batch +t.insert([{_id: 3, slow: true}, {_id: 4, slow: true}, {_id: 5, slow: true}]); // slow batch cursor = t.find({ - $where: function() { - if (this.slow) { - sleep(5 * 1000); - } - return true; - } -}); + $where: function() { + if (this.slow) { + sleep(5 * 1000); + } + return true; + } + }).sort({_id: 1}); cursor.batchSize(3); -cursor.maxTimeMS(1000); +cursor.maxTimeMS(4 * 1000); assert.doesNotThrow(function() { cursor.next(); cursor.next(); @@ -87,16 +87,16 @@ assert.throws(function() { // t.drop(); -t.insert([{}, {}, {}]); // fast batch -t.insert([{}, {}, {slow: true}]); // slow batch +t.insert([{_id: 0}, {_id: 1}, {_id: 2}]); // fast batch +t.insert([{_id: 3}, {_id: 4}, {_id: 5, slow: true}]); // slow batch cursor = t.find({ - $where: function() { - if (this.slow) { - sleep(2 * 1000); - } - return true; - } -}); + $where: function() { + if (this.slow) { + sleep(2 * 1000); + } + return true; + } + }).sort({_id: 1}); cursor.batchSize(3); cursor.maxTimeMS(10 * 1000); assert.doesNotThrow(function() { @@ -118,16 +118,16 @@ assert.doesNotThrow(function() { t.drop(); for (var i = 0; i < 5; i++) { - t.insert([{}, {}, {slow: true}]); + t.insert([{_id: 3 * i}, {_id: (3 * i) + 1}, {_id: (3 * i) + 2, slow: true}]); } cursor = t.find({ - $where: function() { - if (this.slow) { - sleep(2 * 1000); - } - return true; - } -}); + $where: function() { + if (this.slow) { + sleep(2 * 1000); + } + return true; + } + }).sort({_id: 1}); cursor.batchSize(3); cursor.maxTimeMS(6 * 1000); assert.throws(function() { @@ -142,16 +142,16 @@ assert.throws(function() { t.drop(); for (var i = 0; i < 5; i++) { - t.insert([{}, {}, {slow: true}]); + t.insert([{_id: 3 * i}, {_id: (3 * i) + 1}, {_id: (3 * i) + 2, slow: true}]); } cursor = t.find({ - $where: function() { - if (this.slow) { - sleep(2 * 1000); - } - return true; - } -}); + $where: function() { + if (this.slow) { + sleep(2 * 1000); + } + return true; + } + }).sort({_id: 1}); cursor.batchSize(3); cursor.maxTimeMS(20 * 1000); assert.doesNotThrow(function() { @@ -356,16 +356,16 @@ assert.eq(1, t.getDB().adminCommand({configureFailPoint: "maxTimeAlwaysTimeOut", // maxTimeNeverTimeOut positive test for getmore. t.drop(); -t.insert([{}, {}, {}]); // fast batch -t.insert([{slow: true}, {slow: true}, {slow: true}]); // slow batch +t.insert([{_id: 0}, {_id: 1}, {_id: 2}]); // fast batch +t.insert([{_id: 3, slow: true}, {_id: 4, slow: true}, {_id: 5, slow: true}]); // slow batch cursor = t.find({ - $where: function() { - if (this.slow) { - sleep(2 * 1000); - } - return true; - } -}); + $where: function() { + if (this.slow) { + sleep(2 * 1000); + } + return true; + } + }).sort({_id: 1}); cursor.batchSize(3); cursor.maxTimeMS(2 * 1000); assert.doesNotThrow(function() { diff --git a/jstests/core/minmax.js b/jstests/core/minmax.js index 670c7d2f8b2..68dcdb38992 100644 --- a/jstests/core/minmax.js +++ b/jstests/core/minmax.js @@ -1,97 +1,142 @@ -// test min / max query parameters - -addData = function() { - t.save({a: 1, b: 1}); - t.save({a: 1, b: 2}); - t.save({a: 2, b: 1}); - t.save({a: 2, b: 2}); -}; - -t = db.jstests_minmax; -t.drop(); -t.ensureIndex({a: 1, b: 1}); -addData(); - -printjson(t.find().min({a: 1, b: 2}).max({a: 2, b: 1}).toArray()); -assert.eq(1, t.find().min({a: 1, b: 2}).max({a: 2, b: 1}).toArray().length); -assert.eq(2, t.find().min({a: 1, b: 2}).max({a: 2, b: 1.5}).toArray().length); -assert.eq(2, t.find().min({a: 1, b: 2}).max({a: 2, b: 2}).toArray().length); - -// just one bound -assert.eq(3, t.find().min({a: 1, b: 2}).toArray().length); -assert.eq(3, t.find().max({a: 2, b: 1.5}).toArray().length); -assert.eq(3, t.find().min({a: 1, b: 2}).hint({a: 1, b: 1}).toArray().length); -assert.eq(3, t.find().max({a: 2, b: 1.5}).hint({a: 1, b: 1}).toArray().length); - -t.drop(); -t.ensureIndex({a: 1, b: -1}); -addData(); -assert.eq(4, t.find().min({a: 1, b: 2}).toArray().length); -assert.eq(4, t.find().max({a: 2, b: 0.5}).toArray().length); -assert.eq(1, t.find().min({a: 2, b: 1}).toArray().length); -assert.eq(1, t.find().max({a: 1, b: 1.5}).toArray().length); -assert.eq(4, t.find().min({a: 1, b: 2}).hint({a: 1, b: -1}).toArray().length); -assert.eq(4, t.find().max({a: 2, b: 0.5}).hint({a: 1, b: -1}).toArray().length); -assert.eq(1, t.find().min({a: 2, b: 1}).hint({a: 1, b: -1}).toArray().length); -assert.eq(1, t.find().max({a: 1, b: 1.5}).hint({a: 1, b: -1}).toArray().length); - -// hint doesn't match -assert.throws(function() { - t.find().min({a: 1}).hint({a: 1, b: -1}).toArray(); -}); -assert.throws(function() { - t.find().min({a: 1, b: 1}).max({a: 1}).hint({a: 1, b: -1}).toArray(); -}); -assert.throws(function() { - t.find().min({b: 1}).max({a: 1, b: 2}).hint({a: 1, b: -1}).toArray(); -}); -assert.throws(function() { - t.find().min({a: 1}).hint({$natural: 1}).toArray(); -}); -assert.throws(function() { - t.find().max({a: 1}).hint({$natural: 1}).toArray(); -}); - -// Reverse direction scan of the a:1 index between a:6 (inclusive) and a:3 (exclusive). -t.drop(); -t.ensureIndex({a: 1}); -for (i = 0; i < 10; ++i) { - t.save({_id: i, a: i}); -} -if (0) { // SERVER-3766 - reverseResult = t.find().min({a: 6}).max({a: 3}).sort({a: -1}).hint({a: 1}).toArray(); - assert.eq([{_id: 6, a: 6}, {_id: 5, a: 5}, {_id: 4, a: 4}], reverseResult); -} - -// -// SERVER-15015. -// - -// Test ascending index. -t.drop(); -t.ensureIndex({a: 1}); -t.insert({a: 3}); -t.insert({a: 4}); -t.insert({a: 5}); - -var cursor = t.find().min({a: 4}); -assert.eq(4, cursor.next()["a"]); -assert.eq(5, cursor.next()["a"]); -assert(!cursor.hasNext()); - -cursor = t.find().max({a: 4}); -assert.eq(3, cursor.next()["a"]); -assert(!cursor.hasNext()); - -// Test descending index. -t.dropIndexes(); -t.ensureIndex({a: -1}); - -cursor = t.find().min({a: 4}); -assert.eq(4, cursor.next()["a"]); -assert.eq(3, cursor.next()["a"]); -assert(!cursor.hasNext()); - -cursor = t.find().max({a: 4}); -assert.eq(5, cursor.next()["a"]); -assert(!cursor.hasNext()); +// Test min / max query parameters. +(function() { + "use strict"; + + load("jstests/aggregation/extras/utils.js"); // For resultsEq. + + const coll = db.jstests_minmax; + coll.drop(); + + function addData() { + assert.writeOK(coll.save({a: 1, b: 1})); + assert.writeOK(coll.save({a: 1, b: 2})); + assert.writeOK(coll.save({a: 2, b: 1})); + assert.writeOK(coll.save({a: 2, b: 2})); + } + + assert.commandWorked(coll.ensureIndex({a: 1, b: 1})); + addData(); + + assert.eq(1, coll.find().min({a: 1, b: 2}).max({a: 2, b: 1}).toArray().length); + assert.eq(2, coll.find().min({a: 1, b: 2}).max({a: 2, b: 1.5}).toArray().length); + assert.eq(2, coll.find().min({a: 1, b: 2}).max({a: 2, b: 2}).toArray().length); + + // Single bound. + assert.eq(3, coll.find().min({a: 1, b: 2}).toArray().length); + assert.eq(3, coll.find().max({a: 2, b: 1.5}).toArray().length); + assert.eq(3, coll.find().min({a: 1, b: 2}).hint({a: 1, b: 1}).toArray().length); + assert.eq(3, coll.find().max({a: 2, b: 1.5}).hint({a: 1, b: 1}).toArray().length); + + coll.drop(); + assert.commandWorked(coll.ensureIndex({a: 1, b: -1})); + addData(); + assert.eq(4, coll.find().min({a: 1, b: 2}).toArray().length); + assert.eq(4, coll.find().max({a: 2, b: 0.5}).toArray().length); + assert.eq(1, coll.find().min({a: 2, b: 1}).toArray().length); + assert.eq(1, coll.find().max({a: 1, b: 1.5}).toArray().length); + assert.eq(4, coll.find().min({a: 1, b: 2}).hint({a: 1, b: -1}).toArray().length); + assert.eq(4, coll.find().max({a: 2, b: 0.5}).hint({a: 1, b: -1}).toArray().length); + assert.eq(1, coll.find().min({a: 2, b: 1}).hint({a: 1, b: -1}).toArray().length); + assert.eq(1, coll.find().max({a: 1, b: 1.5}).hint({a: 1, b: -1}).toArray().length); + + // Hint doesn't match. + assert.throws(function() { + coll.find().min({a: 1}).hint({a: 1, b: -1}).toArray(); + }); + assert.throws(function() { + coll.find().min({a: 1, b: 1}).max({a: 1}).hint({a: 1, b: -1}).toArray(); + }); + assert.throws(function() { + coll.find().min({b: 1}).max({a: 1, b: 2}).hint({a: 1, b: -1}).toArray(); + }); + assert.throws(function() { + coll.find().min({a: 1}).hint({$natural: 1}).toArray(); + }); + assert.throws(function() { + coll.find().max({a: 1}).hint({$natural: 1}).toArray(); + }); + + coll.drop(); + assert.commandWorked(coll.ensureIndex({a: 1})); + for (let i = 0; i < 10; ++i) { + assert.writeOK(coll.save({_id: i, a: i})); + } + + // Reverse direction scan of the a:1 index between a:6 (inclusive) and a:3 (exclusive) is + // expected to fail, as max must be > min. + let error = assert.throws(function() { + coll.find().min({a: 6}).max({a: 3}).sort({a: -1}).toArray(); + }); + assert.eq(error.code, ErrorCodes.BadValue); + + error = assert.throws(function() { + coll.find().min({a: 6}).max({a: 3}).sort({a: -1}).hint({a: 1}).toArray(); + }); + assert.eq(error.code, ErrorCodes.BadValue); + + // A find with identical min and max values is expected to fail, as max is exclusive. + error = assert.throws(function() { + coll.find().min({a: 2}).max({a: 2}).toArray(); + }); + assert.eq(error.code, ErrorCodes.BadValue); + + error = assert.throws(function() { + coll.find().min({a: 2}).max({a: 2}).hint({a: 1}).toArray(); + }); + assert.eq(error.code, ErrorCodes.BadValue); + + error = assert.throws(function() { + coll.find().min({a: 2}).max({a: 2}).sort({a: -1}).toArray(); + }); + assert.eq(error.code, ErrorCodes.BadValue); + + error = assert.throws(function() { + coll.find().min({a: 2}).max({a: 2}).sort({a: -1}).hint({a: 1}).toArray(); + }); + assert.eq(error.code, ErrorCodes.BadValue); + + coll.drop(); + addData(); + assert.commandWorked(coll.ensureIndex({a: 1, b: 1})); + + error = assert.throws(function() { + coll.find().min({a: 1, b: 2}).max({a: 1, b: 2}).toArray(); + }); + assert.eq(error.code, ErrorCodes.BadValue); + + error = assert.throws(function() { + coll.find().min({a: 1, b: 2}).max({a: 1, b: 2}).hint({a: 1, b: 1}).toArray(); + }); + assert.eq(error.code, ErrorCodes.BadValue); + + // Test ascending index. + coll.drop(); + assert.commandWorked(coll.ensureIndex({a: 1})); + assert.writeOK(coll.insert({a: 3})); + assert.writeOK(coll.insert({a: 4})); + assert.writeOK(coll.insert({a: 5})); + + let cursor = coll.find().min({a: 4}); + assert.eq(4, cursor.next().a); + assert.eq(5, cursor.next().a); + + assert(!cursor.hasNext()); + + cursor = coll.find().max({a: 4}); + assert.eq(3, cursor.next()["a"]); + assert(!cursor.hasNext()); + + // Test descending index. + assert.commandWorked(coll.dropIndexes()); + assert.commandWorked(coll.ensureIndex({a: -1})); + + cursor = coll.find().min({a: 4}); + assert.eq(4, cursor.next().a); + assert.eq(3, cursor.next().a); + + assert(!cursor.hasNext()); + + cursor = coll.find().max({a: 4}); + assert.eq(5, cursor.next()["a"]); + assert(!cursor.hasNext()); +}()); diff --git a/jstests/core/minmax_edge.js b/jstests/core/minmax_edge.js index abd39724a80..759d87dc21e 100644 --- a/jstests/core/minmax_edge.js +++ b/jstests/core/minmax_edge.js @@ -79,8 +79,6 @@ verifyMax({a: {a: 1}}, [0, 1, 2, 3, 4, 5, 6, 7, 8]); verifyMin({a: 'a'}, []); verifyMax({a: 'a'}, [0, 1, 2, 3, 4, 5, 6, 7, 8]); -verifyResultIds(t.find().min({a: 4}).max({a: 4}).toArray(), []); - // Now with a compound index. reset(t); assert.commandWorked(t.ensureIndex({a: 1, b: -1})); @@ -101,8 +99,6 @@ verifyMax({a: {a: 1}, b: 1}, [0, 1, 2, 3, 4, 5, 6, 7, 8]); verifyMin({a: 'a', b: 1}, []); verifyMax({a: 'a', b: 1}, [0, 1, 2, 3, 4, 5, 6, 7, 8]); -verifyResultIds(t.find().min({a: 4, b: 1}).max({a: 4, b: 1}).toArray(), []); - // Edge cases on b values verifyMin({a: 1, b: Infinity}, [0, 1, 2, 3, 4, 5, 6, 7, 8]); verifyMin({a: 2, b: Infinity}, [3, 4, 5, 6, 7, 8]); @@ -146,8 +142,6 @@ verifyMax({a: {a: 1}}, []); verifyMin({a: 'a'}, [0, 1, 2, 3, 4, 5, 6, 7, 8]); verifyMax({a: 'a'}, []); -verifyResultIds(t.find().min({a: 4}).max({a: 4}).toArray(), []); - // Now with a compound index. reset(t); t.ensureIndex({a: -1, b: -1}); diff --git a/jstests/libs/client-self-signed.pem b/jstests/libs/client-self-signed.pem new file mode 100644 index 00000000000..27fbf225059 --- /dev/null +++ b/jstests/libs/client-self-signed.pem @@ -0,0 +1,47 @@ +-----BEGIN CERTIFICATE----- +MIIDXDCCAkQCCQCDkcJhHXlrtzANBgkqhkiG9w0BAQsFADBwMQ8wDQYDVQQDDAZj +bGllbnQxEzARBgNVBAsMCktlcm5lbFVzZXIxEDAOBgNVBAoMB01vbmdvREIxFjAU +BgNVBAcMDU5ldyBZb3JrIENpdHkxETAPBgNVBAgMCE5ldyBZb3JrMQswCQYDVQQG +EwJVUzAeFw0xODA1MDgxOTA2MTdaFw0yMDA3MTcxOTA2MTdaMHAxDzANBgNVBAMM +BmNsaWVudDETMBEGA1UECwwKS2VybmVsVXNlcjEQMA4GA1UECgwHTW9uZ29EQjEW +MBQGA1UEBwwNTmV3IFlvcmsgQ2l0eTERMA8GA1UECAwITmV3IFlvcmsxCzAJBgNV +BAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxIP1BCsp6e+B +5BpefH+yDvdVwDteI3H7CKh8X0TJYAYHAy5bOJ9fS0PB50gAXYqDFTHWTnGGxTaq +67d4Hc5ZB+UgnNralBvhVYB6jFpV4oBqdht//VhcdLBq8n6aRmM63zP82dKA5Ehz +q7jhwLI0v/m3lZfkE+qKWb+E87KS47BstOHDIqAUIeUJutrC/usS7aatRhHqtmg2 +8pEVJV8/YBPL4DWZkKoqrg+hJjXNjpy7ec2GAvgk+5ZWvjmNUBTtrE+47l8Eq3nM +TLsTDqtPHqv/udZScaCltPkw2ffGMhhp/eFLF+F3E9NpZC5mpUfe4+E5iZAXL9/r +IRO4ESbI9wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB4DscFT2z5hgzHC64SVjHk +Rpodjct0/kLkcJpaAZ/VTU6J6JI0TRDI0lsYV+UlogPxNWHgo7OGmG26tUPh61TX +XCq0Y1SbUsJFJa61d4fl30cBsA/w+eN1GRMua+eo1DTeFC8n0bVUqCjaMzumZWnX +bOR5eUUGw2r/JbCvCU2QNuL6/LKg2e2/C6pwkxQzFpKVTgvc0mq9+rifAIo0a9JV +41FIpgbJrojXCjP1gs2On3Uvr0XJsiMd9+T6duZt5MNh4M3Mfrr+tzATM61JM0Mg +VryKakzERFMLnTH3aS/JfTMGlCvUJEiMU2N9mePmSucNnJHX6wYjmOr++6RoLBhc +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAxIP1BCsp6e+B5BpefH+yDvdVwDteI3H7CKh8X0TJYAYHAy5b +OJ9fS0PB50gAXYqDFTHWTnGGxTaq67d4Hc5ZB+UgnNralBvhVYB6jFpV4oBqdht/ +/VhcdLBq8n6aRmM63zP82dKA5Ehzq7jhwLI0v/m3lZfkE+qKWb+E87KS47BstOHD +IqAUIeUJutrC/usS7aatRhHqtmg28pEVJV8/YBPL4DWZkKoqrg+hJjXNjpy7ec2G +Avgk+5ZWvjmNUBTtrE+47l8Eq3nMTLsTDqtPHqv/udZScaCltPkw2ffGMhhp/eFL +F+F3E9NpZC5mpUfe4+E5iZAXL9/rIRO4ESbI9wIDAQABAoIBAEV/SUJNJYiG3E/Y +B3xOCKGEVP4jwLoWUCoSs9FZWUItP4OffEE9E41hjAyICSELoCn5swmQ2pGaML4l +50PbOt8w5+Sw/UYNyhXb1vsV43Oz6HKWbSAihKf70AasQVENPL+8+fnqmQyils2v +F593JnKSFJvB2LY2ZvEwawtoJsyAdENw+NacVnEoMbaJqn0x0XsFb0RAvV9PQaFD +U29yg3T3TxnoDHxmnPaLUhn4ht7bOc66PjeJEl2jvnhNdSwPND9dg+hF/hX966Io +vNfKyp5q5nr6dnshjMWciyAu3v0FhaQ+7MTG2pij1yR1L1XM/gxL7u65bCCw3D2d +BEiiB2ECgYEA7HoiLTImR2IN3QSkKi/j6YzLIhpZgfl1XW3P2EKXcbToghDsbZtV +81ZtPwX5R5gCSKR7c05hPOgio2ext71jomRQYDHHTFu2gbKyk0iC3fhY7u44ugRQ +KOMqQE2WZUV4Ll9QOflMoSWypDmSzgG5hYF86dSW1Leuoo8nDUnvcocCgYEA1L0/ +YNZoF+h4OsO3t7zJ38X6vmn7vwkWwlFC8puaseUmL0ENWGZxtBZzuzcVHWzSTxzk ++lpKNUloXeEE39btVWonDUx4uPajHsXE3i8/jjgOxden36cGxXxV1yWaSGTC7ikM +TPSzhtyMxlsmG7cXgIELCiWF0H2Mt+RGwTnz4hECgYEApQD/keu5O/vWUo7ngaxS +6mbcYppiNEf9Ncjmyy6D/8pMplzqq/xEd/upVN11OBiAF3H11douDVKowL4dCmky ++V/nmYeCWexwp4UqB40EC6A7ZdSBboKN8Em59I5uz5Eur6TAFQO4DYWBZgfqQKaB +bf+RSpOU/y5/w1wB0srxMkcCgYEAyq36geIDjjOWyDXLBKxCkIVoXJ1QkTKxkjQl +WkpRWYb9mCub8Uq4rWBupkDWRxB0VZcruDDpXlcvrbFCGZgEyUm3Jv5iTaX8xsaY +xy2wmWhi+q55a6UP2Hqatb1Hfg0xggFtjKvdlnDtVdVlOyZ2p6FJyULyeC4FMPW1 +S0ZRSmECgYEAyjTgEjvIUsd4NkVdMcBBHO/WKl7gMQDcUGOnlrpyEPumXC1+H2jy +GWbnBoWyXJ+SrMP3MFpCEG7BZ7P5oFjLDBJ8+xUiHWPLLGRRhp2tyySeR/urjgRO +kLX6W+WWErw2V5+KmgEJb/SpbAGcJhDyLCN56L/Osqrod9avQ4/YW60= +-----END RSA PRIVATE KEY----- diff --git a/jstests/multiVersion/3_upgrade_replset.js b/jstests/multiVersion/3_upgrade_replset.js index bccab5068c7..ea8826c1ad7 100644 --- a/jstests/multiVersion/3_upgrade_replset.js +++ b/jstests/multiVersion/3_upgrade_replset.js @@ -58,6 +58,11 @@ rst.upgradeSet({binVersion: "latest"}); jsTest.log("Replica set upgraded."); +// We save a reference to the old primary so that we can call reconnect() on it before +// joinFindInsert() would attempt to send the node an update operation that signals the parallel +// shell running the background operations to stop. +var oldPrimary = primary; + // Wait for primary var primary = rst.getPrimary(); @@ -73,8 +78,15 @@ jsTest.log("Replica set downgraded."); // Allow even more valid writes to go through sleep(10 * 1000); +// Since the primary from before the upgrade took place was restarted as part of the +// upgrade/downgrade process, we explicitly reconnect to it so that sending it an update operation +// silently fails with an unchecked NotMaster error rather than a network error. +reconnect(oldPrimary.getDB("admin")); joinFindInsert(); +// Since the primary from after the upgrade took place was restarted as part of the downgrade +// process, we explicitly reconnect to it. +reconnect(primary.getDB("admin")); var totalInserts = primary.getCollection(insertNS).find().sort({_id: -1}).next()._id + 1; var dataFound = primary.getCollection(insertNS).count(); diff --git a/jstests/multiVersion/downgrade_replset.js b/jstests/multiVersion/downgrade_replset.js index a2f5a5346ba..fb37654f78b 100644 --- a/jstests/multiVersion/downgrade_replset.js +++ b/jstests/multiVersion/downgrade_replset.js @@ -58,8 +58,16 @@ jsTest.log("Downgrading replica set.."); rst.upgradeSet({binVersion: oldVersion, storageEngine: storageEngine}); jsTest.log("Downgrade complete."); +// We save a reference to the old primary so that we can call reconnect() on it before +// joinFindInsert() would attempt to send the node an update operation that signals the parallel +// shell running the background operations to stop. +var oldPrimary = primary; primary = rst.getPrimary(); printjson(rst.status()); +// Since the primary from before the upgrade took place was restarted as part of the +// upgrade/downgrade process, we explicitly reconnect to it so that sending it an update operation +// silently fails with an unchecked NotMaster error rather than a network error. +reconnect(oldPrimary.getDB("admin")); joinFindInsert(); rst.stopSet(); diff --git a/jstests/multiVersion/libs/multi_rs.js b/jstests/multiVersion/libs/multi_rs.js index 87d5995ef48..d9b55a16229 100644 --- a/jstests/multiVersion/libs/multi_rs.js +++ b/jstests/multiVersion/libs/multi_rs.js @@ -25,7 +25,6 @@ ReplSetTest.prototype.upgradeSet = function(options, user, pwd) { var node = nodesToUpgrade[i]; if (node == primary) { node = this.stepdown(node); - this.waitForState(node, ReplSetTest.State.SECONDARY); primary = this.getPrimary(); } @@ -84,7 +83,10 @@ ReplSetTest.prototype.stepdown = function(nodeId) { print('Caught exception after stepDown cmd: ' + tojson(ex)); } - return this.reconnect(node); + // Wait for the state to be secondary to avoid the race of closing new connections on stepdown. + this.waitForState(node, ReplSetTest.State.SECONDARY); + + return node; }; ReplSetTest.prototype.reconnect = function(node) { diff --git a/jstests/multiVersion/minor_version_downgrade_replset.js b/jstests/multiVersion/minor_version_downgrade_replset.js index 3dae0c0a23f..769bfc22979 100644 --- a/jstests/multiVersion/minor_version_downgrade_replset.js +++ b/jstests/multiVersion/minor_version_downgrade_replset.js @@ -58,8 +58,17 @@ jsTest.log("Downgrading replica set.."); rst.upgradeSet({binVersion: oldVersion, storageEngine: storageEngine}); jsTest.log("Downgrade complete."); +// We save a reference to the old primary so that we can call reconnect() on it before +// joinFindInsert() would attempt to send the node an update operation that signals the parallel +// shell running the background operations to stop. +var oldPrimary = primary; + primary = rst.getPrimary(); printjson(rst.status()); +// Since the old primary was restarted as part of the downgrade process, we explicitly reconnect +// to it so that sending it an update operation silently fails with an unchecked NotMaster error +// rather than a network error. +reconnect(oldPrimary.getDB("admin")); joinFindInsert(); rst.stopSet(); diff --git a/jstests/multiVersion/minor_version_upgrade_replset.js b/jstests/multiVersion/minor_version_upgrade_replset.js index 7f784f5c100..bcbdf6e8f3f 100644 --- a/jstests/multiVersion/minor_version_upgrade_replset.js +++ b/jstests/multiVersion/minor_version_upgrade_replset.js @@ -59,6 +59,11 @@ rst.upgradeSet({binVersion: "latest"}); jsTest.log("Replica set upgraded."); +// We save a reference to the old primary so that we can call reconnect() on it before +// joinFindInsert() would attempt to send the node an update operation that signals the parallel +// shell running the background operations to stop. +var oldPrimary = primary; + // Wait for primary var primary = rst.getPrimary(); @@ -67,6 +72,10 @@ printjson(rst.status()); // Allow more valid writes to go through sleep(10 * 1000); +// Since the old primary was restarted as part of the upgrade process, we explicitly reconnect to it +// so that sending it an update operation silently fails with an unchecked NotMaster error rather +// than a network error. +reconnect(oldPrimary.getDB("admin")); joinFindInsert(); var totalInserts = primary.getCollection(insertNS).find().sort({_id: -1}).next()._id + 1; diff --git a/jstests/noPassthrough/ftdc_setdirectory.js b/jstests/noPassthrough/ftdc_setdirectory.js index c1173055e2a..46f87e51f90 100644 --- a/jstests/noPassthrough/ftdc_setdirectory.js +++ b/jstests/noPassthrough/ftdc_setdirectory.js @@ -54,11 +54,11 @@ load('jstests/libs/ftdc.js'); // 2. Set path and succeed assert.commandWorked( - setParam(admin1, {"diagnosticDataCollectionDirectoryPath": testPath2})); + setParam(admin1, {"diagnosticDataCollectionDirectoryPath": testPath1})); // 3. Set path again and fail assert.commandFailed( - setParam(admin1, {"diagnosticDataCollectionDirectoryPath": testPath2})); + setParam(admin1, {"diagnosticDataCollectionDirectoryPath": testPath1})); // 4. Enable successfully assert.commandWorked(setParam(admin1, {"diagnosticDataCollectionEnabled": 1})); @@ -100,7 +100,7 @@ load('jstests/libs/ftdc.js'); // 2. Set path fail assert.commandFailed( - setParam(admin3, {"diagnosticDataCollectionDirectoryPath": testPath2})); + setParam(admin3, {"diagnosticDataCollectionDirectoryPath": testPath3})); // 3. Enable successfully assert.commandWorked(setParam(admin3, {"diagnosticDataCollectionEnabled": 1})); diff --git a/jstests/noPassthrough/shard_does_not_hang_on_bad_config_server.js b/jstests/noPassthrough/shard_does_not_hang_on_bad_config_server.js index 184e0ed00d9..c7260e44948 100644 --- a/jstests/noPassthrough/shard_does_not_hang_on_bad_config_server.js +++ b/jstests/noPassthrough/shard_does_not_hang_on_bad_config_server.js @@ -1,39 +1,40 @@ -(function() {
- 'use strict';
-
- var conn = MongoRunner.runMongod();
-
- var res;
-
- var connA = new Mongo(conn.host);
- res = assert.commandFailed(connA.adminCommand({
- moveChunk: 'DummyDB.DummyColl',
- configdb: 'localhost:1',
- fromShard: 'DummyFromShard',
- toShard: 'DummyToShard',
- min: {Key: -1},
- max: {Key: 1},
- maxChunkSizeBytes: 1024,
- maxTimeMS: 10000
- }));
- assert.eq(ErrorCodes.ExceededTimeLimit, res.code);
-
- // Sharding state is now initialized at this point, although with an unreachable config.
-
- var connB = new Mongo(conn.host);
- res = assert.commandFailed(connB.adminCommand({
- moveChunk: 'DummyDB.DummyColl',
- configdb: 'localhost:1',
- fromShard: 'DummyFromShard',
- toShard: 'DummyToShard',
- min: {Key: -1},
- max: {Key: 1},
- maxChunkSizeBytes: 1024,
- maxTimeMS: 10000
- }));
- // Cannot reach config server.
- assert.eq(ErrorCodes.HostUnreachable, res.code);
-
- MongoRunner.stopMongod(conn);
-
-})();
+(function() { + 'use strict'; + + var conn = MongoRunner.runMongod(); + + var res; + + var connA = new Mongo(conn.host); + res = assert.commandFailed(connA.adminCommand({ + moveChunk: 'DummyDB.DummyColl', + configdb: 'localhost:1', + fromShard: 'DummyFromShard', + toShard: 'DummyToShard', + min: {Key: -1}, + max: {Key: 1}, + maxChunkSizeBytes: 1024, + maxTimeMS: 10000 + })); + assert.eq(ErrorCodes.ExceededTimeLimit, res.code); + + // Sharding state is now initialized at this point, although with an unreachable config. + + var connB = new Mongo(conn.host); + res = assert.commandFailed(connB.adminCommand({ + moveChunk: 'DummyDB.DummyColl', + configdb: 'localhost:1', + fromShard: 'DummyFromShard', + toShard: 'DummyToShard', + min: {Key: -1}, + max: {Key: 1}, + maxChunkSizeBytes: 1024, + maxTimeMS: 10000 + })); + + // Cannot reach config server. + assert(res.code == ErrorCodes.HostUnreachable || res.code == ErrorCodes.ExceededTimeLimit, + 'Result was: ' + tojson(res)); + MongoRunner.stopMongod(conn); + +})(); diff --git a/jstests/sharding/top_chunk_autosplit.js b/jstests/sharding/top_chunk_autosplit.js index 2a8e40a787a..5daf46c2f67 100644 --- a/jstests/sharding/top_chunk_autosplit.js +++ b/jstests/sharding/top_chunk_autosplit.js @@ -4,7 +4,9 @@ function shardSetup(shardConfig, dbName, collName) { // Set the balancer mode to only balance on autoSplit assert.writeOK(st.s.getDB('config').settings.update( - {_id: 'balancer'}, {'$unset': {stopped: ''}, '$set': {mode: 'autoSplitOnly'}})); + {_id: 'balancer'}, + {'$unset': {stopped: ''}, '$set': {mode: 'autoSplitOnly'}}, + {writeConcern: {w: 'majority'}})); return st; } diff --git a/jstests/ssl/x509_invalid.js b/jstests/ssl/x509_invalid.js new file mode 100644 index 00000000000..25310603c50 --- /dev/null +++ b/jstests/ssl/x509_invalid.js @@ -0,0 +1,61 @@ +// Test X509 auth when --sslAllowInvalidCertificates is enabled + +(function() { + 'use strict'; + + const CLIENT_NAME = 'C=US,ST=New York,L=New York City,O=MongoDB,OU=KernelUser,CN=client'; + const CLIENT_CERT = 'jstests/libs/client.pem'; + const SERVER_CERT = 'jstests/libs/server.pem'; + const CA_CERT = 'jstests/libs/ca.pem'; + const SELF_SIGNED_CERT = 'jstests/libs/client-self-signed.pem'; + + function testClient(conn, cert, name, shouldSucceed) { + let auth = {mechanism: 'MONGODB-X509'}; + if (name !== null) { + auth.user = name; + } + const script = 'assert(db.getSiblingDB(\'$external\').auth(' + tojson(auth) + '));'; + clearRawMongoProgramOutput(); + const exitCode = runMongoProgram('mongo', + '--ssl', + '--sslAllowInvalidHostnames', + '--sslPEMKeyFile', + cert, + '--sslCAFile', + CA_CERT, + '--port', + conn.port, + '--eval', + script); + + assert.eq(shouldSucceed, exitCode === 0, "exitCode = " + tojson(exitCode)); + assert.eq( + !shouldSucceed, + rawMongoProgramOutput().includes('No verified subject name available from client')); + } + + function runTest(conn) { + const admin = conn.getDB('admin'); + admin.createUser({user: "admin", pwd: "admin", roles: ["root"]}); + admin.auth('admin', 'admin'); + + const external = conn.getDB('$external'); + external.createUser({user: CLIENT_NAME, roles: [{'role': 'readWrite', 'db': 'test'}]}); + + testClient(conn, CLIENT_CERT, CLIENT_NAME, true); + testClient(conn, SELF_SIGNED_CERT, CLIENT_NAME, false); + testClient(conn, CLIENT_CERT, null, true); + testClient(conn, SELF_SIGNED_CERT, null, false); + } + + // Standalone. + const mongod = MongoRunner.runMongod({ + auth: '', + sslMode: 'requireSSL', + sslPEMKeyFile: SERVER_CERT, + sslCAFile: CA_CERT, + sslAllowInvalidCertificates: '', + }); + runTest(mongod); + MongoRunner.stopMongod(mongod); +})(); diff --git a/jstests/ssl/x509_startup_warning.js b/jstests/ssl/x509_startup_warning.js new file mode 100644 index 00000000000..888e29255e3 --- /dev/null +++ b/jstests/ssl/x509_startup_warning.js @@ -0,0 +1,33 @@ +// Test for startuo warning when X509 auth and sslAllowInvalidCertificates are enabled + +(function() { + 'use strict'; + + function runTest(opts, expectWarning) { + clearRawMongoProgramOutput(); + const mongod = MongoRunner.runMongod(Object.assign({ + auth: '', + sslMode: 'requireSSL', + sslPEMKeyFile: 'jstests/libs/server.pem', + sslCAFile: 'jstests/libs/ca.pem', + }, + opts)); + assert.eq(expectWarning, + rawMongoProgramOutput().includes( + 'WARNING: While invalid X509 certificates may be used')); + MongoRunner.stopMongod(mongod); + } + + // Don't expect a warning when we're not using both options together. + runTest({}, false); + runTest({sslAllowInvalidCertificates: '', setParameter: 'authenticationMechanisms=SCRAM-SHA-1'}, + false); + runTest({setParameter: 'authenticationMechanisms=MONGODB-X509'}, false); + runTest({clusterAuthMode: 'x509'}, false); + + // Do expect a warning when we're combining options. + runTest( + {sslAllowInvalidCertificates: '', setParameter: 'authenticationMechanisms=MONGODB-X509'}, + true); + runTest({sslAllowInvalidCertificates: '', clusterAuthMode: 'x509'}, true); +})(); diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 1208446d70f..66ba4054a11 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -242,6 +242,7 @@ if env.TargetOSIs('windows'): env.Alias('generated-sources', generatedResourceConstantFile) config_header_substs = ( + ('@mongo_config_altivec_vec_vbpermq_output_index@', 'MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX'), ('@mongo_config_byte_order@', 'MONGO_CONFIG_BYTE_ORDER'), ('@mongo_config_debug_build@', 'MONGO_CONFIG_DEBUG_BUILD'), ('@mongo_config_have___declspec_thread@', 'MONGO_CONFIG_HAVE___DECLSPEC_THREAD'), diff --git a/src/mongo/config.h.in b/src/mongo/config.h.in index c1ef64647b0..011bc5a41ac 100644 --- a/src/mongo/config.h.in +++ b/src/mongo/config.h.in @@ -28,6 +28,10 @@ #pragma once + +// Define altivec vec_vbpermq output index +@mongo_config_altivec_vec_vbpermq_output_index@ + // Define to target byte order (1234 vs 4321) @mongo_config_byte_order@ diff --git a/src/mongo/db/commands/authentication_commands.cpp b/src/mongo/db/commands/authentication_commands.cpp index f6fcc301b05..1eb8569b2be 100644 --- a/src/mongo/db/commands/authentication_commands.cpp +++ b/src/mongo/db/commands/authentication_commands.cpp @@ -174,6 +174,7 @@ bool CmdAuthenticate::run(OperationContext* txn, } else { user = UserName(cmdObj.getStringField("user"), dbname); } + uassert(ErrorCodes::AuthenticationFailed, "No user name provided", !user.getUser().empty()); if (Command::testCommandsEnabled && user.getDB() == "admin" && user.getUser() == internalSecurity.user->getName().getUser()) { @@ -323,6 +324,9 @@ Status CmdAuthenticate::_authenticateX509(OperationContext* txn, Client* client = Client::getCurrent(); AuthorizationSession* authorizationSession = AuthorizationSession::get(client); auto clientName = client->session()->getX509PeerInfo().subjectName; + uassert(ErrorCodes::AuthenticationFailed, + "No verified subject name available from client", + !clientName.empty()); if (!getSSLManager()->getSSLConfiguration().hasCA) { return Status(ErrorCodes::AuthenticationFailed, diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index c6272425bf5..a99ce97d57c 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -48,6 +48,7 @@ #include "mongo/db/auth/auth_index_d.h" #include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/sasl_options.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/collection_catalog_entry.h" #include "mongo/db/catalog/database.h" @@ -136,6 +137,7 @@ #include "mongo/util/quick_exit.h" #include "mongo/util/ramlog.h" #include "mongo/util/scopeguard.h" +#include "mongo/util/sequence_util.h" #include "mongo/util/signal_handlers.h" #include "mongo/util/stacktrace.h" #include "mongo/util/startup_test.h" @@ -144,6 +146,10 @@ #include "mongo/util/time_support.h" #include "mongo/util/version.h" +#ifdef MONGO_CONFIG_SSL +#include "mongo/util/net/ssl_options.h" +#endif + #if !defined(_WIN32) #include <sys/file.h> #endif @@ -641,6 +647,18 @@ ExitCode _initAndListen(int listenPort) { logMongodStartupWarnings(storageGlobalParams, serverGlobalParams); +#if MONGO_CONFIG_SSL + if (sslGlobalParams.sslAllowInvalidCertificates && + ((serverGlobalParams.clusterAuthMode.load() == ServerGlobalParams::ClusterAuthMode_x509) || + sequenceContains(saslGlobalParams.authenticationMechanisms, "MONGODB-X509"))) { + log() << "** WARNING: While invalid X509 certificates may be used to" << startupWarningsLog; + log() << "** connect to this server, they will not be considered" + << startupWarningsLog; + log() << "** permissible for authentication." << startupWarningsLog; + log() << startupWarningsLog; + } +#endif + { stringstream ss; ss << endl; diff --git a/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp index 9c93718c171..fe730879834 100644 --- a/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp +++ b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp @@ -66,6 +66,10 @@ static const std::vector<StringData> kMemKeys{ "Inactive(file)"_sd, }; +static const std::vector<StringData> kNetstatKeys{ + "Tcp:"_sd, "Ip:"_sd, "TcpExt:"_sd, "IpExt:"_sd, +}; + /** * Collect metrics from the Linux /proc file system. */ @@ -99,6 +103,17 @@ public: subObjBuilder.doneFast(); } + { + BSONObjBuilder subObjBuilder(builder.subobjStart("netstat"_sd)); + processStatusErrors(procparser::parseProcNetstatFile( + kNetstatKeys, "/proc/net/netstat"_sd, &subObjBuilder), + &subObjBuilder); + processStatusErrors( + procparser::parseProcNetstatFile(kNetstatKeys, "/proc/net/snmp"_sd, &subObjBuilder), + &subObjBuilder); + subObjBuilder.doneFast(); + } + // Skip the disks section if we could not find any disks. // This can happen when we do not have permission to /sys/block for instance. if (!_disksStringData.empty()) { diff --git a/src/mongo/db/fts/unicode/byte_vector_altivec.h b/src/mongo/db/fts/unicode/byte_vector_altivec.h index f7a07ac0743..0797443f72f 100644 --- a/src/mongo/db/fts/unicode/byte_vector_altivec.h +++ b/src/mongo/db/fts/unicode/byte_vector_altivec.h @@ -34,6 +34,7 @@ #include <cstdint> +#include "mongo/config.h" #include "mongo/platform/bits.h" namespace mongo { @@ -98,7 +99,7 @@ public: // big endian by comparison. const Native bits = {120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 0}; - return vec_extract(vec_vbpermq(_data, bits), 0); + return vec_extract(vec_vbpermq(_data, bits), MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX); } /** diff --git a/src/mongo/db/matcher/expression_leaf.cpp b/src/mongo/db/matcher/expression_leaf.cpp index 0bf8d8685bc..c055f46ca3d 100644 --- a/src/mongo/db/matcher/expression_leaf.cpp +++ b/src/mongo/db/matcher/expression_leaf.cpp @@ -658,6 +658,11 @@ void InMatchExpression::_doSetCollator(const CollatorInterface* collator) { _collator = collator; _eltCmp = BSONElementComparator(BSONElementComparator::FieldNamesMode::kIgnore, _collator); + // Re-sort the list of equalities according to our current comparator. This is necessary to work + // around https://svn.boost.org/trac10/ticket/13140. + std::sort( + _originalEqualityVector.begin(), _originalEqualityVector.end(), _eltCmp.makeLessThan()); + // We need to re-compute '_equalitySet', since our set comparator has changed. _equalitySet = _eltCmp.makeBSONEltFlatSet(_originalEqualityVector); } @@ -677,8 +682,13 @@ Status InMatchExpression::setEqualities(std::vector<BSONElement> equalities) { _hasEmptyArray = true; } } + _originalEqualityVector = std::move(equalities); + // Sort the list of equalities to work around https://svn.boost.org/trac10/ticket/13140. + std::sort( + _originalEqualityVector.begin(), _originalEqualityVector.end(), _eltCmp.makeLessThan()); + _equalitySet = _eltCmp.makeBSONEltFlatSet(_originalEqualityVector); return Status::OK(); diff --git a/src/mongo/db/matcher/expression_leaf.h b/src/mongo/db/matcher/expression_leaf.h index 5761f2ca2a2..6e4858a5252 100644 --- a/src/mongo/db/matcher/expression_leaf.h +++ b/src/mongo/db/matcher/expression_leaf.h @@ -393,6 +393,12 @@ private: // Original container of equality elements, including duplicates. Needed for re-computing // '_equalitySet' in case '_collator' changes after elements have been added. + // + // We keep the equalities in sorted order according to the current BSON element comparator. This + // list of equalities will be used to construct a boost::flat_set, which maintains the set of + // elements in sorted order within a contiguous region of memory. Sorting and then constructing + // a flat_set is O(n log n), whereas the boost::flat_set constructor is O(n ^ 2) due to + // https://svn.boost.org/trac10/ticket/13140. std::vector<BSONElement> _originalEqualityVector; // Set of equality elements associated with this expression. '_eltCmp' is used as a comparator diff --git a/src/mongo/db/query/query_planner.cpp b/src/mongo/db/query/query_planner.cpp index 850276f35e7..2388154276c 100644 --- a/src/mongo/db/query/query_planner.cpp +++ b/src/mongo/db/query/query_planner.cpp @@ -659,7 +659,7 @@ Status QueryPlanner::plan(const CanonicalQuery& query, // Now we have the final min and max. This index is only relevant for // the min/max query if min < max. - if (0 >= + if (0 > finishedMinObj.woCompare(finishedMaxObj, indexEntry.keyPattern, false)) { // Found a relevant index. idxNo = i; diff --git a/src/mongo/db/query/query_planner_collation_test.cpp b/src/mongo/db/query/query_planner_collation_test.cpp index 54a549723c8..97860b36add 100644 --- a/src/mongo/db/query/query_planner_collation_test.cpp +++ b/src/mongo/db/query/query_planner_collation_test.cpp @@ -220,19 +220,6 @@ TEST_F(QueryPlannerTest, MinMaxWithStringBoundsCannotBeCoveredWithCollator) { "{locale: 'reverse'}, node: {ixscan: {pattern: {a: 1, b: 1}}}}}}}"); } -TEST_F(QueryPlannerTest, MinMaxWithoutStringBoundsBoundsCanBeCoveredWithCollator) { - CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString); - addIndex(fromjson("{a: 1, b: 1}"), &collator); - - runQueryAsCommand( - fromjson("{find: 'testns', min: {a: 1, b: 2}, max: {a: 1, b: 2}, " - "projection: {_id: 0, a: 1, b: 1}, collation: {locale: 'reverse'}}")); - - assertNumSolutions(1U); - assertSolutionExists( - "{proj: {spec: {_id: 0, a: 1, b: 1}, node: {ixscan: {pattern: {a: 1, b: 1}}}}}"); -} - TEST_F(QueryPlannerTest, SimpleRegexCanUseAnIndexWithACollatorWithLooseBounds) { CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString); addIndex(fromjson("{a: 1}"), &collator); diff --git a/src/mongo/db/query/query_planner_common.cpp b/src/mongo/db/query/query_planner_common.cpp index 33eb72100cc..337f3a045fc 100644 --- a/src/mongo/db/query/query_planner_common.cpp +++ b/src/mongo/db/query/query_planner_common.cpp @@ -71,8 +71,8 @@ void QueryPlannerCommon::reverseScans(QuerySolutionNode* node) { } if (!isn->bounds.isValidFor(isn->index.keyPattern, isn->direction)) { - LOG(5) << "Invalid bounds: " << redact(isn->bounds.toString()); - invariant(0); + severe() << "Invalid bounds: " << redact(isn->bounds.toString()); + MONGO_UNREACHABLE; } // TODO: we can just negate every value in the already computed properties. diff --git a/src/mongo/db/query/query_planner_test.cpp b/src/mongo/db/query/query_planner_test.cpp index 7fc29a4e42f..f4f608e0b58 100644 --- a/src/mongo/db/query/query_planner_test.cpp +++ b/src/mongo/db/query/query_planner_test.cpp @@ -1055,12 +1055,7 @@ TEST_F(QueryPlannerTest, MaxValid) { TEST_F(QueryPlannerTest, MinMaxSameValue) { addIndex(BSON("a" << 1)); - runQueryHintMinMax(BSONObj(), BSONObj(), fromjson("{a: 1}"), fromjson("{a: 1}")); - - assertNumSolutions(1U); - assertSolutionExists( - "{fetch: {filter: null, " - "node: {ixscan: {filter: null, pattern: {a: 1}}}}}"); + runInvalidQueryHintMinMax(BSONObj(), BSONObj(), fromjson("{a: 1}"), fromjson("{a: 1}")); } TEST_F(QueryPlannerTest, MaxWithoutIndex) { diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index 31f35d384f8..1f3ff6f12be 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -332,6 +332,7 @@ env.CppUnitTest( 'sync_tail_test.cpp', ], LIBDEPS=[ + 'oplog_buffer_blocking_queue', 'oplog_interface_local', 'replmocks', 'sync_tail', diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp index 8349428c3fd..53f020541b0 100644 --- a/src/mongo/db/repl/initial_syncer.cpp +++ b/src/mongo/db/repl/initial_syncer.cpp @@ -365,30 +365,44 @@ BSONObj InitialSyncer::getInitialSyncProgress() const { return _getInitialSyncProgress_inlock(); } +void InitialSyncer::_appendInitialSyncProgressMinimal_inlock(BSONObjBuilder* bob) const { + _stats.append(bob); + if (!_initialSyncState) { + return; + } + bob->appendNumber("fetchedMissingDocs", _initialSyncState->fetchedMissingDocs); + bob->appendNumber("appliedOps", _initialSyncState->appliedOps); + if (!_initialSyncState->beginTimestamp.isNull()) { + bob->append("initialSyncOplogStart", _initialSyncState->beginTimestamp); + } + if (!_initialSyncState->stopTimestamp.isNull()) { + bob->append("initialSyncOplogEnd", _initialSyncState->stopTimestamp); + } +} + BSONObj InitialSyncer::_getInitialSyncProgress_inlock() const { - BSONObjBuilder bob; try { - _stats.append(&bob); + BSONObjBuilder bob; + _appendInitialSyncProgressMinimal_inlock(&bob); if (_initialSyncState) { - bob.appendNumber("fetchedMissingDocs", _initialSyncState->fetchedMissingDocs); - bob.appendNumber("appliedOps", _initialSyncState->appliedOps); - if (!_initialSyncState->beginTimestamp.isNull()) { - bob.append("initialSyncOplogStart", _initialSyncState->beginTimestamp); - } - if (!_initialSyncState->stopTimestamp.isNull()) { - bob.append("initialSyncOplogEnd", _initialSyncState->stopTimestamp); - } if (_initialSyncState->dbsCloner) { BSONObjBuilder dbsBuilder(bob.subobjStart("databases")); _initialSyncState->dbsCloner->getStats().append(&dbsBuilder); dbsBuilder.doneFast(); } } + // In 3.4, BSONObjBuilder::obj() does not check the validity of the document it returns so + // we check it explicitly here. + auto obj = bob.obj(); + uassert(ErrorCodes::InvalidBSON, + str::stream() << "Invalid BSONObj size: " << obj.objsize(), + obj.isValid()); + return obj; } catch (const DBException& e) { - bob.resetToEmpty(); - bob.append("error", e.toString()); log() << "Error creating initial sync progress object: " << e.toString(); } + BSONObjBuilder bob; + _appendInitialSyncProgressMinimal_inlock(&bob); return bob.obj(); } diff --git a/src/mongo/db/repl/initial_syncer.h b/src/mongo/db/repl/initial_syncer.h index 6faeddb385e..cf91932f062 100644 --- a/src/mongo/db/repl/initial_syncer.h +++ b/src/mongo/db/repl/initial_syncer.h @@ -35,6 +35,7 @@ #include "mongo/base/status.h" #include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/timestamp.h" #include "mongo/db/namespace_string.h" #include "mongo/db/repl/callback_completion_guard.h" @@ -480,6 +481,7 @@ private: Fetcher::Documents::const_iterator end, const OplogFetcher::DocumentsInfo& info); + void _appendInitialSyncProgressMinimal_inlock(BSONObjBuilder* bob) const; BSONObj _getInitialSyncProgress_inlock() const; StatusWith<Operations> _getNextApplierBatch_inlock(); diff --git a/src/mongo/db/repl/initial_syncer_test.cpp b/src/mongo/db/repl/initial_syncer_test.cpp index cb1dccafaa7..3895f10fe3f 100644 --- a/src/mongo/db/repl/initial_syncer_test.cpp +++ b/src/mongo/db/repl/initial_syncer_test.cpp @@ -332,7 +332,7 @@ protected: options.getMyLastOptime = [this]() { return _myLastOpTime; }; options.setMyLastOptime = [this](const OpTime& opTime) { _setMyLastOptime(opTime); }; options.resetOptimes = [this]() { _setMyLastOptime(OpTime()); }; - options.getSlaveDelay = [this]() { return Seconds(0); }; + options.getSlaveDelay = []() { return Seconds(0); }; options.syncSourceSelector = this; _options = options; @@ -3588,4 +3588,66 @@ TEST_F(InitialSyncerTest, GetInitialSyncProgressReturnsCorrectProgress) { << attempt1; } +TEST_F(InitialSyncerTest, GetInitialSyncProgressOmitsClonerStatsIfClonerStatsExceedBsonLimit) { + auto initialSyncer = &getInitialSyncer(); + auto opCtx = makeOpCtx(); + + _syncSourceSelector->setChooseNewSyncSourceResult_forTest(HostAndPort("localhost", 27017)); + ASSERT_OK(initialSyncer->startup(opCtx.get(), 2U)); + + const std::size_t numCollections = 200000U; + + auto net = getNet(); + int baseRollbackId = 1; + { + executor::NetworkInterfaceMock::InNetworkGuard guard(net); + + // Base rollback ID. + net->scheduleSuccessfulResponse(makeRollbackCheckerResponse(baseRollbackId)); + net->runReadyNetworkOperations(); + + // Last oplog entry. + processSuccessfulLastOplogEntryFetcherResponse({makeOplogEntry(1)}); + + // Ignore oplog tailing query. + auto noi = net->getNextReadyRequest(); + auto request = noi->getRequest(); + assertRemoteCommandNameEquals("find", request); + ASSERT_TRUE(request.cmdObj.getBoolField("oplogReplay")); + net->blackHole(noi); + + // listDatabases + NamespaceString nss("a.a"); + request = net->scheduleSuccessfulResponse(makeListDatabasesResponse({nss.db().toString()})); + assertRemoteCommandNameEquals("listDatabases", request); + net->runReadyNetworkOperations(); + + // listCollections for "a" + std::vector<BSONObj> collectionInfos; + for (std::size_t i = 0; i < numCollections; ++i) { + const std::string collName = str::stream() << "coll-" << i; + collectionInfos.push_back(BSON("name" << collName << "options" << BSONObj())); + } + request = net->scheduleSuccessfulResponse( + makeCursorResponse(0LL, NamespaceString(nss.getCommandNS()), collectionInfos)); + assertRemoteCommandNameEquals("listCollections", request); + net->runReadyNetworkOperations(); + } + + // This returns a valid document because we omit the cloner stats when they do not fit in a + // BSON document. + auto progress = initialSyncer->getInitialSyncProgress(); + ASSERT_EQUALS(progress["initialSyncStart"].type(), Date) << progress; + ASSERT_FALSE(progress.hasField("databases")) << progress; + + // Initial sync will attempt to log stats again at shutdown in a callback, where it should not + // terminate because we now return a valid stats document. + ASSERT_OK(initialSyncer->shutdown()); + + // Deliver cancellation signal to callbacks. + executor::NetworkInterfaceMock::InNetworkGuard(net)->runReadyNetworkOperations(); + + initialSyncer->join(); +} + } // namespace diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index ad1a9fd93ba..10b60b7a205 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -2077,9 +2077,17 @@ Status ReplicationCoordinatorImpl::processReplSetGetStatus( BSONObj initialSyncProgress; if (responseStyle == ReplSetGetStatusResponseStyle::kInitialSync) { - LockGuard lk(_mutex); - if (_initialSyncer) { - initialSyncProgress = _initialSyncer->getInitialSyncProgress(); + std::shared_ptr<InitialSyncer> initialSyncerCopy; + { + LockGuard lk(_mutex); + initialSyncerCopy = _initialSyncer; + } + + // getInitialSyncProgress must be called outside the ReplicationCoordinatorImpl::_mutex + // lock. Else it might deadlock with InitialSyncer::_multiApplierCallback where it first + // acquires InitialSyncer::_mutex and then ReplicationCoordinatorImpl::_mutex. + if (initialSyncerCopy) { + initialSyncProgress = initialSyncerCopy->getInitialSyncProgress(); } } diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp index 3f65e3fc6b7..dfd1d564f90 100644 --- a/src/mongo/db/repl/sync_tail.cpp +++ b/src/mongo/db/repl/sync_tail.cpp @@ -775,6 +775,11 @@ private: void SyncTail::oplogApplication(ReplicationCoordinator* replCoord) { OpQueueBatcher batcher(this); + _oplogApplication(replCoord, &batcher); +} + +void SyncTail::_oplogApplication(ReplicationCoordinator* replCoord, + OpQueueBatcher* batcher) noexcept { std::unique_ptr<ApplyBatchFinalizer> finalizer{ getGlobalServiceContext()->getGlobalStorageEngine()->isDurable() ? new ApplyBatchFinalizerForJournal(replCoord) @@ -800,7 +805,7 @@ void SyncTail::oplogApplication(ReplicationCoordinator* replCoord) { long long termWhenBufferIsEmpty = replCoord->getTerm(); // Blocks up to a second waiting for a batch to be ready to apply. If one doesn't become // ready in time, we'll loop again so we can do the above checks periodically. - OpQueue ops = batcher.getNextBatch(Seconds(1)); + OpQueue ops = batcher->getNextBatch(Seconds(1)); if (ops.empty()) { if (ops.mustShutdown()) { return; diff --git a/src/mongo/db/repl/sync_tail.h b/src/mongo/db/repl/sync_tail.h index 1ad240c7890..51131de5099 100644 --- a/src/mongo/db/repl/sync_tail.h +++ b/src/mongo/db/repl/sync_tail.h @@ -230,6 +230,8 @@ protected: private: class OpQueueBatcher; + void _oplogApplication(ReplicationCoordinator* replCoord, OpQueueBatcher* batcher) noexcept; + std::string _hostname; BackgroundSync* _networkQueue; diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp index 30f0eeacad2..09953633f7e 100644 --- a/src/mongo/db/repl/sync_tail_test.cpp +++ b/src/mongo/db/repl/sync_tail_test.cpp @@ -47,6 +47,7 @@ #include "mongo/db/query/internal_plans.h" #include "mongo/db/repl/bgsync.h" #include "mongo/db/repl/oplog.h" +#include "mongo/db/repl/oplog_buffer_blocking_queue.h" #include "mongo/db/repl/oplog_interface_local.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/repl/replication_coordinator_mock.h" @@ -56,6 +57,7 @@ #include "mongo/db/service_context.h" #include "mongo/db/service_context_d_test_fixture.h" #include "mongo/stdx/mutex.h" +#include "mongo/unittest/death_test.h" #include "mongo/unittest/unittest.h" #include "mongo/util/concurrency/old_thread_pool.h" #include "mongo/util/md5.hpp" @@ -1089,6 +1091,43 @@ TEST_F(SyncTailTest, ASSERT_EQUALS(ErrorCodes::CollectionIsEmpty, iter->next().getStatus()); } +namespace { + +class ReplicationCoordinatorSignalDrainCompleteThrows : public ReplicationCoordinatorMock { +public: + explicit ReplicationCoordinatorSignalDrainCompleteThrows(const ReplSettings& settings) + : ReplicationCoordinatorMock(settings) {} + bool isInPrimaryOrSecondaryState() const final { + return true; + } + void signalDrainComplete(OperationContext*, long long) final { + uasserted(ErrorCodes::OperationFailed, "failed to signal drain complete"); + } +}; + +} // namespace + +DEATH_TEST_F(SyncTailTest, + OplogApplicationLogsExceptionFromSignalDrainCompleteBeforeAborting, + "failed to signal drain complete") { + // Leave oplog buffer empty so that SyncTail calls + // ReplicationCoordinator::signalDrainComplete() during oplog application. + auto oplogBuffer = stdx::make_unique<OplogBufferBlockingQueue>(); + BackgroundSync bgsync(nullptr, // ReplicationCoordinatorExternalState. Not used. + std::move(oplogBuffer)); + + auto applyOperationFn = [](MultiApplier::OperationPtrs*, SyncTail*) { return Status::OK(); }; + SyncTail syncTail(&bgsync, applyOperationFn); + + auto currentReplCoord = ReplicationCoordinator::get(_opCtx.get()); + ReplicationCoordinatorSignalDrainCompleteThrows replCoord(currentReplCoord->getSettings()); + ASSERT_TRUE(replCoord.setFollowerMode(MemberState::RS_PRIMARY)); + + // SyncTail::oplogApplication() creates its own OperationContext in the current thread context. + _opCtx = {}; + syncTail.oplogApplication(&replCoord); +} + class IdempotencyTest : public SyncTailTest { protected: OplogEntry createCollection(); diff --git a/src/mongo/db/repl/topology_coordinator_impl.cpp b/src/mongo/db/repl/topology_coordinator_impl.cpp index f5302008206..671dce8220b 100644 --- a/src/mongo/db/repl/topology_coordinator_impl.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl.cpp @@ -1622,9 +1622,12 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS if (_maintenanceModeCalls) { response->append("maintenanceMode", _maintenanceModeCalls); } - std::string s = _getHbmsg(now); - if (!s.empty()) - response->append("infoMessage", s); + response->append("lastHeartbeatMessage", ""); + response->append("syncingTo", ""); + response->append("syncSourceHost", ""); + response->append("syncSourceId", -1); + + response->append("infoMessage", _getHbmsg(now)); *result = Status(ErrorCodes::InvalidReplicaSetConfig, "Our replica set config is invalid or we are not a member of it"); return; @@ -1650,15 +1653,20 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS if (!_syncSource.empty() && !_iAmPrimary()) { bb.append("syncingTo", _syncSource.toString()); + bb.append("syncSourceHost", _syncSource.toString()); + const MemberConfig* member = _rsConfig.findMemberByHostAndPort(_syncSource); + bb.append("syncSourceId", member ? member->getId() : -1); + } else { + bb.append("syncingTo", ""); + bb.append("syncSourceHost", ""); + bb.append("syncSourceId", -1); } if (_maintenanceModeCalls) { bb.append("maintenanceMode", _maintenanceModeCalls); } - std::string s = _getHbmsg(now); - if (!s.empty()) - bb.append("infoMessage", s); + bb.append("infoMessage", _getHbmsg(now)); if (myState.primary()) { bb.append("electionTime", _electionTime); @@ -1667,6 +1675,7 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS } bb.appendIntOrLL("configVersion", _rsConfig.getConfigVersion()); bb.append("self", true); + bb.append("lastHeartbeatMessage", ""); membersOut.push_back(bb.obj()); } else { // add non-self member @@ -1706,9 +1715,7 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS Milliseconds ping = _getPing(itConfig.getHostAndPort()); if (ping != UninitializedPing) { bb.append("pingMs", durationCount<Milliseconds>(ping)); - std::string s = it->getLastHeartbeatMsg(); - if (!s.empty()) - bb.append("lastHeartbeatMessage", s); + bb.append("lastHeartbeatMessage", it->getLastHeartbeatMsg()); } if (it->hasAuthIssue()) { bb.append("authenticated", false); @@ -1716,8 +1723,17 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS const HostAndPort& syncSource = it->getSyncSource(); if (!syncSource.empty() && !state.primary()) { bb.append("syncingTo", syncSource.toString()); + bb.append("syncSourceHost", syncSource.toString()); + const MemberConfig* member = _rsConfig.findMemberByHostAndPort(syncSource); + bb.append("syncSourceId", member ? member->getId() : -1); + } else { + bb.append("syncingTo", ""); + bb.append("syncSourceHost", ""); + bb.append("syncSourceId", -1); } + bb.append("infoMessage", ""); + if (state == MemberState::RS_PRIMARY) { bb.append("electionTime", it->getElectionTime()); bb.appendDate( @@ -1740,6 +1756,13 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS // Add sync source info if (!_syncSource.empty() && !myState.primary() && !myState.removed()) { response->append("syncingTo", _syncSource.toString()); + response->append("syncSourceHost", _syncSource.toString()); + const MemberConfig* member = _rsConfig.findMemberByHostAndPort(_syncSource); + response->append("syncSourceId", member ? member->getId() : -1); + } else { + response->append("syncingTo", ""); + response->append("syncSourceHost", ""); + response->append("syncSourceId", -1); } if (_rsConfig.isConfigServer()) { diff --git a/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp b/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp index 8f60269cd71..57ca089e316 100644 --- a/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp @@ -178,7 +178,8 @@ protected: MemberState memberState, const OpTime& electionTime, const OpTime& lastOpTimeSender, - const OpTime& lastOpTimeReceiver) { + const OpTime& lastOpTimeReceiver, + const HostAndPort& syncingTo = HostAndPort()) { return _receiveHeartbeatHelper(Status::OK(), member, setName, @@ -186,7 +187,8 @@ protected: electionTime.getTimestamp(), lastOpTimeSender, lastOpTimeReceiver, - Milliseconds(1)); + Milliseconds(1), + syncingTo); } HeartbeatResponseAction receiveDownHeartbeat( @@ -204,7 +206,8 @@ protected: Timestamp(), OpTime(), lastOpTimeReceiver, - roundTripTime); + roundTripTime, + HostAndPort()); } HeartbeatResponseAction heartbeatFromMember(const HostAndPort& member, @@ -219,7 +222,8 @@ protected: Timestamp(), lastOpTimeSender, OpTime(), - roundTripTime); + roundTripTime, + HostAndPort()); } private: @@ -230,7 +234,8 @@ private: Timestamp electionTime, const OpTime& lastOpTimeSender, const OpTime& lastOpTimeReceiver, - Milliseconds roundTripTime) { + Milliseconds roundTripTime, + const HostAndPort& syncingTo) { ReplSetHeartbeatResponse hb; hb.setConfigVersion(1); hb.setState(memberState); @@ -238,6 +243,7 @@ private: hb.setAppliedOpTime(lastOpTimeSender); hb.setElectionTime(electionTime); hb.setTerm(getTopoCoord().getTerm()); + hb.setSyncingTo(syncingTo); StatusWith<ReplSetHeartbeatResponse> hbResponse = responseStatus.isOK() ? StatusWith<ReplSetHeartbeatResponse>(hb) @@ -3988,6 +3994,314 @@ TEST_F(HeartbeatResponseTestV1, UpdateHeartbeatDataTermPreventsPriorityTakeover) ASSERT_EQUALS(2, getCurrentPrimaryIndex()); } +TEST_F(TopoCoordTest, + StatusResponseAlwaysIncludesStringStatusFieldsForReplicaSetMembersNoHeartbeats) { + + Date_t heartbeatTime = Date_t::fromMillisSinceEpoch(5000); + Seconds uptimeSecs(10); + Date_t curTime = heartbeatTime + uptimeSecs; + OpTime oplogProgress(Timestamp(3, 4), 0); + + ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); + ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s); + updateConfig(BSON("_id" + << "rs0" + << "version" + << 5 + << "members" + << BSON_ARRAY(BSON("_id" << 0 << "host" + << "host0:27017") + << BSON("_id" << 1 << "host" + << "host1:27017")) + << "protocolVersion" + << 1), + 0); + { + BSONObjBuilder statusBuilder; + Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); + getTopoCoord().prepareStatusResponse( + TopologyCoordinator::ReplSetStatusArgs{ + curTime, + static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)), + oplogProgress, + oplogProgress, + oplogProgress, + OpTime(), + BSONObj()}, + &statusBuilder, + &resultStatus); + + ASSERT_OK(resultStatus); + BSONObj rsStatus = statusBuilder.obj(); + BSONObj member0Status = rsStatus["members"].Array()[0].Obj(); + BSONObj member1Status = rsStatus["members"].Array()[1].Obj(); + + // These fields should all be empty, since this node has not received heartbeats and has + // no sync source yet. + ASSERT_EQUALS("", rsStatus["syncingTo"].String()); + ASSERT_EQUALS("", rsStatus["syncSourceHost"].String()); + ASSERT_EQUALS(-1, rsStatus["syncSourceId"].numberInt()); + ASSERT_EQUALS("", member0Status["syncingTo"].String()); + ASSERT_EQUALS("", member0Status["syncSourceHost"].String()); + ASSERT_EQUALS(-1, member0Status["syncSourceId"].numberInt()); + ASSERT_EQUALS("", member0Status["lastHeartbeatMessage"].String()); + ASSERT_EQUALS("", member0Status["infoMessage"].String()); + ASSERT_EQUALS("", member1Status["syncingTo"].String()); + ASSERT_EQUALS("", member1Status["syncSourceHost"].String()); + ASSERT_EQUALS(-1, member1Status["syncSourceId"].numberInt()); + ASSERT_EQUALS("", member1Status["lastHeartbeatMessage"].String()); + ASSERT_EQUALS("", member1Status["infoMessage"].String()); + } +} + +TEST_F(TopoCoordTest, + StatusResponseAlwaysIncludesStringStatusFieldsForReplicaSetMembersWithHeartbeats) { + + Date_t heartbeatTime = Date_t::fromMillisSinceEpoch(5000); + Seconds uptimeSecs(10); + Date_t curTime = heartbeatTime + uptimeSecs; + OpTime oplogProgress(Timestamp(3, 4), 0); + + ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); + ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s); + updateConfig(BSON("_id" + << "rs0" + << "version" + << 5 + << "members" + << BSON_ARRAY(BSON("_id" << 0 << "host" + << "host0:27017") + << BSON("_id" << 1 << "host" + << "host1:27017")) + << "protocolVersion" + << 1), + 0); + + ASSERT(getTopoCoord().getSyncSourceAddress().empty()); + + // Receive heartbeats and choose a sync source. + setSelfMemberState(MemberState::RS_SECONDARY); + + OpTime election = OpTime(); + + // Record two rounds of pings so the node can pick a sync source. + receiveUpHeartbeat(HostAndPort("host1"), + "rs0", + MemberState::RS_PRIMARY, + election, + oplogProgress, + oplogProgress); + receiveUpHeartbeat(HostAndPort("host1"), + "rs0", + MemberState::RS_PRIMARY, + election, + oplogProgress, + oplogProgress); + + getTopoCoord().chooseNewSyncSource( + now(), OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration); + ASSERT_EQUALS(HostAndPort("host1"), getTopoCoord().getSyncSourceAddress()); + + { + BSONObjBuilder statusBuilder; + Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); + getTopoCoord().prepareStatusResponse( + TopologyCoordinator::ReplSetStatusArgs{ + curTime, + static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)), + oplogProgress, + oplogProgress, + oplogProgress, + OpTime(), + BSONObj()}, + &statusBuilder, + &resultStatus); + + ASSERT_OK(resultStatus); + BSONObj rsStatus = statusBuilder.obj(); + BSONObj member0Status = rsStatus["members"].Array()[0].Obj(); + BSONObj member1Status = rsStatus["members"].Array()[1].Obj(); + + // Node 0 (self) has received heartbeats and has a sync source. + ASSERT_EQUALS("host1:27017", rsStatus["syncingTo"].String()); + ASSERT_EQUALS("host1:27017", rsStatus["syncSourceHost"].String()); + ASSERT_EQUALS(1, rsStatus["syncSourceId"].numberInt()); + ASSERT_EQUALS("host1:27017", member0Status["syncingTo"].String()); + ASSERT_EQUALS("host1:27017", member0Status["syncSourceHost"].String()); + ASSERT_EQUALS(1, member0Status["syncSourceId"].numberInt()); + ASSERT_EQUALS("syncing from: host1:27017", member0Status["infoMessage"].String()); + ASSERT_EQUALS("", member0Status["lastHeartbeatMessage"].String()); + ASSERT_EQUALS("", member1Status["syncingTo"].String()); + ASSERT_EQUALS("", member1Status["syncSourceHost"].String()); + ASSERT_EQUALS(-1, member1Status["syncSourceId"].numberInt()); + ASSERT_EQUALS("", member1Status["infoMessage"].String()); + ASSERT_EQUALS("", member1Status["lastHeartbeatMessage"].String()); + } +} + +TEST_F(TopoCoordTest, replSetGetStatusForThreeMemberedReplicaSet) { + + Date_t heartbeatTime = Date_t::fromMillisSinceEpoch(5000); + Seconds uptimeSecs(10); + Date_t curTime = heartbeatTime + uptimeSecs; + OpTime oplogProgress(Timestamp(3, 4), 0); + + ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); + ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s); + updateConfig(BSON("_id" + << "rs0" + << "version" + << 5 + << "settings" + << BSON("chainingAllowed" << false) + << "members" + << BSON_ARRAY(BSON("_id" << 30 << "host" + << "hself:27017") + << BSON("_id" << 20 << "host" + << "hprimary:27017") + << BSON("_id" << 10 << "host" + << "h1:27017")) + << "protocolVersion" + << 1), + 0); + + ASSERT(getTopoCoord().getSyncSourceAddress().empty()); + + // Receive heartbeats and choose a sync source. + setSelfMemberState(MemberState::RS_SECONDARY); + + OpTime election = OpTime(); + + // Record two rounds of pings so the node can pick a sync source. + receiveUpHeartbeat(HostAndPort("hprimary"), + "rs0", + MemberState::RS_PRIMARY, + election, + oplogProgress, + oplogProgress); + receiveUpHeartbeat(HostAndPort("hprimary"), + "rs0", + MemberState::RS_PRIMARY, + election, + oplogProgress, + oplogProgress); + + // Mimic that h1 sends a heartbeat response with hprimary as syncsource. + receiveUpHeartbeat(HostAndPort("h1"), + "rs0", + MemberState::RS_SECONDARY, + election, + oplogProgress, + oplogProgress, + HostAndPort("hprimary")); + receiveUpHeartbeat(HostAndPort("h1"), + "rs0", + MemberState::RS_SECONDARY, + election, + oplogProgress, + oplogProgress, + HostAndPort("hprimary")); + + // Since chainingAllowed is disabled, hself should choose hprimary. + getTopoCoord().chooseNewSyncSource( + now(), OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration); + ASSERT_EQUALS(HostAndPort("hprimary"), getTopoCoord().getSyncSourceAddress()); + + BSONObjBuilder statusBuilder; + Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); + getTopoCoord().prepareStatusResponse( + TopologyCoordinator::ReplSetStatusArgs{ + curTime, + static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)), + oplogProgress, + oplogProgress, + oplogProgress, + OpTime(), + BSONObj()}, + &statusBuilder, + &resultStatus); + + ASSERT_OK(resultStatus); + BSONObj rsStatus = statusBuilder.obj(); + BSONObj member0Status = rsStatus["members"].Array()[0].Obj(); + BSONObj member1Status = rsStatus["members"].Array()[1].Obj(); + BSONObj member2Status = rsStatus["members"].Array()[2].Obj(); + + ASSERT_EQUALS("hprimary:27017", rsStatus["syncingTo"].String()); + ASSERT_EQUALS("hprimary:27017", rsStatus["syncSourceHost"].String()); + ASSERT_EQUALS(20, rsStatus["syncSourceId"].numberInt()); + + // h1 + ASSERT_EQUALS(10, member0Status["_id"].numberInt()); + ASSERT_EQUALS("hprimary:27017", member0Status["syncingTo"].String()); + ASSERT_EQUALS("hprimary:27017", member0Status["syncSourceHost"].String()); + ASSERT_EQUALS(20, member0Status["syncSourceId"].numberInt()); + ASSERT_EQUALS("", member0Status["infoMessage"].String()); + ASSERT_EQUALS("", member0Status["lastHeartbeatMessage"].String()); + + // hprimary + ASSERT_EQUALS(20, member1Status["_id"].numberInt()); + ASSERT_EQUALS("", member1Status["syncingTo"].String()); + ASSERT_EQUALS("", member1Status["syncSourceHost"].String()); + ASSERT_EQUALS(-1, member1Status["syncSourceId"].numberInt()); + ASSERT_EQUALS("", member1Status["infoMessage"].String()); + ASSERT_EQUALS("", member1Status["lastHeartbeatMessage"].String()); + + // hself + ASSERT_EQUALS(30, member2Status["_id"].numberInt()); + ASSERT_EQUALS("hprimary:27017", member2Status["syncingTo"].String()); + ASSERT_EQUALS("hprimary:27017", member2Status["syncSourceHost"].String()); + ASSERT_EQUALS(20, member2Status["syncSourceId"].numberInt()); + ASSERT_EQUALS("syncing from primary: hprimary:27017", member2Status["infoMessage"].String()); + ASSERT_EQUALS("", member2Status["lastHeartbeatMessage"].String()); +} + +TEST_F(TopoCoordTest, StatusResponseAlwaysIncludesStringStatusFieldsForNonMembers) { + Date_t heartbeatTime = Date_t::fromMillisSinceEpoch(5000); + Seconds uptimeSecs(10); + Date_t curTime = heartbeatTime + uptimeSecs; + OpTime oplogProgress(Timestamp(3, 4), 0); + + ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); + ASSERT_EQUALS(MemberState::RS_STARTUP, getTopoCoord().getMemberState().s); + updateConfig(BSON("_id" + << "rs0" + << "version" + << 5 + << "members" + << BSON_ARRAY(BSON("_id" << 0 << "host" + << "host0:27017")) + << "protocolVersion" + << 1), + -1); // This node is no longer part of this replica set. + + BSONObjBuilder statusBuilder; + Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); + getTopoCoord().prepareStatusResponse( + TopologyCoordinator::ReplSetStatusArgs{ + curTime, + static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)), + oplogProgress, + oplogProgress, + oplogProgress, + OpTime(), + BSONObj()}, + &statusBuilder, + &resultStatus); + + ASSERT_NOT_OK(resultStatus); + ASSERT_EQUALS(ErrorCodes::InvalidReplicaSetConfig, resultStatus); + + BSONObj rsStatus = statusBuilder.obj(); + + // These fields should all be empty, since this node is not a member of a replica set. + ASSERT_EQUALS("", rsStatus["lastHeartbeatMessage"].String()); + ASSERT_EQUALS("", rsStatus["syncingTo"].String()); + ASSERT_EQUALS("", rsStatus["syncSourceHost"].String()); + ASSERT_EQUALS(-1, rsStatus["syncSourceId"].numberInt()); + ASSERT_EQUALS("", rsStatus["infoMessage"].String()); +} + TEST_F(HeartbeatResponseTestV1, ScheduleElectionIfAMajorityOfVotersIsVisibleEvenThoughATrueMajorityIsNot) { updateConfig(BSON("_id" diff --git a/src/mongo/db/repl/vote_requester.cpp b/src/mongo/db/repl/vote_requester.cpp index bac35602398..861b7b4b9e5 100644 --- a/src/mongo/db/repl/vote_requester.cpp +++ b/src/mongo/db/repl/vote_requester.cpp @@ -99,9 +99,14 @@ void VoteRequester::Algorithm::processResponse(const RemoteCommandRequest& reque } _responders.insert(request.target); ReplSetRequestVotesResponse voteResponse; - const auto status = voteResponse.initialize(response.data); + auto status = getStatusFromCommandResult(response.data); + if (status.isOK()) { + status = voteResponse.initialize(response.data); + } if (!status.isOK()) { logLine << "received an invalid response from " << request.target << ": " << status; + logLine << "; response message: " << response.data; + return; } if (voteResponse.getVoteGranted()) { diff --git a/src/mongo/db/repl/vote_requester_test.cpp b/src/mongo/db/repl/vote_requester_test.cpp index 1dd97c456ea..02adb0a919a 100644 --- a/src/mongo/db/repl/vote_requester_test.cpp +++ b/src/mongo/db/repl/vote_requester_test.cpp @@ -31,6 +31,7 @@ #include <memory> #include "mongo/base/status.h" +#include "mongo/db/commands.h" #include "mongo/db/jsobj.h" #include "mongo/db/repl/repl_set_request_votes_args.h" #include "mongo/db/repl/replication_executor.h" @@ -109,7 +110,17 @@ protected: } void processResponse(const RemoteCommandRequest& request, const ResponseStatus& response) { - _requester->processResponse(request, response); + if (!response.isOK()) { + _requester->processResponse(request, response); + return; + } + BSONObjBuilder builder; + builder.appendElements(response.data); + // Appends ok:1.0 (status ok) to response data if 'ok' field is missing. + Command::appendCommandStatus(builder, Status::OK()); + ResponseStatus responseWithCmdStatus = response; + responseWithCmdStatus.data = builder.obj(); + _requester->processResponse(request, responseWithCmdStatus); } int getNumResponders() { @@ -140,6 +151,19 @@ protected: NetworkInterfaceMock::Response(response.toBSON(), BSONObj(), Milliseconds(10))); } + ResponseStatus votedYesStatusNotOkBecauseFailedToStoreLastVote() { + ReplSetRequestVotesResponse response; + BSONObjBuilder result; + response.setVoteGranted(true); + response.setTerm(1); + response.addToBSON(&result); + auto status = + Status(ErrorCodes::InterruptedDueToReplStateChange, "operation was interrupted"); + Command::appendCommandStatus(result, status); + return ResponseStatus( + NetworkInterfaceMock::Response(result.obj(), BSONObj(), Milliseconds(10))); + } + ResponseStatus votedNoBecauseConfigVersionDoesNotMatch() { ReplSetRequestVotesResponse response; response.setVoteGranted(false); @@ -236,6 +260,19 @@ TEST_F(VoteRequesterTest, ImmediateGoodResponseWinElection) { ASSERT_EQUALS(1, getNumResponders()); } +TEST_F(VoteRequesterTest, VoterFailedToStoreLastVote) { + startCapturingLogMessages(); + ASSERT_FALSE(hasReceivedSufficientResponses()); + processResponse(requestFrom("host1"), votedYesStatusNotOkBecauseFailedToStoreLastVote()); + ASSERT_FALSE(hasReceivedSufficientResponses()); + ASSERT_EQUALS(1, countLogLinesContaining("received an invalid response from host1:27017")); + processResponse(requestFrom("host2"), votedYes()); + ASSERT_TRUE(hasReceivedSufficientResponses()); + ASSERT(VoteRequester::Result::kSuccessfullyElected == getResult()); + ASSERT_EQUALS(2, getNumResponders()); + stopCapturingLogMessages(); +} + TEST_F(VoteRequesterTest, BadConfigVersionWinElection) { startCapturingLogMessages(); ASSERT_FALSE(hasReceivedSufficientResponses()); diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp index 781ccf1fccc..8dae501c57e 100644 --- a/src/mongo/db/s/migration_destination_manager.cpp +++ b/src/mongo/db/s/migration_destination_manager.cpp @@ -486,7 +486,7 @@ void MigrationDestinationManager::_migrateThread(BSONObj min, { stdx::lock_guard<stdx::mutex> sl(_mutex); _state = FAIL; - _errmsg = exceptionToStatus().toString(); + _errmsg = redact(exceptionToStatus().toString()); } error() << "migrate failed: " << redact(exceptionToStatus()) << migrateLog; @@ -1011,9 +1011,9 @@ bool MigrationDestinationManager::_applyMigrateOp(OperationContext* txn, BSONObj localDoc; if (willOverrideLocalId( txn, ns, min, max, shardKeyPattern, cx.db(), updatedDoc, &localDoc)) { - string errMsg = str::stream() << "cannot migrate chunk, local document " << localDoc - << " has same _id as reloaded remote document " - << updatedDoc; + string errMsg = str::stream() + << "cannot migrate chunk, local document " << redact(localDoc) + << " has same _id as reloaded remote document " << redact(updatedDoc); warning() << errMsg; @@ -1046,8 +1046,8 @@ bool MigrationDestinationManager::_flushPendingWrites(OperationContext* txn, return false; } - log() << "migrate commit succeeded flushing to secondaries for '" << ns << "' " << min << " -> " - << max << migrateLog; + log() << "migrate commit succeeded flushing to secondaries for '" << ns << "' " << redact(min) + << " -> " << redact(max) << migrateLog; return true; } @@ -1069,9 +1069,9 @@ Status MigrationDestinationManager::_notePending(OperationContext* txn, return {ErrorCodes::StaleShardVersion, str::stream() << "could not note chunk " << "[" - << min + << redact(min) << "," - << max + << redact(max) << ")" << " as pending because the epoch for " << nss.ns() diff --git a/src/mongo/dbtests/basictests.cpp b/src/mongo/dbtests/basictests.cpp index 9734c1d8aab..5ea9cc152b1 100644 --- a/src/mongo/dbtests/basictests.cpp +++ b/src/mongo/dbtests/basictests.cpp @@ -217,73 +217,6 @@ public: }; } -class sleeptest { -public: - void run() { - Timer t; - int matches = 0; - for (int p = 0; p < 3; p++) { - sleepsecs(1); - int sec = (t.millis() + 2) / 1000; - if (sec == 1) - matches++; - else - mongo::unittest::log() << "temp millis: " << t.millis() << endl; - ASSERT(sec >= 0 && sec <= 2); - t.reset(); - } - if (matches < 2) - mongo::unittest::log() << "matches:" << matches << endl; - ASSERT(matches >= 2); - - sleepmicros(1527123); - ASSERT(t.micros() > 1000000); - ASSERT(t.micros() < 2000000); - - t.reset(); - sleepmillis(1727); - ASSERT(t.millis() >= 1000); - ASSERT(t.millis() <= 2500); - - { - int total = 1200; - int ms = 2; - t.reset(); - for (int i = 0; i < (total / ms); i++) { - sleepmillis(ms); - } - { - int x = t.millis(); - if (x < 1000 || x > 2500) { - cout << "sleeptest finds sleep accuracy to be not great. x: " << x << endl; - ASSERT(x >= 1000); - ASSERT(x <= 20000); - } - } - } - -#ifdef __linux__ - { - int total = 1200; - int micros = 100; - t.reset(); - int numSleeps = 1000 * (total / micros); - for (int i = 0; i < numSleeps; i++) { - sleepmicros(micros); - } - { - int y = t.millis(); - if (y < 1000 || y > 2500) { - cout << "sleeptest y: " << y << endl; - ASSERT(y >= 1000); - /* ASSERT( y <= 100000 ); */ - } - } - } -#endif - } -}; - class SleepBackoffTest { public: void run() { @@ -494,7 +427,6 @@ public: add<stringbuildertests::reset1>(); add<stringbuildertests::reset2>(); - add<sleeptest>(); add<SleepBackoffTest>(); add<AssertTests>(); diff --git a/src/mongo/executor/SConscript b/src/mongo/executor/SConscript index ed040af2910..0359c40a020 100644 --- a/src/mongo/executor/SConscript +++ b/src/mongo/executor/SConscript @@ -168,6 +168,7 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/base/system_error', '$BUILD_DIR/mongo/client/authentication', + '$BUILD_DIR/mongo/util/net/network', '$BUILD_DIR/third_party/shim_asio', 'task_executor_interface', ] diff --git a/src/mongo/executor/async_secure_stream.cpp b/src/mongo/executor/async_secure_stream.cpp index 0c7969024f7..a7c051bc34b 100644 --- a/src/mongo/executor/async_secure_stream.cpp +++ b/src/mongo/executor/async_secure_stream.cpp @@ -74,6 +74,11 @@ void AsyncSecureStream::connect(asio::ip::tcp::resolver::iterator endpoints, return _userHandler(ec); } + ec = setStreamKeepAlive(&_stream.next_layer()); + if (ec) { + return _userHandler(ec); + } + _connected = true; return _handleConnect(std::move(iter)); })); diff --git a/src/mongo/executor/async_stream.cpp b/src/mongo/executor/async_stream.cpp index 075a9094dd4..aeafd8a0b8b 100644 --- a/src/mongo/executor/async_stream.cpp +++ b/src/mongo/executor/async_stream.cpp @@ -70,6 +70,11 @@ void AsyncStream::connect(tcp::resolver::iterator iter, ConnectHandler&& connect return connectHandler(ec); } + ec = setStreamKeepAlive(&_stream); + if (ec) { + return connectHandler(ec); + } + _connected = true; return connectHandler(ec); })); diff --git a/src/mongo/executor/async_stream_common.cpp b/src/mongo/executor/async_stream_common.cpp index 92708862c94..97cfb3115b0 100644 --- a/src/mongo/executor/async_stream_common.cpp +++ b/src/mongo/executor/async_stream_common.cpp @@ -57,6 +57,11 @@ void logFailureInSetStreamNoDelay(std::error_code ec) { severe() << "Failed to set no-delay mode on stream: " << ec.message(); } +void logFailureInSetStreamKeepAlive(std::error_code ec) { + invariant(ec); + severe() << "Failed to set keep-alive mode on stream: " << ec.message(); +} + void logUnexpectedErrorInCheckOpen(std::error_code ec) { invariant(ec); log() << "unexpected error when checking if a stream was open: " << ec.message() diff --git a/src/mongo/executor/async_stream_common.h b/src/mongo/executor/async_stream_common.h index 67c12bbff3b..5ca7dcad5fe 100644 --- a/src/mongo/executor/async_stream_common.h +++ b/src/mongo/executor/async_stream_common.h @@ -31,6 +31,7 @@ #include <utility> #include "mongo/util/assert_util.h" +#include "mongo/util/net/sock.h" namespace mongo { namespace executor { @@ -93,6 +94,7 @@ void cancelStream(ASIOStream* stream) { void logFailureInSetStreamNonBlocking(std::error_code ec); void logFailureInSetStreamNoDelay(std::error_code ec); +void logFailureInSetStreamKeepAlive(std::error_code ec); template <typename ASIOStream> std::error_code setStreamNonBlocking(ASIOStream* stream) { @@ -114,6 +116,17 @@ std::error_code setStreamNoDelay(ASIOStream* stream) { return ec; } +template <typename ASIOStream> +std::error_code setStreamKeepAlive(ASIOStream* stream) { + std::error_code ec; + stream->set_option(asio::socket_base::keep_alive(true), ec); + if (ec) { + logFailureInSetStreamKeepAlive(ec); + } + setSocketKeepAliveParams(stream->native_handle()); + return ec; +} + void logUnexpectedErrorInCheckOpen(std::error_code ec); template <typename ASIOStream> diff --git a/src/mongo/s/catalog/sharding_catalog_assign_key_range_to_zone_test.cpp b/src/mongo/s/catalog/sharding_catalog_assign_key_range_to_zone_test.cpp index 104a97a3f6a..39d2142e6c9 100644 --- a/src/mongo/s/catalog/sharding_catalog_assign_key_range_to_zone_test.cpp +++ b/src/mongo/s/catalog/sharding_catalog_assign_key_range_to_zone_test.cpp @@ -31,6 +31,7 @@ #include "mongo/bson/json.h" #include "mongo/client/read_preference.h" #include "mongo/db/namespace_string.h" +#include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/catalog/sharding_catalog_manager.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/catalog/type_collection.h" @@ -183,6 +184,53 @@ TEST_F(AssignKeyRangeToZoneTestFixture, MaxWithInvalidShardKeyShouldFail) { assertNoZoneDoc(); } + +TEST_F(AssignKeyRangeToZoneTestFixture, AssignZoneWithDollarPrefixedShardKeysShouldFail) { + ASSERT_NOT_OK(catalogManager()->assignKeyRangeToZone( + operationContext(), + shardedNS(), + ChunkRange(BSON("x" << BSON("$A" << 1)), BSON("x" << BSON("$B" << 1))), + zoneName())); + assertNoZoneDoc(); + + ASSERT_NOT_OK(catalogManager()->assignKeyRangeToZone( + operationContext(), + shardedNS(), + ChunkRange(BSON("x" << 0), BSON("x" << BSON("$maxKey" << 1))), + zoneName())); + assertNoZoneDoc(); +} + +TEST_F(AssignKeyRangeToZoneTestFixture, RemoveZoneWithDollarPrefixedShardKeysShouldFail) { + ChunkRange zoneWithDollarKeys(BSON("x" << BSON("$A" << 1)), BSON("x" << BSON("$B" << 1))); + + // Manually insert a zone with illegal keys in order to bypass the checks performed by + // assignKeyRangeToZone + BSONObj updateQuery(BSON("_id" << BSON(TagsType::ns(shardedNS().ns()) + << TagsType::min(zoneWithDollarKeys.getMin())))); + + BSONObjBuilder updateBuilder; + updateBuilder.append( + "_id", BSON(TagsType::ns(shardedNS().ns()) << TagsType::min(zoneWithDollarKeys.getMin()))); + updateBuilder.append(TagsType::ns(), shardedNS().ns()); + updateBuilder.append(TagsType::min(), zoneWithDollarKeys.getMin()); + updateBuilder.append(TagsType::max(), zoneWithDollarKeys.getMax()); + updateBuilder.append(TagsType::tag(), "TestZone"); + + ASSERT_OK(catalogClient()->updateConfigDocument( + operationContext(), + TagsType::ConfigNS, + updateQuery, + updateBuilder.obj(), + true, + WriteConcernOptions(1, WriteConcernOptions::SyncMode::UNSET, Seconds(0)))); + assertOnlyZone(shardedNS(), zoneWithDollarKeys, "TestZone"); + + ASSERT_OK(catalogManager()->removeKeyRangeFromZone( + operationContext(), shardedNS(), zoneWithDollarKeys)); + assertNoZoneDoc(); +} + TEST_F(AssignKeyRangeToZoneTestFixture, MinThatIsAShardKeyPrefixShouldConvertToFullShardKey) { NamespaceString ns("compound.shard"); CollectionType shardedCollection; diff --git a/src/mongo/s/catalog/sharding_catalog_manager_chunk_operations_impl.cpp b/src/mongo/s/catalog/sharding_catalog_manager_chunk_operations_impl.cpp index 5b200006aba..5f3b57dc5ed 100644 --- a/src/mongo/s/catalog/sharding_catalog_manager_chunk_operations_impl.cpp +++ b/src/mongo/s/catalog/sharding_catalog_manager_chunk_operations_impl.cpp @@ -377,9 +377,16 @@ Status ShardingCatalogManagerImpl::commitChunkSplit(OperationContext* txn, } // verify that splits don't create too-big shard keys - Status shardKeyStatus = ShardKeyPattern::checkShardKeySize(endKey); - if (!shardKeyStatus.isOK()) { - return shardKeyStatus; + Status shardKeySizeStatus = ShardKeyPattern::checkShardKeySize(endKey); + if (!shardKeySizeStatus.isOK()) { + return shardKeySizeStatus; + } + + // verify that splits don't use disallowed BSON object format + Status shardKeyStorageStatus = + ShardKeyPattern::checkShardKeyIsValidForMetadataStorage(endKey); + if (!shardKeyStorageStatus.isOK()) { + return shardKeyStorageStatus; } // splits only update the 'minor' portion of version diff --git a/src/mongo/s/catalog/sharding_catalog_manager_zone_operations_impl.cpp b/src/mongo/s/catalog/sharding_catalog_manager_zone_operations_impl.cpp index 6af584d3c30..d16e6083f67 100644 --- a/src/mongo/s/catalog/sharding_catalog_manager_zone_operations_impl.cpp +++ b/src/mongo/s/catalog/sharding_catalog_manager_zone_operations_impl.cpp @@ -313,6 +313,17 @@ Status ShardingCatalogManagerImpl::assignKeyRangeToZone(OperationContext* txn, const NamespaceString& ns, const ChunkRange& givenRange, const std::string& zoneName) { + auto zoneBoundsStorageStatus = + ShardKeyPattern::checkShardKeyIsValidForMetadataStorage(givenRange.getMin()); + if (zoneBoundsStorageStatus.isOK()) { + zoneBoundsStorageStatus = + ShardKeyPattern::checkShardKeyIsValidForMetadataStorage(givenRange.getMax()); + } + + if (!zoneBoundsStorageStatus.isOK()) { + return zoneBoundsStorageStatus; + } + Lock::ExclusiveLock lk(txn->lockState(), _kZoneOpLock); auto configServer = Grid::get(txn)->shardRegistry()->getConfigShard(); diff --git a/src/mongo/s/catalog/sharding_catalog_merge_chunks_test.cpp b/src/mongo/s/catalog/sharding_catalog_merge_chunks_test.cpp index d457bd5de2d..cced8fe729a 100644 --- a/src/mongo/s/catalog/sharding_catalog_merge_chunks_test.cpp +++ b/src/mongo/s/catalog/sharding_catalog_merge_chunks_test.cpp @@ -460,5 +460,68 @@ TEST_F(MergeChunkTest, ChunkBoundariesOutOfOrderFails) { "shard0000")); } +TEST_F(MergeChunkTest, MergingChunksWithDollarPrefixShouldSucceed) { + ChunkType chunk1; + chunk1.setNS("TestDB.TestColl"); + + auto origVersion = ChunkVersion(1, 0, OID::gen()); + chunk1.setVersion(origVersion); + chunk1.setShard(ShardId("shard0000")); + + auto chunk2(chunk1); + auto chunk3(chunk1); + + auto chunkMin = BSON("a" << kMinBSONKey); + auto chunkBound1 = BSON("a" << BSON("$maxKey" << 1)); + auto chunkBound2 = BSON("a" << BSON("$mixKey" << 1)); + auto chunkMax = BSON("a" << kMaxBSONKey); + + // first chunk boundaries + chunk1.setMin(chunkMin); + chunk1.setMax(chunkBound1); + // second chunk boundaries + chunk2.setMin(chunkBound1); + chunk2.setMax(chunkBound2); + // third chunk boundaries + chunk3.setMin(chunkBound2); + chunk3.setMax(chunkMax); + + ASSERT_OK(setupChunks({chunk1, chunk2, chunk3})); + + // Record chunk boundaries for passing into commitChunkMerge + std::vector<BSONObj> chunkBoundaries{chunkMin, chunkBound1, chunkBound2, chunkMax}; + + ASSERT_OK(catalogManager()->commitChunkMerge(operationContext(), + NamespaceString("TestDB.TestColl"), + origVersion.epoch(), + chunkBoundaries, + "shard0000")); + + auto findResponse = uassertStatusOK( + getConfigShard()->exhaustiveFindOnConfig(operationContext(), + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + repl::ReadConcernLevel::kLocalReadConcern, + NamespaceString(ChunkType::ConfigNS), + BSON(ChunkType::ns() << "TestDB.TestColl"), + BSON(ChunkType::DEPRECATED_lastmod() << -1), + boost::none)); + + const auto& chunksVector = findResponse.docs; + + // There should be exactly one chunk left in the collection + ASSERT_EQ(1u, chunksVector.size()); + + // MergedChunk should have range [chunkMin, chunkMax] + auto mergedChunk = uassertStatusOK(ChunkType::fromBSON(chunksVector.front())); + ASSERT_BSONOBJ_EQ(chunkMin, mergedChunk.getMin()); + ASSERT_BSONOBJ_EQ(chunkMax, mergedChunk.getMax()); + + { + // Check for increment on mergedChunk's minor version + ASSERT_EQ(origVersion.majorVersion(), mergedChunk.getVersion().majorVersion()); + ASSERT_EQ(origVersion.minorVersion() + 1, mergedChunk.getVersion().minorVersion()); + } +} + } // namespace } // namespace mongo diff --git a/src/mongo/s/catalog/sharding_catalog_split_chunk_test.cpp b/src/mongo/s/catalog/sharding_catalog_split_chunk_test.cpp index 8373636edad..27c4ed35950 100644 --- a/src/mongo/s/catalog/sharding_catalog_split_chunk_test.cpp +++ b/src/mongo/s/catalog/sharding_catalog_split_chunk_test.cpp @@ -364,5 +364,33 @@ TEST_F(SplitChunkTest, SplitPointsOutOfRangeAtMaxShouldFail) { ASSERT_EQ(ErrorCodes::InvalidOptions, splitStatus); } +TEST_F(SplitChunkTest, SplitPointsWithDollarPrefixShouldFail) { + ChunkType chunk; + chunk.setNS("TestDB.TestColl"); + + auto origVersion = ChunkVersion(1, 0, OID::gen()); + chunk.setVersion(origVersion); + chunk.setShard(ShardId("shard0000")); + + auto chunkMin = BSON("a" << kMinBSONKey); + auto chunkMax = BSON("a" << kMaxBSONKey); + chunk.setMin(chunkMin); + chunk.setMax(chunkMax); + ASSERT_OK(setupChunks({chunk})); + + ASSERT_NOT_OK(catalogManager()->commitChunkSplit(operationContext(), + NamespaceString("TestDB.TestColl"), + origVersion.epoch(), + ChunkRange(chunkMin, chunkMax), + {BSON("a" << BSON("$minKey" << 1))}, + "shard0000")); + ASSERT_NOT_OK(catalogManager()->commitChunkSplit(operationContext(), + NamespaceString("TestDB.TestColl"), + origVersion.epoch(), + ChunkRange(chunkMin, chunkMax), + {BSON("a" << BSON("$maxKey" << 1))}, + "shard0000")); +} + } // namespace } // namespace mongo diff --git a/src/mongo/s/catalog/type_mongos.cpp b/src/mongo/s/catalog/type_mongos.cpp index c656f59b4ee..c5e02a69f15 100644 --- a/src/mongo/s/catalog/type_mongos.cpp +++ b/src/mongo/s/catalog/type_mongos.cpp @@ -43,6 +43,7 @@ const BSONField<long long> MongosType::uptime("up"); const BSONField<bool> MongosType::waiting("waiting"); const BSONField<std::string> MongosType::mongoVersion("mongoVersion"); const BSONField<long long> MongosType::configVersion("configVersion"); +const BSONField<BSONArray> MongosType::advisoryHostFQDNs("advisoryHostFQDNs"); StatusWith<MongosType> MongosType::fromBSON(const BSONObj& source) { MongosType mt; @@ -95,6 +96,26 @@ StatusWith<MongosType> MongosType::fromBSON(const BSONObj& source) { mt._configVersion = mtConfigVersion; } + if (source.hasField(advisoryHostFQDNs.name())) { + mt._advisoryHostFQDNs = std::vector<std::string>(); + BSONElement array; + Status status = bsonExtractTypedField(source, advisoryHostFQDNs.name(), Array, &array); + if (!status.isOK()) + return status; + + BSONObjIterator it(array.Obj()); + while (it.more()) { + BSONElement arrayElement = it.next(); + if (arrayElement.type() != String) { + return Status(ErrorCodes::TypeMismatch, + str::stream() << "Elements in \"" << advisoryHostFQDNs.name() + << "\" array must be strings but found " + << typeName(arrayElement.type())); + } + mt._advisoryHostFQDNs->push_back(arrayElement.String()); + } + } + return mt; } @@ -133,6 +154,8 @@ BSONObj MongosType::toBSON() const { builder.append(mongoVersion.name(), getMongoVersion()); if (_configVersion) builder.append(configVersion.name(), getConfigVersion()); + if (_advisoryHostFQDNs) + builder.append(advisoryHostFQDNs.name(), getAdvisoryHostFQDNs()); return builder.obj(); } @@ -163,6 +186,10 @@ void MongosType::setConfigVersion(const long long configVersion) { _configVersion = configVersion; } +void MongosType::setAdvisoryHostFQDNs(const std::vector<std::string>& advisoryHostFQDNs) { + _advisoryHostFQDNs = advisoryHostFQDNs; +} + std::string MongosType::toString() const { return toBSON().toString(); } diff --git a/src/mongo/s/catalog/type_mongos.h b/src/mongo/s/catalog/type_mongos.h index 02e7fea601b..e0fb0ea84b1 100644 --- a/src/mongo/s/catalog/type_mongos.h +++ b/src/mongo/s/catalog/type_mongos.h @@ -30,6 +30,7 @@ #include <boost/optional.hpp> #include <string> +#include <vector> #include "mongo/db/jsobj.h" #include "mongo/util/time_support.h" @@ -53,6 +54,7 @@ public: static const BSONField<bool> waiting; static const BSONField<std::string> mongoVersion; static const BSONField<long long> configVersion; + static const BSONField<BSONArray> advisoryHostFQDNs; /** * Returns the BSON representation of the entry. @@ -112,6 +114,11 @@ public: } void setConfigVersion(const long long configVersion); + std::vector<std::string> getAdvisoryHostFQDNs() const { + return _advisoryHostFQDNs.value_or(std::vector<std::string>()); + } + void setAdvisoryHostFQDNs(const std::vector<std::string>& advisoryHostFQDNs); + private: // Convention: (M)andatory, (O)ptional, (S)pecial rule. @@ -127,6 +134,8 @@ private: boost::optional<std::string> _mongoVersion; // (O) the config version of the pinging mongos boost::optional<long long> _configVersion; + // (O) the results of hostname canonicalization on the pinging mongos + boost::optional<std::vector<std::string>> _advisoryHostFQDNs; }; } // namespace mongo diff --git a/src/mongo/s/catalog/type_mongos_test.cpp b/src/mongo/s/catalog/type_mongos_test.cpp index 8fc449ab6f7..dad38d140cf 100644 --- a/src/mongo/s/catalog/type_mongos_test.cpp +++ b/src/mongo/s/catalog/type_mongos_test.cpp @@ -39,21 +39,24 @@ namespace { using namespace mongo; TEST(Validity, MissingName) { - BSONObj obj = - BSON(MongosType::ping(Date_t::fromMillisSinceEpoch(1)) << MongosType::uptime(100) - << MongosType::waiting(false) - << MongosType::mongoVersion("x.x.x") - << MongosType::configVersion(0)); + BSONObj obj = BSON(MongosType::ping(Date_t::fromMillisSinceEpoch(1)) + << MongosType::uptime(100) + << MongosType::waiting(false) + << MongosType::mongoVersion("x.x.x") + << MongosType::configVersion(0) + << MongosType::advisoryHostFQDNs(BSONArrayBuilder().arr())); auto mongosTypeResult = MongosType::fromBSON(obj); ASSERT_EQ(ErrorCodes::NoSuchKey, mongosTypeResult.getStatus()); } TEST(Validity, MissingPing) { - BSONObj obj = BSON(MongosType::name("localhost:27017") << MongosType::uptime(100) - << MongosType::waiting(false) - << MongosType::mongoVersion("x.x.x") - << MongosType::configVersion(0)); + BSONObj obj = BSON(MongosType::name("localhost:27017") + << MongosType::uptime(100) + << MongosType::waiting(false) + << MongosType::mongoVersion("x.x.x") + << MongosType::configVersion(0) + << MongosType::advisoryHostFQDNs(BSONArrayBuilder().arr())); auto mongosTypeResult = MongosType::fromBSON(obj); ASSERT_EQ(ErrorCodes::NoSuchKey, mongosTypeResult.getStatus()); @@ -64,7 +67,8 @@ TEST(Validity, MissingUp) { << MongosType::ping(Date_t::fromMillisSinceEpoch(1)) << MongosType::waiting(false) << MongosType::mongoVersion("x.x.x") - << MongosType::configVersion(0)); + << MongosType::configVersion(0) + << MongosType::advisoryHostFQDNs(BSONArrayBuilder().arr())); auto mongosTypeResult = MongosType::fromBSON(obj); ASSERT_EQ(ErrorCodes::NoSuchKey, mongosTypeResult.getStatus()); @@ -75,7 +79,8 @@ TEST(Validity, MissingWaiting) { << MongosType::ping(Date_t::fromMillisSinceEpoch(1)) << MongosType::uptime(100) << MongosType::mongoVersion("x.x.x") - << MongosType::configVersion(0)); + << MongosType::configVersion(0) + << MongosType::advisoryHostFQDNs(BSONArrayBuilder().arr())); auto mongosTypeResult = MongosType::fromBSON(obj); ASSERT_EQ(ErrorCodes::NoSuchKey, mongosTypeResult.getStatus()); @@ -86,7 +91,8 @@ TEST(Validity, MissingMongoVersion) { << MongosType::ping(Date_t::fromMillisSinceEpoch(1)) << MongosType::uptime(100) << MongosType::waiting(false) - << MongosType::configVersion(0)); + << MongosType::configVersion(0) + << MongosType::advisoryHostFQDNs(BSONArrayBuilder().arr())); auto mongosTypeResult = MongosType::fromBSON(obj); ASSERT_OK(mongosTypeResult.getStatus()); @@ -104,7 +110,8 @@ TEST(Validity, MissingConfigVersion) { << MongosType::ping(Date_t::fromMillisSinceEpoch(1)) << MongosType::uptime(100) << MongosType::waiting(false) - << MongosType::mongoVersion("x.x.x")); + << MongosType::mongoVersion("x.x.x") + << MongosType::advisoryHostFQDNs(BSONArrayBuilder().arr())); auto mongosTypeResult = MongosType::fromBSON(obj); ASSERT_OK(mongosTypeResult.getStatus()); @@ -117,7 +124,7 @@ TEST(Validity, MissingConfigVersion) { ASSERT_OK(mtype.validate()); } -TEST(Validity, Valid) { +TEST(Validity, MissingAdvisoryHostFQDNs) { BSONObj obj = BSON(MongosType::name("localhost:27017") << MongosType::ping(Date_t::fromMillisSinceEpoch(1)) << MongosType::uptime(100) @@ -126,8 +133,59 @@ TEST(Validity, Valid) { << MongosType::configVersion(0)); auto mongosTypeResult = MongosType::fromBSON(obj); + ASSERT_OK(mongosTypeResult.getStatus()); + MongosType& mType = mongosTypeResult.getValue(); + // advisoryHostFQDNs is optional + ASSERT_OK(mType.validate()); +} +TEST(Validity, EmptyAdvisoryHostFQDNs) { + BSONObj obj = BSON(MongosType::name("localhost:27017") + << MongosType::ping(Date_t::fromMillisSinceEpoch(1)) + << MongosType::uptime(100) + << MongosType::waiting(false) + << MongosType::mongoVersion("x.x.x") + << MongosType::configVersion(0) + << MongosType::advisoryHostFQDNs(BSONArrayBuilder().arr())); + + auto mongosTypeResult = MongosType::fromBSON(obj); + ASSERT_OK(mongosTypeResult.getStatus()); + + MongosType& mType = mongosTypeResult.getValue(); + ASSERT_OK(mType.validate()); + + ASSERT_EQUALS(mType.getAdvisoryHostFQDNs().size(), 0UL); +} + +TEST(Validity, BadTypeAdvisoryHostFQDNs) { + BSONObj obj = BSON(MongosType::name("localhost:27017") + << MongosType::ping(Date_t::fromMillisSinceEpoch(1)) + << MongosType::uptime(100) + << MongosType::waiting(false) + << MongosType::mongoVersion("x.x.x") + << MongosType::configVersion(0) + << MongosType::advisoryHostFQDNs(BSON_ARRAY("foo" << 0 << "baz"))); + + auto mongosTypeResult = MongosType::fromBSON(obj); + ASSERT_EQ(ErrorCodes::TypeMismatch, mongosTypeResult.getStatus()); +} + +TEST(Validity, Valid) { + BSONObj obj = BSON(MongosType::name("localhost:27017") + << MongosType::ping(Date_t::fromMillisSinceEpoch(1)) + << MongosType::uptime(100) + << MongosType::waiting(false) + << MongosType::mongoVersion("x.x.x") + << MongosType::configVersion(0) + << MongosType::advisoryHostFQDNs(BSON_ARRAY("foo" + << "bar" + << "baz"))); + + auto mongosTypeResult = MongosType::fromBSON(obj); + ASSERT_OK(mongosTypeResult.getStatus()); + + MongosType& mType = mongosTypeResult.getValue(); ASSERT_OK(mType.validate()); ASSERT_EQUALS(mType.getName(), "localhost:27017"); @@ -136,6 +194,10 @@ TEST(Validity, Valid) { ASSERT_EQUALS(mType.getWaiting(), false); ASSERT_EQUALS(mType.getMongoVersion(), "x.x.x"); ASSERT_EQUALS(mType.getConfigVersion(), 0); + ASSERT_EQUALS(mType.getAdvisoryHostFQDNs().size(), 3UL); + ASSERT_EQUALS(mType.getAdvisoryHostFQDNs()[0], "foo"); + ASSERT_EQUALS(mType.getAdvisoryHostFQDNs()[1], "bar"); + ASSERT_EQUALS(mType.getAdvisoryHostFQDNs()[2], "baz"); } TEST(Validity, BadType) { diff --git a/src/mongo/s/shard_key_pattern.cpp b/src/mongo/s/shard_key_pattern.cpp index 53be53632ad..d62a3a4644f 100644 --- a/src/mongo/s/shard_key_pattern.cpp +++ b/src/mongo/s/shard_key_pattern.cpp @@ -98,12 +98,15 @@ std::vector<FieldRef*> parseShardKeyPattern(const BSONObj& keyPattern) { return parsedPaths.release(); } -bool isShardKeyElement(const BSONElement& element, bool allowRegex) { - if (element.eoo() || element.type() == Array) +bool isValidShardKeyElement(const BSONElement& element) { + return !element.eoo() && element.type() != Array; +} + +bool isValidShardKeyElementForStorage(const BSONElement& element) { + if (!isValidShardKeyElement(element)) return false; - // TODO: Disallow regex all the time - if (!allowRegex && element.type() == RegEx) + if (element.type() == RegEx) return false; if (element.type() == Object && !element.embeddedObject().okForStorage()) @@ -127,6 +130,17 @@ Status ShardKeyPattern::checkShardKeySize(const BSONObj& shardKey) { << " bytes"}; } +Status ShardKeyPattern::checkShardKeyIsValidForMetadataStorage(const BSONObj& shardKey) { + for (const auto& elem : shardKey) { + if (!isValidShardKeyElementForStorage(elem)) { + return {ErrorCodes::BadValue, + str::stream() << "Shard key element " << elem << " is not valid for storage"}; + } + } + + return Status::OK(); +} + ShardKeyPattern::ShardKeyPattern(const BSONObj& keyPattern) : _keyPatternPaths(parseShardKeyPattern(keyPattern)), _keyPattern(_keyPatternPaths.empty() ? BSONObj() : keyPattern) {} @@ -169,7 +183,7 @@ bool ShardKeyPattern::isShardKey(const BSONObj& shardKey) const { for (const auto& patternEl : keyPatternBSON) { BSONElement keyEl = shardKey[patternEl.fieldNameStringData()]; - if (!isShardKeyElement(keyEl, true)) + if (!isValidShardKeyElement(keyEl)) return false; } @@ -194,7 +208,7 @@ BSONObj ShardKeyPattern::normalizeShardKey(const BSONObj& shardKey) const { BSONElement keyEl = shardKey[patternEl.fieldNameStringData()]; - if (!isShardKeyElement(keyEl, true)) + if (!isValidShardKeyElement(keyEl)) return BSONObj(); keyBuilder.appendAs(keyEl, patternEl.fieldName()); @@ -234,7 +248,7 @@ BSONObj ShardKeyPattern::extractShardKeyFromMatchable(const MatchableDocument& m BSONElement matchEl = extractKeyElementFromMatchable(matchable, patternEl.fieldNameStringData()); - if (!isShardKeyElement(matchEl, true)) + if (!isValidShardKeyElement(matchEl)) return BSONObj(); if (isHashedPatternEl(patternEl)) { @@ -319,7 +333,7 @@ BSONObj ShardKeyPattern::extractShardKeyFromQuery(const CanonicalQuery& query) c const FieldRef& patternPath = **it; BSONElement equalEl = findEqualityElement(equalities, patternPath); - if (!isShardKeyElement(equalEl, false)) + if (!isValidShardKeyElementForStorage(equalEl)) return BSONObj(); if (isHashedPattern()) { diff --git a/src/mongo/s/shard_key_pattern.h b/src/mongo/s/shard_key_pattern.h index 73038fcf9ed..08ec561d0b6 100644 --- a/src/mongo/s/shard_key_pattern.h +++ b/src/mongo/s/shard_key_pattern.h @@ -75,6 +75,12 @@ public: static Status checkShardKeySize(const BSONObj& shardKey); /** + * Validates whether the specified shard key is valid to be written as part of the sharding + * metadata. + */ + static Status checkShardKeyIsValidForMetadataStorage(const BSONObj& shardKey); + + /** * Constructs a shard key pattern from a BSON pattern document. If the document is not a * valid shard key pattern, !isValid() will be true and key extraction will fail. */ diff --git a/src/mongo/s/shard_key_pattern_test.cpp b/src/mongo/s/shard_key_pattern_test.cpp index e3b31a7a5d2..b2ab6834f80 100644 --- a/src/mongo/s/shard_key_pattern_test.cpp +++ b/src/mongo/s/shard_key_pattern_test.cpp @@ -128,10 +128,13 @@ TEST(ShardKeyPattern, NormalizeShardKey) { BSON("a.b" << 10 << "c" << 30)); ASSERT_BSONOBJ_EQ(normKey(pattern, BSON("c" << 30 << "a.b" << 10)), BSON("a.b" << 10 << "c" << 30)); + ASSERT_BSONOBJ_EQ(normKey(pattern, BSON("a.b" << BSON("$notAndOperator" << 10) << "c" << 30)), + BSON("a.b" << BSON("$notAndOperator" << 10) << "c" << 30)); + ASSERT_BSONOBJ_EQ(normKey(pattern, BSON("a.b" << BSON("$gt" << 10) << "c" << 30)), + BSON("a.b" << BSON("$gt" << 10) << "c" << 30)); ASSERT_BSONOBJ_EQ(normKey(pattern, BSON("b" << 10)), BSONObj()); ASSERT_BSONOBJ_EQ(normKey(pattern, BSON("a" << 10 << "c" << 30)), BSONObj()); - ASSERT_BSONOBJ_EQ(normKey(pattern, BSON("a.b" << BSON("$gt" << 10) << "c" << 30)), BSONObj()); } static BSONObj docKey(const ShardKeyPattern& pattern, const BSONObj& doc) { @@ -157,13 +160,16 @@ TEST(ShardKeyPattern, ExtractDocShardKeySingle) { << "$id" << 1); ASSERT_BSONOBJ_EQ(docKey(pattern, BSON("a" << ref)), BSON("a" << ref)); + ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:{$dollarPrefixKey:true}}")), + fromjson("{a:{$dollarPrefixKey:true}}")); + ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:{$gt:10}}")), fromjson("{a:{$gt:10}}")); + ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:{$gt:{$dollarPrefixKey:10}}}}")), + fromjson("{a:{$gt:{$dollarPrefixKey:10}}}}")); ASSERT_BSONOBJ_EQ(docKey(pattern, BSONObj()), BSONObj()); ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{b:10}")), BSONObj()); ASSERT_BSONOBJ_EQ(docKey(pattern, BSON("" << 10)), BSONObj()); ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:[1,2]}")), BSONObj()); - ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:{$invalid:true}}")), BSONObj()); - ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:{$gt:10}}")), BSONObj()); // BSONObjIterator breaks this for now // ASSERT_EQUALS(docKey(pattern, BSON("a" << 10 << "a" << 20)), BSONObj()); } @@ -183,15 +189,17 @@ TEST(ShardKeyPattern, ExtractDocShardKeyCompound) { << "a" << 10)), fromjson("{a:10, b:'20'}")); + ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:10, b:{$dollarPrefixKey:true}}")), + fromjson("{a:10, b:{$dollarPrefixKey:true}}")); + ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:10, b:{$gt:20}}")), + fromjson("{a:10, b:{$gt:20}}")); ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:10, b:[1, 2]}")), BSONObj()); - ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:10, b:{$invalid:true}}")), BSONObj()); ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{b:20}")), BSONObj()); ASSERT_BSONOBJ_EQ(docKey(pattern, BSON("" << 10 << "b" << "20")), BSONObj()); - ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:10, b:{$gt:20}}")), BSONObj()); // Ordering ASSERT_EQUALS(docKey(pattern, BSON("b" << 20 << "a" << 10)).firstElement().numberInt(), 10); @@ -284,7 +292,7 @@ TEST(ShardKeyPattern, ExtractQueryShardKeySingle) { ASSERT_BSONOBJ_EQ(queryKey(pattern, fromjson("{a:10,b:{$invalid:'20'}}")), BSONObj()); // Doc key extraction shouldn't work with query - ASSERT_BSONOBJ_EQ(docKey(pattern, fromjson("{a:{$eq:[10, 20]}, c:30}")), BSONObj()); + ASSERT_BSONOBJ_EQ(queryKey(pattern, fromjson("{a:{$eq:[10, 20]}, c:30}")), BSONObj()); // $eq/$or/$and/$all ASSERT_BSONOBJ_EQ(queryKey(pattern, fromjson("{a:{$eq:10}}")), fromjson("{a:10}")); diff --git a/src/mongo/s/sharding_uptime_reporter.cpp b/src/mongo/s/sharding_uptime_reporter.cpp index 61a438dd93e..a7eefc14e03 100644 --- a/src/mongo/s/sharding_uptime_reporter.cpp +++ b/src/mongo/s/sharding_uptime_reporter.cpp @@ -42,6 +42,7 @@ #include "mongo/util/exit.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" +#include "mongo/util/net/hostname_canonicalization.h" #include "mongo/util/net/sock.h" #include "mongo/util/version.h" @@ -50,15 +51,18 @@ namespace { const Seconds kUptimeReportInterval(10); -std::string constructInstanceIdString() { - return str::stream() << getHostNameCached() << ":" << serverGlobalParams.port; +std::string constructInstanceIdString(const std::string& hostName) { + return str::stream() << hostName << ":" << serverGlobalParams.port; } /** * Reports the uptime status of the current instance to the config.pings collection. This method * is best-effort and never throws. */ -void reportStatus(OperationContext* txn, const std::string& instanceId, const Timer& upTimeTimer) { +void reportStatus(OperationContext* txn, + const std::string& instanceId, + const std::string& hostName, + const Timer& upTimeTimer) { MongosType mType; mType.setName(instanceId); mType.setPing(jsTime()); @@ -66,6 +70,8 @@ void reportStatus(OperationContext* txn, const std::string& instanceId, const Ti // balancer is never active in mongos. Here for backwards compatibility only. mType.setWaiting(true); mType.setMongoVersion(VersionInfoInterface::instance().version().toString()); + mType.setAdvisoryHostFQDNs( + getHostFQDNs(hostName, HostnameCanonicalizationMode::kForwardAndReverse)); try { Grid::get(txn)->catalogClient(txn)->updateConfigDocument( @@ -95,13 +101,14 @@ void ShardingUptimeReporter::startPeriodicThread() { _thread = stdx::thread([this] { Client::initThread("Uptime reporter"); - const std::string instanceId(constructInstanceIdString()); + const std::string hostName(getHostNameCached()); + const std::string instanceId(constructInstanceIdString(hostName)); const Timer upTimeTimer; while (!inShutdown()) { { auto txn = cc().makeOperationContext(); - reportStatus(txn.get(), instanceId, upTimeTimer); + reportStatus(txn.get(), instanceId, hostName, upTimeTimer); auto status = Grid::get(txn.get())->getBalancerConfiguration()->refreshAndCheck(txn.get()); diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index 71f4b846470..59eb411cc3b 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -63,6 +63,10 @@ * Member variables: * nodes {Array.<Mongo>} - connection to replica set members */ + +/* Global default timeout variable */ +const kReplDefaultTimeoutMS = 10 * 60 * 1000; + var ReplSetTest = function(opts) { 'use strict'; @@ -85,7 +89,9 @@ var ReplSetTest = function(opts) { var _unbridgedPorts; var _unbridgedNodes; - this.kDefaultTimeoutMS = 10 * 60 * 1000; + var _causalConsistency; + + this.kDefaultTimeoutMS = kReplDefaultTimeoutMS; var oplogName = 'oplog.rs'; // Publicly exposed variables @@ -1508,10 +1514,12 @@ var ReplSetTest = function(opts) { var otherOplogEntry = readers[i].next(); if (!bsonBinaryEqual(oplogEntry, otherOplogEntry)) { var query = prevOplogEntry ? {ts: {$lte: prevOplogEntry.ts}} : {}; - rst.nodes.forEach(node => this.dumpOplog(node, query)); - assert(false, - msgPrefix + ", non-matching oplog entry for nodes: " + - firstReader.mongo.host + " " + readers[i].mongo.host); + rst.nodes.forEach(node => this.dumpOplog(node, query, 100)); + var log = msgPrefix + + ", non-matching oplog entries for the following nodes: \n" + + firstReader.mongo.host + ": " + tojsononeline(oplogEntry) + "\n" + + readers[i].mongo.host + ": " + tojsononeline(otherOplogEntry); + assert(false, log); } } prevOplogEntry = oplogEntry; @@ -1764,7 +1772,7 @@ var ReplSetTest = function(opts) { return; } - if (_alldbpaths) { + if ((!opts || !opts.noCleanData) && _alldbpaths) { print("ReplSetTest stopSet deleting all dbpaths"); for (var i = 0; i < _alldbpaths.length; i++) { resetDbpath(_alldbpaths[i]); @@ -1886,6 +1894,12 @@ var ReplSetTest = function(opts) { }; /** + * Declare kDefaultTimeoutMS as a static property so we don't have to initialize + * a ReplSetTest object to use it. + */ +ReplSetTest.kDefaultTimeoutMS = kReplDefaultTimeoutMS; + +/** * Set of states that the replica set can be in. Used for the wait functions. */ ReplSetTest.State = { diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js index 7a0fdae70f3..daca0de8f14 100644 --- a/src/mongo/shell/shardingtest.js +++ b/src/mongo/shell/shardingtest.js @@ -396,8 +396,11 @@ var ShardingTest = function(params) { } } - for (var i = 0; i < _alldbpaths.length; i++) { - resetDbpath(MongoRunner.dataPath + _alldbpaths[i]); + if (!opts || !opts.noCleanData) { + print("ShardingTest stop deleting all dbpaths"); + for (var i = 0; i < _alldbpaths.length; i++) { + resetDbpath(MongoRunner.dataPath + _alldbpaths[i]); + } } var timeMillis = new Date().getTime() - _startTime.getTime(); diff --git a/src/mongo/transport/session.cpp b/src/mongo/transport/session.cpp index 2415c9dbb8f..226f5d8ddd1 100644 --- a/src/mongo/transport/session.cpp +++ b/src/mongo/transport/session.cpp @@ -45,6 +45,8 @@ AtomicUInt64 sessionIdCounter(0); Session::Session() : _id(sessionIdCounter.addAndFetch(1)), _tags(kEmptyTagMask) {} +Session::Session(Id id) : _id(id), _tags(kEmptyTagMask) {} + Ticket Session::sourceMessage(Message* message, Date_t expiration) { return getTransportLayer()->sourceMessage(shared_from_this(), message, expiration); } diff --git a/src/mongo/transport/session.h b/src/mongo/transport/session.h index 3d81543a7cc..99617f0eb34 100644 --- a/src/mongo/transport/session.h +++ b/src/mongo/transport/session.h @@ -143,6 +143,16 @@ protected: */ Session(); + /** + * Construct a new session with an ID + * + * This is used by TransportLayerLegacy to make sure session ID's match the connection ID of + * their MessagingPort. + * + * TransportLayers must ensure that session Id's remain unique across all TransportLayers. + */ + explicit Session(Id id); + private: const Id _id; diff --git a/src/mongo/transport/transport_layer_legacy.cpp b/src/mongo/transport/transport_layer_legacy.cpp index d5da7a53d52..680853ffe04 100644 --- a/src/mongo/transport/transport_layer_legacy.cpp +++ b/src/mongo/transport/transport_layer_legacy.cpp @@ -84,7 +84,8 @@ std::shared_ptr<TransportLayerLegacy::LegacySession> TransportLayerLegacy::Legac TransportLayerLegacy::LegacySession::LegacySession(std::unique_ptr<AbstractMessagingPort> amp, TransportLayerLegacy* tl) - : _remote(amp->remote()), + : Session(amp->connectionId()), + _remote(amp->remote()), _local(amp->localAddr().toString(true)), _tl(tl), _tags(kEmptyTagMask), diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index fe0380c1345..7918ca945ac 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -300,6 +300,12 @@ if env['MONGO_ALLOCATOR'] == 'tcmalloc': ], ) +env.Library( + target='winutil', + source=[ + 'winutil.cpp', + ], +) env.Library( target='ntservice', diff --git a/src/mongo/util/net/SConscript b/src/mongo/util/net/SConscript index b53be63d404..59b546fad65 100644 --- a/src/mongo/util/net/SConscript +++ b/src/mongo/util/net/SConscript @@ -59,6 +59,7 @@ networkEnv.Library( '$BUILD_DIR/mongo/util/options_parser/options_parser', '$BUILD_DIR/third_party/shim_asio', 'hostandport', + '$BUILD_DIR/mongo/util/winutil', ], ) diff --git a/src/mongo/util/net/sock.cpp b/src/mongo/util/net/sock.cpp index dcc9b371892..457cd9d2bc8 100644 --- a/src/mongo/util/net/sock.cpp +++ b/src/mongo/util/net/sock.cpp @@ -49,6 +49,10 @@ #if defined(__OpenBSD__) #include <sys/uio.h> #endif +#else +#include <mstcpip.h> +#include <winsock2.h> +#include <ws2tcpip.h> #endif #include "mongo/config.h" @@ -65,6 +69,7 @@ #include "mongo/util/net/socket_poll.h" #include "mongo/util/net/ssl_manager.h" #include "mongo/util/quick_exit.h" +#include "mongo/util/winutil.h" namespace mongo { @@ -105,6 +110,7 @@ void networkWarnWithDescription(const Socket& socket, StringData call, int error const double kMaxConnectTimeoutMS = 5000; + } // namespace static bool ipv6 = false; @@ -140,23 +146,104 @@ void setSockTimeouts(int sock, double secs) { #endif } -#if defined(_WIN32) -void disableNagle(int sock) { - int x = 1; - if (setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, (char*)&x, sizeof(x))) - error() << "disableNagle failed: " << errnoWithDescription(); - if (setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, (char*)&x, sizeof(x))) - error() << "SO_KEEPALIVE failed: " << errnoWithDescription(); -} +#ifdef _WIN32 +#ifdef _UNICODE +#define X_STR_CONST(str) (L##str) #else +#define X_STR_CONST(str) (str) +#endif +const CString kKeepAliveGroup( + X_STR_CONST("SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters")); +const CString kKeepAliveTime(X_STR_CONST("KeepAliveTime")); +const CString kKeepAliveInterval(X_STR_CONST("KeepAliveInterval")); +#undef X_STR_CONST +#endif -void disableNagle(int sock) { - int x = 1; +void setSocketKeepAliveParams(int sock, + unsigned int maxKeepIdleSecs, + unsigned int maxKeepIntvlSecs) { +#ifdef _WIN32 + // Defaults per MSDN when registry key does not exist. + // Expressed in seconds here to be consistent with posix, + // though Windows uses milliseconds. + const DWORD kWindowsKeepAliveTimeSecsDefault = 2 * 60 * 60; + const DWORD kWindowsKeepAliveIntervalSecsDefault = 1; + + const auto getKey = [](const CString& key, DWORD default_value) { + auto withval = windows::getDWORDRegistryKey(kKeepAliveGroup, key); + if (withval.isOK()) { + auto val = withval.getValue(); + // Return seconds + return val ? (val.get() / 1000) : default_value; + } + error() << "can't get KeepAlive parameter: " << withval.getStatus(); + return default_value; + }; + + const auto keepIdleSecs = getKey(kKeepAliveTime, kWindowsKeepAliveTimeSecsDefault); + const auto keepIntvlSecs = getKey(kKeepAliveInterval, kWindowsKeepAliveIntervalSecsDefault); + + if ((keepIdleSecs > maxKeepIdleSecs) || (keepIntvlSecs > maxKeepIntvlSecs)) { + DWORD sent = 0; + struct tcp_keepalive keepalive; + keepalive.onoff = TRUE; + keepalive.keepalivetime = std::min<DWORD>(keepIdleSecs, maxKeepIdleSecs) * 1000; + keepalive.keepaliveinterval = std::min<DWORD>(keepIntvlSecs, maxKeepIntvlSecs) * 1000; + if (WSAIoctl(sock, + SIO_KEEPALIVE_VALS, + &keepalive, + sizeof(keepalive), + nullptr, + 0, + &sent, + nullptr, + nullptr)) { + error() << "failed setting keepalive values: " << WSAGetLastError(); + } + } +#elif defined(__APPLE__) || defined(__linux__) + const auto updateSockOpt = + [sock](int level, int optnum, unsigned int maxval, StringData optname) { + unsigned int optval = 1; + socklen_t len = sizeof(optval); + + if (getsockopt(sock, level, optnum, (char*)&optval, &len)) { + error() << "can't get " << optname << ": " << errnoWithDescription(); + } + + if (optval > maxval) { + optval = maxval; + if (setsockopt(sock, level, optnum, (char*)&optval, sizeof(optval))) { + error() << "can't set " << optname << ": " << errnoWithDescription(); + } + } + }; + +#ifdef __APPLE__ + updateSockOpt(IPPROTO_TCP, TCP_KEEPALIVE, maxKeepIdleSecs, "TCP_KEEPALIVE"); +#endif +#ifdef __linux__ #ifdef SOL_TCP - int level = SOL_TCP; + const int level = SOL_TCP; +#else + const int level = SOL_SOCKET; +#endif + updateSockOpt(level, TCP_KEEPIDLE, maxKeepIdleSecs, "TCP_KEEPIDLE"); + updateSockOpt(level, TCP_KEEPINTVL, maxKeepIntvlSecs, "TCP_KEEPINTVL"); +#endif + +#endif +} + +void disableNagle(int sock) { + int x = 1; +#ifdef _WIN32 + const int level = IPPROTO_TCP; +#elif defined(SOL_TCP) + const int level = SOL_TCP; #else - int level = SOL_SOCKET; + const int level = SOL_SOCKET; #endif if (setsockopt(sock, level, TCP_NODELAY, (char*)&x, sizeof(x))) @@ -165,34 +252,10 @@ void disableNagle(int sock) { #ifdef SO_KEEPALIVE if (setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, (char*)&x, sizeof(x))) error() << "SO_KEEPALIVE failed: " << errnoWithDescription(); - -#ifdef __linux__ - socklen_t len = sizeof(x); - if (getsockopt(sock, level, TCP_KEEPIDLE, (char*)&x, &len)) - error() << "can't get TCP_KEEPIDLE: " << errnoWithDescription(); - - if (x > 300) { - x = 300; - if (setsockopt(sock, level, TCP_KEEPIDLE, (char*)&x, sizeof(x))) { - error() << "can't set TCP_KEEPIDLE: " << errnoWithDescription(); - } - } - - len = sizeof(x); // just in case it changed - if (getsockopt(sock, level, TCP_KEEPINTVL, (char*)&x, &len)) - error() << "can't get TCP_KEEPINTVL: " << errnoWithDescription(); - - if (x > 300) { - x = 300; - if (setsockopt(sock, level, TCP_KEEPINTVL, (char*)&x, sizeof(x))) { - error() << "can't set TCP_KEEPINTVL: " << errnoWithDescription(); - } - } -#endif #endif -} -#endif + setSocketKeepAliveParams(sock); +} string getAddrInfoStrError(int code) { #if !defined(_WIN32) diff --git a/src/mongo/util/net/sock.h b/src/mongo/util/net/sock.h index 44a83b6ef5b..f6fea719279 100644 --- a/src/mongo/util/net/sock.h +++ b/src/mongo/util/net/sock.h @@ -70,6 +70,9 @@ extern const int portRecvFlags; const int SOCK_FAMILY_UNKNOWN_ERROR = 13078; +void setSocketKeepAliveParams(int sock, + unsigned int maxKeepIdleSecs = 300, + unsigned int maxKeepIntvlSecs = 300); void disableNagle(int sock); // Generate a string representation for getaddrinfo return codes diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp index 025f4932867..1b120a3be83 100644 --- a/src/mongo/util/net/ssl_manager.cpp +++ b/src/mongo/util/net/ssl_manager.cpp @@ -1235,6 +1235,7 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi if (_allowInvalidCertificates) { warning() << "SSL peer certificate validation failed: " << X509_verify_cert_error_string(result); + return {boost::none}; } else { str::stream msg; msg << "SSL peer certificate validation failed: " diff --git a/src/mongo/util/ntservice.cpp b/src/mongo/util/ntservice.cpp index 083b2fcc210..03cf4727cb2 100644 --- a/src/mongo/util/ntservice.cpp +++ b/src/mongo/util/ntservice.cpp @@ -287,7 +287,7 @@ void installServiceOrDie(const wstring& serviceName, SC_HANDLE schSCManager = ::OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS); if (schSCManager == NULL) { DWORD err = ::GetLastError(); - log() << "Error connecting to the Service Control Manager: " << GetWinErrMsg(err); + log() << "Error connecting to the Service Control Manager: " << windows::GetErrMsg(err); quickExit(EXIT_NTSERVICE_ERROR); } @@ -336,7 +336,7 @@ void installServiceOrDie(const wstring& serviceName, NULL); // user account password if (schService == NULL) { DWORD err = ::GetLastError(); - log() << "Error creating service: " << GetWinErrMsg(err); + log() << "Error creating service: " << windows::GetErrMsg(err); ::CloseServiceHandle(schSCManager); quickExit(EXIT_NTSERVICE_ERROR); } @@ -441,7 +441,7 @@ void removeServiceOrDie(const wstring& serviceName) { SC_HANDLE schSCManager = ::OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS); if (schSCManager == NULL) { DWORD err = ::GetLastError(); - log() << "Error connecting to the Service Control Manager: " << GetWinErrMsg(err); + log() << "Error connecting to the Service Control Manager: " << windows::GetErrMsg(err); quickExit(EXIT_NTSERVICE_ERROR); } diff --git a/src/mongo/util/procparser.cpp b/src/mongo/util/procparser.cpp index 36f2ae0254e..af7a81fc7a2 100644 --- a/src/mongo/util/procparser.cpp +++ b/src/mongo/util/procparser.cpp @@ -406,6 +406,101 @@ Status parseProcMemInfoFile(StringData filename, return parseProcMemInfo(keys, swString.getValue(), builder); } +// +// Here is an example of the type of string it supports (long lines elided for clarity). +// > cat /proc/net/netstat +// TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed ... +// TcpExt: 3437 5938 13368 ... +// IpExt: InNoRoutes InTruncatedPkts InMcastPkts ... +// IpExt: 999 1 4819969 ... +// +// Parser assumes file consists of alternating lines of keys and values +// key and value lines consist of space-separated tokens +// first token is a key prefix that is prepended in the output to each key +// all prefixed keys and corresponding values are copied to output as-is +// + +Status parseProcNetstat(const std::vector<StringData>& keys, + StringData data, + BSONObjBuilder* builder) { + + using string_split_iterator = boost::split_iterator<StringData::const_iterator>; + + string_split_iterator keysIt; + bool foundKeys = false; + + // Split the file by lines. + uint32_t lineNum = 0; + for (string_split_iterator + lineIt = string_split_iterator( + data.begin(), + data.end(), + boost::token_finder([](char c) { return c == '\n'; }, boost::token_compress_on)); + lineIt != string_split_iterator(); + ++lineIt, ++lineNum) { + + if (lineNum % 2 == 0) { + + // even numbered lines are keys + keysIt = string_split_iterator( + (*lineIt).begin(), + (*lineIt).end(), + boost::token_finder([](char c) { return c == ' '; }, boost::token_compress_on)); + + } else { + + // odd numbered lines are values + string_split_iterator valuesIt = string_split_iterator( + (*lineIt).begin(), + (*lineIt).end(), + boost::token_finder([](char c) { return c == ' '; }, boost::token_compress_on)); + + StringData prefix; + + // iterate over the keys and values in parallel + for (uint32_t keyNum = 0; + keysIt != string_split_iterator() && valuesIt != string_split_iterator(); + ++keysIt, ++valuesIt, ++keyNum) { + + if (keyNum == 0) { + + // first token is a prefix to be applied to remaining keys + prefix = StringData((*keysIt).begin(), (*keysIt).end()); + + // ignore line if prefix isn't in requested list + if (!keys.empty() && std::find(keys.begin(), keys.end(), prefix) == keys.end()) + break; + + } else { + + // remaining tokens are key/value pairs + StringData key((*keysIt).begin(), (*keysIt).end()); + StringData stringValue((*valuesIt).begin(), (*valuesIt).end()); + uint64_t value; + if (parseNumberFromString(stringValue, &value).isOK()) { + builder->appendNumber(prefix.toString() + key.toString(), value); + foundKeys = true; + } + } + } + } + } + + return foundKeys ? Status::OK() + : Status(ErrorCodes::NoSuchKey, "Failed to find any keys in netstats string"); +} + +Status parseProcNetstatFile(const std::vector<StringData>& keys, + StringData filename, + BSONObjBuilder* builder) { + auto swString = readFileAsString(filename); + if (!swString.isOK()) { + return swString.getStatus(); + } + return parseProcNetstat(keys, swString.getValue(), builder); +} + + // Here is an example of the type of string it supports: // // For more information, see: diff --git a/src/mongo/util/procparser.h b/src/mongo/util/procparser.h index 2f9e97e343c..bbaf543802a 100644 --- a/src/mongo/util/procparser.h +++ b/src/mongo/util/procparser.h @@ -88,6 +88,24 @@ Status parseProcMemInfoFile(StringData filename, BSONObjBuilder* builder); /** + * Read a string matching /proc/net/netstat format, and write the keys + * found in that string into builder. + * + * data - string to parse + * builder - BSON output + */ +Status parseProcNetstat(const std::vector<StringData>& keys, + StringData data, + BSONObjBuilder* builder); + +/** + * Read from file, and write the keys found in that file into builder. + */ +Status parseProcNetstatFile(const std::vector<StringData>& keys, + StringData filename, + BSONObjBuilder* builder); + +/** * Read a string matching /proc/diskstats format, and write the specified list of disks in builder. * * disks - vector of block devices to include in output. For each disk selected, 11 fields are diff --git a/src/mongo/util/procparser_test.cpp b/src/mongo/util/procparser_test.cpp index 81bf17ac8df..ff620b8ac61 100644 --- a/src/mongo/util/procparser_test.cpp +++ b/src/mongo/util/procparser_test.cpp @@ -85,6 +85,11 @@ StringMap toNestedStringMap(BSONObj& obj) { ASSERT_OK(procparser::parseProcMemInfo(_keys, _x, &builder)); \ auto obj = builder.obj(); \ auto stringMap = toStringMap(obj); +#define ASSERT_PARSE_NETSTAT(_keys, _x) \ + BSONObjBuilder builder; \ + ASSERT_OK(procparser::parseProcNetstat(_keys, _x, &builder)); \ + auto obj = builder.obj(); \ + auto stringMap = toStringMap(obj); #define ASSERT_PARSE_DISKSTATS(_disks, _x) \ BSONObjBuilder builder; \ ASSERT_OK(procparser::parseProcDiskStats(_disks, _x, &builder)); \ @@ -368,6 +373,133 @@ TEST(FTDCProcMemInfo, TestLocalNonExistentMemInfo) { ASSERT_NOT_OK(procparser::parseProcMemInfoFile("/proc/does_not_exist", keys, &builder)); } +TEST(FTDCProcNetstat, TestNetstat) { + + // test keys + std::vector<StringData> keys{"pfx1", "pfx2", "pfx3"}; + + // Normal case + { + ASSERT_PARSE_NETSTAT(keys, + "pfx1 key1 key2 key3\n" + "pfx1 1 2 3\n" + "pfxX key1 key2\n" + "pfxX key1 key2\n" + "pfx2 key4 key5\n" + "pfx2 4 5\n"); + ASSERT_KEY_AND_VALUE("pfx1key1", 1UL); + ASSERT_KEY_AND_VALUE("pfx1key2", 2UL); + ASSERT_NO_KEY("pfxXkey1"); + ASSERT_NO_KEY("pfxXkey2"); + ASSERT_KEY_AND_VALUE("pfx1key3", 3UL) + ASSERT_KEY_AND_VALUE("pfx2key4", 4UL); + ASSERT_KEY_AND_VALUE("pfx2key5", 5UL); + } + + // Mismatched keys and values + { + ASSERT_PARSE_NETSTAT(keys, + "pfx1 key1 key2 key3\n" + "pfx1 1 2 3 4\n" + "pfx2 key4 key5\n" + "pfx2 4\n" + "pfx3 key6 key7\n"); + ASSERT_KEY_AND_VALUE("pfx1key1", 1UL); + ASSERT_KEY_AND_VALUE("pfx1key2", 2UL); + ASSERT_KEY_AND_VALUE("pfx1key3", 3UL); + ASSERT_NO_KEY("pfx1key4"); + ASSERT_KEY_AND_VALUE("pfx2key4", 4UL); + ASSERT_NO_KEY("pfx2key5"); + ASSERT_NO_KEY("pfx3key6"); + ASSERT_NO_KEY("pfx3key7"); + } + + // Non-numeric value + { + ASSERT_PARSE_NETSTAT(keys, + "pfx1 key1 key2 key3\n" + "pfx1 1 foo 3\n"); + ASSERT_KEY_AND_VALUE("pfx1key1", 1UL); + ASSERT_NO_KEY("pfx1key2"); + ASSERT_KEY_AND_VALUE("pfx1key3", 3UL) + } + + // No newline + { + ASSERT_PARSE_NETSTAT(keys, + "pfx1 key1 key2 key3\n" + "pfx1 1 2 3\n" + "pfx2 key4 key5\n" + "pfx2 4 5"); + ASSERT_KEY_AND_VALUE("pfx1key1", 1UL); + ASSERT_KEY_AND_VALUE("pfx1key2", 2UL); + ASSERT_KEY_AND_VALUE("pfx1key3", 3UL) + ASSERT_KEY_AND_VALUE("pfx2key4", 4UL); + ASSERT_KEY_AND_VALUE("pfx2key5", 5UL); + } + + // Single line only + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcNetstat(keys, "pfx1 key1 key2 key3\n", &builder)); + } + + // Empty string + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcNetstat(keys, "", &builder)); + } +} + +// Test we can parse the /proc/net/netstat on this machine and assert we have some expected fields +// Some keys can vary between distros, so we test only for the existence of a few basic ones +TEST(FTDCProcNetstat, TestLocalNetstat) { + + BSONObjBuilder builder; + + std::vector<StringData> keys{"TcpExt:"_sd, "IpExt:"_sd}; + + ASSERT_OK(procparser::parseProcNetstatFile(keys, "/proc/net/netstat", &builder)); + + BSONObj obj = builder.obj(); + auto stringMap = toStringMap(obj); + log() << "OBJ:" << obj; + ASSERT_KEY("TcpExt:TCPTimeouts"); + ASSERT_KEY("TcpExt:TCPPureAcks"); + ASSERT_KEY("TcpExt:TCPAbortOnTimeout"); + ASSERT_KEY("TcpExt:EmbryonicRsts"); + ASSERT_KEY("TcpExt:ListenDrops"); + ASSERT_KEY("TcpExt:ListenOverflows"); + ASSERT_KEY("TcpExt:DelayedACKs"); + ASSERT_KEY("IpExt:OutOctets"); + ASSERT_KEY("IpExt:InOctets"); +} + +// Test we can parse the /proc/net/snmp on this machine and assert we have some expected fields +// Some keys can vary between distros, so we test only for the existence of a few basic ones +TEST(FTDCProcNetstat, TestLocalNetSnmp) { + + BSONObjBuilder builder; + + std::vector<StringData> keys{"Tcp:"_sd, "Ip:"_sd}; + + ASSERT_OK(procparser::parseProcNetstatFile(keys, "/proc/net/snmp", &builder)); + + BSONObj obj = builder.obj(); + auto stringMap = toStringMap(obj); + log() << "OBJ:" << obj; + ASSERT_KEY("Ip:InReceives"); + ASSERT_KEY("Ip:OutRequests"); + ASSERT_KEY("Tcp:InSegs"); + ASSERT_KEY("Tcp:OutSegs"); +} + +TEST(FTDCProcNetstat, TestLocalNonExistentNetstat) { + std::vector<StringData> keys{}; + BSONObjBuilder builder; + + ASSERT_NOT_OK(procparser::parseProcNetstatFile(keys, "/proc/does_not_exist", &builder)); +} TEST(FTDCProcDiskStats, TestDiskStats) { diff --git a/src/mongo/util/winutil.cpp b/src/mongo/util/winutil.cpp new file mode 100644 index 00000000000..9d82a029b1d --- /dev/null +++ b/src/mongo/util/winutil.cpp @@ -0,0 +1,64 @@ +// winutil.cpp + +/* Copyright 2017 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#if defined(_WIN32) + +#include "mongo/platform/basic.h" + +#include "mongo/util/winutil.h" + +#include "mongo/base/error_codes.h" +#include "mongo/base/status.h" + +namespace mongo { + +StatusWith<boost::optional<DWORD>> windows::getDWORDRegistryKey(const CString& group, + const CString& key) { + CRegKey regkey; + if (ERROR_SUCCESS != regkey.Open(HKEY_LOCAL_MACHINE, group, KEY_READ)) { + return Status(ErrorCodes::InternalError, "Unable to access windows registry"); + } + + DWORD val; + const auto res = regkey.QueryDWORDValue(key, val); + if (ERROR_INVALID_DATA == res) { + return Status(ErrorCodes::TypeMismatch, + "Invalid data type in windows registry, expected DWORD"); + } + + if (ERROR_SUCCESS != res) { + return boost::none; + } + + return val; +} + +} // namespace mongo + +#endif diff --git a/src/mongo/util/winutil.h b/src/mongo/util/winutil.h index a49fcc2f406..a9138e77113 100644 --- a/src/mongo/util/winutil.h +++ b/src/mongo/util/winutil.h @@ -33,11 +33,19 @@ #if defined(_WIN32) #include "text.h" +#include <atlbase.h> +#include <atlstr.h> +#include <boost/optional.hpp> +#include <sstream> +#include <string> #include <windows.h> +#include <mongo/base/status_with.h> + namespace mongo { +namespace windows { -inline std::string GetWinErrMsg(DWORD err) { +inline std::string GetErrMsg(DWORD err) { LPTSTR errMsg; ::FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, @@ -56,6 +64,23 @@ inline std::string GetWinErrMsg(DWORD err) { return output.str(); } -} + +/** + * Retrieve a DWORD value from the Local Machine Windows Registry for element: + * group\key + * e.g. HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\KeepAliveTime + * + * On success, returns: + * boost::none if the key does not exist. + * The value read from the registry. + * + * On failure, returns: + * ErrorCodes::InternalError - Unable to access the registry group. + * ErrorCodes::TypeMismatch - Key exists, but is of the wrong type. + */ +StatusWith<boost::optional<DWORD>> getDWORDRegistryKey(const CString& group, const CString& key); + +} // namespace windows +} // namespace mongo #endif diff --git a/src/third_party/mozjs-45/extract/js/src/gc/GCRuntime.h b/src/third_party/mozjs-45/extract/js/src/gc/GCRuntime.h index e0ad996150d..be8348a6f47 100644 --- a/src/third_party/mozjs-45/extract/js/src/gc/GCRuntime.h +++ b/src/third_party/mozjs-45/extract/js/src/gc/GCRuntime.h @@ -937,7 +937,7 @@ class GCRuntime void beginSweepPhase(bool lastGC); void findZoneGroups(); - bool findZoneEdgesForWeakMaps(); + bool findInterZoneEdges(); void getNextZoneGroup(); void endMarkingZoneGroup(); void beginSweepingZoneGroup(); diff --git a/src/third_party/mozjs-45/extract/js/src/gc/Zone.cpp b/src/third_party/mozjs-45/extract/js/src/gc/Zone.cpp index b001ecff9e6..a13d855335c 100644 --- a/src/third_party/mozjs-45/extract/js/src/gc/Zone.cpp +++ b/src/third_party/mozjs-45/extract/js/src/gc/Zone.cpp @@ -30,6 +30,7 @@ JS::Zone::Zone(JSRuntime* rt) compartments(), gcGrayRoots(), gcWeakKeys(SystemAllocPolicy(), rt->randomHashCodeScrambler()), + hasDeadProxies(false), gcMallocBytes(0), gcMallocGCTriggered(false), usage(&rt->gc.usage), diff --git a/src/third_party/mozjs-45/extract/js/src/gc/Zone.h b/src/third_party/mozjs-45/extract/js/src/gc/Zone.h index 5519823ada2..bc31394e5bc 100644 --- a/src/third_party/mozjs-45/extract/js/src/gc/Zone.h +++ b/src/third_party/mozjs-45/extract/js/src/gc/Zone.h @@ -322,6 +322,10 @@ struct Zone : public JS::shadow::Zone, // can't be determined by examining this zone by itself. ZoneSet gcZoneGroupEdges; + // Zones with dead proxies require an extra scan through the wrapper map, + // so track whether any dead proxies are known to exist. + bool hasDeadProxies; + // Malloc counter to measure memory pressure for GC scheduling. It runs from // gcMaxMallocBytes down to zero. This counter should be used only when it's // not possible to know the size of a free. diff --git a/src/third_party/mozjs-45/extract/js/src/jscompartment.h b/src/third_party/mozjs-45/extract/js/src/jscompartment.h index de5592fd65a..bcf52ce0b06 100644 --- a/src/third_party/mozjs-45/extract/js/src/jscompartment.h +++ b/src/third_party/mozjs-45/extract/js/src/jscompartment.h @@ -602,6 +602,8 @@ struct JSCompartment void findOutgoingEdges(js::gc::ComponentFinder<JS::Zone>& finder); + MOZ_MUST_USE bool findDeadProxyZoneEdges(bool* foundAny); + js::DtoaCache dtoaCache; // Random number generator for Math.random(). diff --git a/src/third_party/mozjs-45/extract/js/src/jsgc.cpp b/src/third_party/mozjs-45/extract/js/src/jsgc.cpp index 776ebef6563..4ff0ca95182 100644 --- a/src/third_party/mozjs-45/extract/js/src/jsgc.cpp +++ b/src/third_party/mozjs-45/extract/js/src/jsgc.cpp @@ -4528,8 +4528,8 @@ DropStringWrappers(JSRuntime* rt) * * If compartment A has an edge to an unmarked object in compartment B, then we * must not sweep A in a later slice than we sweep B. That's because a write - * barrier in A that could lead to the unmarked object in B becoming - * marked. However, if we had already swept that object, we would be in trouble. + * barrier in A could lead to the unmarked object in B becoming marked. + * However, if we had already swept that object, we would be in trouble. * * If we consider these dependencies as a graph, then all the compartments in * any strongly-connected component of this graph must be swept in the same @@ -4573,6 +4573,30 @@ JSCompartment::findOutgoingEdges(ComponentFinder<JS::Zone>& finder) } } +bool +JSCompartment::findDeadProxyZoneEdges(bool* foundAny) +{ + // As an optimization, return whether any dead proxy objects are found in + // this compartment so that if a zone has none, its cross compartment + // wrappers do not need to be scanned. + *foundAny = false; + for (js::WrapperMap::Enum e(crossCompartmentWrappers); !e.empty(); e.popFront()) { + Value value = e.front().value().get(); + if (value.isObject()) { + if (IsDeadProxyObject(&value.toObject())) { + *foundAny = true; + Zone* wrappedZone = static_cast<JSObject*>(e.front().key().wrapped)->zone(); + if (!wrappedZone->isGCMarking()) + continue; + if (!wrappedZone->gcZoneGroupEdges.put(zone())) + return false; + } + } + } + + return true; +} + void Zone::findOutgoingEdges(ComponentFinder<JS::Zone>& finder) { @@ -4596,7 +4620,7 @@ Zone::findOutgoingEdges(ComponentFinder<JS::Zone>& finder) } bool -GCRuntime::findZoneEdgesForWeakMaps() +GCRuntime::findInterZoneEdges() { /* * Weakmaps which have keys with delegates in a different zone introduce the @@ -4613,6 +4637,20 @@ GCRuntime::findZoneEdgesForWeakMaps() return false; } + for (GCZonesIter zone(rt); !zone.done(); zone.next()) { + if (zone->hasDeadProxies) { + bool foundInZone = false; + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) { + bool foundInCompartment = false; + if (!comp->findDeadProxyZoneEdges(&foundInCompartment)) + return false; + foundInZone = foundInZone || foundInCompartment; + } + if (!foundInZone) + zone->hasDeadProxies = false; + } + } + return true; } @@ -4625,7 +4663,7 @@ GCRuntime::findZoneGroups() #endif ComponentFinder<Zone> finder(rt->mainThread.nativeStackLimit[StackForSystemCode]); - if (!isIncremental || !findZoneEdgesForWeakMaps()) + if (!isIncremental || !findInterZoneEdges()) finder.useOneComponent(); for (GCZonesIter zone(rt); !zone.done(); zone.next()) { @@ -4886,6 +4924,8 @@ js::NotifyGCNukeWrapper(JSObject* obj) * remember to mark it. */ RemoveFromGrayList(obj); + + obj->zone()->hasDeadProxies = true; } enum { @@ -7474,7 +7514,11 @@ JS::IsIncrementalGCInProgress(JSRuntime* rt) JS_PUBLIC_API(bool) JS::IsIncrementalBarrierNeeded(JSRuntime* rt) { - return rt->gc.state() == gc::MARK && !rt->isHeapBusy(); + if (rt->isHeapBusy()) + return false; + + auto state = rt->gc.state(); + return state != gc::NO_INCREMENTAL && state <= gc::SWEEP; } JS_PUBLIC_API(bool) diff --git a/src/third_party/mozjs-45/get-sources.sh b/src/third_party/mozjs-45/get-sources.sh index 089a52d2793..f98cd314503 100644 --- a/src/third_party/mozjs-45/get-sources.sh +++ b/src/third_party/mozjs-45/get-sources.sh @@ -2,7 +2,7 @@ # how we got the last firefox sources -VERSION=45.8.0esr +VERSION=45.9.0esr TARBALL=firefox-$VERSION.source.tar.xz if [ ! -f $TARBALL ]; then wget "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/$VERSION/source/$TARBALL" diff --git a/src/third_party/mozjs-45/platform/aarch64/linux/build/js-confdefs.h b/src/third_party/mozjs-45/platform/aarch64/linux/build/js-confdefs.h index 2b1a51d09f7..4fa58adf915 100644 --- a/src/third_party/mozjs-45/platform/aarch64/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-45/platform/aarch64/linux/build/js-confdefs.h @@ -74,10 +74,10 @@ #define MALLOC_H <malloc.h> #define MALLOC_USABLE_SIZE_CONST_PTR #define MOZILLA_UAVERSION "45.0" -#define MOZILLA_VERSION "45.8.0" -#define MOZILLA_VERSION_U 45.8.0 +#define MOZILLA_VERSION "45.9.0" +#define MOZILLA_VERSION_U 45.9.0 #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_GLUE_IN_PROGRAM 1 diff --git a/src/third_party/mozjs-45/platform/aarch64/linux/include/js-config.h b/src/third_party/mozjs-45/platform/aarch64/linux/include/js-config.h index e4dcddbe680..14bbe72b530 100644 --- a/src/third_party/mozjs-45/platform/aarch64/linux/include/js-config.h +++ b/src/third_party/mozjs-45/platform/aarch64/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-45/platform/ppc64le/linux/build/js-confdefs.h b/src/third_party/mozjs-45/platform/ppc64le/linux/build/js-confdefs.h index 2b1a51d09f7..4fa58adf915 100644 --- a/src/third_party/mozjs-45/platform/ppc64le/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-45/platform/ppc64le/linux/build/js-confdefs.h @@ -74,10 +74,10 @@ #define MALLOC_H <malloc.h> #define MALLOC_USABLE_SIZE_CONST_PTR #define MOZILLA_UAVERSION "45.0" -#define MOZILLA_VERSION "45.8.0" -#define MOZILLA_VERSION_U 45.8.0 +#define MOZILLA_VERSION "45.9.0" +#define MOZILLA_VERSION_U 45.9.0 #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_GLUE_IN_PROGRAM 1 diff --git a/src/third_party/mozjs-45/platform/ppc64le/linux/include/js-config.h b/src/third_party/mozjs-45/platform/ppc64le/linux/include/js-config.h index e4dcddbe680..14bbe72b530 100644 --- a/src/third_party/mozjs-45/platform/ppc64le/linux/include/js-config.h +++ b/src/third_party/mozjs-45/platform/ppc64le/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-45/platform/s390x/linux/build/js-confdefs.h b/src/third_party/mozjs-45/platform/s390x/linux/build/js-confdefs.h index c94367cff3f..305ca47991a 100644 --- a/src/third_party/mozjs-45/platform/s390x/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-45/platform/s390x/linux/build/js-confdefs.h @@ -75,10 +75,10 @@ #define MALLOC_H <malloc.h> #define MALLOC_USABLE_SIZE_CONST_PTR #define MOZILLA_UAVERSION "45.0" -#define MOZILLA_VERSION "45.8.0" -#define MOZILLA_VERSION_U 45.8.0 +#define MOZILLA_VERSION "45.9.0" +#define MOZILLA_VERSION_U 45.9.0 #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_GLUE_IN_PROGRAM 1 diff --git a/src/third_party/mozjs-45/platform/s390x/linux/include/js-config.h b/src/third_party/mozjs-45/platform/s390x/linux/include/js-config.h index e4dcddbe680..14bbe72b530 100644 --- a/src/third_party/mozjs-45/platform/s390x/linux/include/js-config.h +++ b/src/third_party/mozjs-45/platform/s390x/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-45/platform/x86_64/freebsd/build/js-confdefs.h b/src/third_party/mozjs-45/platform/x86_64/freebsd/build/js-confdefs.h index c36f3704f05..efac368476f 100644 --- a/src/third_party/mozjs-45/platform/x86_64/freebsd/build/js-confdefs.h +++ b/src/third_party/mozjs-45/platform/x86_64/freebsd/build/js-confdefs.h @@ -69,10 +69,10 @@ #define MALLOC_H <malloc_np.h> #define MALLOC_USABLE_SIZE_CONST_PTR const #define MOZILLA_UAVERSION "45.0" -#define MOZILLA_VERSION "45.8.0" -#define MOZILLA_VERSION_U 45.8.0 +#define MOZILLA_VERSION "45.9.0" +#define MOZILLA_VERSION_U 45.9.0 #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_GLUE_IN_PROGRAM 1 diff --git a/src/third_party/mozjs-45/platform/x86_64/freebsd/include/js-config.h b/src/third_party/mozjs-45/platform/x86_64/freebsd/include/js-config.h index dcc2617a294..44ff0abdcb3 100644 --- a/src/third_party/mozjs-45/platform/x86_64/freebsd/include/js-config.h +++ b/src/third_party/mozjs-45/platform/x86_64/freebsd/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-45/platform/x86_64/linux/build/js-confdefs.h b/src/third_party/mozjs-45/platform/x86_64/linux/build/js-confdefs.h index 9b3d5f6ca7b..88a222e62db 100644 --- a/src/third_party/mozjs-45/platform/x86_64/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-45/platform/x86_64/linux/build/js-confdefs.h @@ -79,10 +79,10 @@ #define MALLOC_H <malloc.h> #define MALLOC_USABLE_SIZE_CONST_PTR #define MOZILLA_UAVERSION "45.0" -#define MOZILLA_VERSION "45.8.0" -#define MOZILLA_VERSION_U 45.8.0 +#define MOZILLA_VERSION "45.9.0" +#define MOZILLA_VERSION_U 45.9.0 #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_GLUE_IN_PROGRAM 1 diff --git a/src/third_party/mozjs-45/platform/x86_64/linux/include/js-config.h b/src/third_party/mozjs-45/platform/x86_64/linux/include/js-config.h index e4dcddbe680..14bbe72b530 100644 --- a/src/third_party/mozjs-45/platform/x86_64/linux/include/js-config.h +++ b/src/third_party/mozjs-45/platform/x86_64/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-45/platform/x86_64/openbsd/build/js-confdefs.h b/src/third_party/mozjs-45/platform/x86_64/openbsd/build/js-confdefs.h index d4ccca388b9..f5c76afa048 100644 --- a/src/third_party/mozjs-45/platform/x86_64/openbsd/build/js-confdefs.h +++ b/src/third_party/mozjs-45/platform/x86_64/openbsd/build/js-confdefs.h @@ -67,10 +67,10 @@ #define MALLOC_H <malloc.h> #define MALLOC_USABLE_SIZE_CONST_PTR const #define MOZILLA_UAVERSION "45.0" -#define MOZILLA_VERSION "45.8.0" -#define MOZILLA_VERSION_U 45.8.0 +#define MOZILLA_VERSION "45.9.0" +#define MOZILLA_VERSION_U 45.9.0 #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so.1.0" #define MOZ_GLUE_IN_PROGRAM 1 diff --git a/src/third_party/mozjs-45/platform/x86_64/openbsd/include/js-config.h b/src/third_party/mozjs-45/platform/x86_64/openbsd/include/js-config.h index 071a002314c..22839f3e910 100644 --- a/src/third_party/mozjs-45/platform/x86_64/openbsd/include/js-config.h +++ b/src/third_party/mozjs-45/platform/x86_64/openbsd/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-45/platform/x86_64/osx/build/js-confdefs.h b/src/third_party/mozjs-45/platform/x86_64/osx/build/js-confdefs.h index 48623cf2b8b..6bab49d5c2c 100644 --- a/src/third_party/mozjs-45/platform/x86_64/osx/build/js-confdefs.h +++ b/src/third_party/mozjs-45/platform/x86_64/osx/build/js-confdefs.h @@ -62,10 +62,10 @@ #define MALLOC_H <malloc/malloc.h> #define MALLOC_USABLE_SIZE_CONST_PTR const #define MOZILLA_UAVERSION "45.0" -#define MOZILLA_VERSION "45.8.0" -#define MOZILLA_VERSION_U 45.8.0 +#define MOZILLA_VERSION "45.9.0" +#define MOZILLA_VERSION_U 45.9.0 #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".dylib" #define MOZ_MEMORY 1 diff --git a/src/third_party/mozjs-45/platform/x86_64/osx/include/js-config.h b/src/third_party/mozjs-45/platform/x86_64/osx/include/js-config.h index dcc2617a294..44ff0abdcb3 100644 --- a/src/third_party/mozjs-45/platform/x86_64/osx/include/js-config.h +++ b/src/third_party/mozjs-45/platform/x86_64/osx/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-45/platform/x86_64/solaris/build/js-confdefs.h b/src/third_party/mozjs-45/platform/x86_64/solaris/build/js-confdefs.h index 24af9ec84b2..25ed3de630b 100644 --- a/src/third_party/mozjs-45/platform/x86_64/solaris/build/js-confdefs.h +++ b/src/third_party/mozjs-45/platform/x86_64/solaris/build/js-confdefs.h @@ -69,10 +69,10 @@ #define MALLOC_H <malloc.h> #define MALLOC_USABLE_SIZE_CONST_PTR const #define MOZILLA_UAVERSION "45.0" -#define MOZILLA_VERSION "45.8.0" -#define MOZILLA_VERSION_U 45.8.0 +#define MOZILLA_VERSION "45.9.0" +#define MOZILLA_VERSION_U 45.9.0 #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_GLUE_IN_PROGRAM 1 diff --git a/src/third_party/mozjs-45/platform/x86_64/solaris/include/js-config.h b/src/third_party/mozjs-45/platform/x86_64/solaris/include/js-config.h index 8dd55eebb3f..71189026cae 100644 --- a/src/third_party/mozjs-45/platform/x86_64/solaris/include/js-config.h +++ b/src/third_party/mozjs-45/platform/x86_64/solaris/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-45/platform/x86_64/windows/build/js-confdefs.h b/src/third_party/mozjs-45/platform/x86_64/windows/build/js-confdefs.h index 387840310ed..9de9bc97897 100644 --- a/src/third_party/mozjs-45/platform/x86_64/windows/build/js-confdefs.h +++ b/src/third_party/mozjs-45/platform/x86_64/windows/build/js-confdefs.h @@ -30,10 +30,10 @@ #define MALLOC_H <malloc.h> #define MALLOC_USABLE_SIZE_CONST_PTR const #define MOZILLA_UAVERSION "45.0" -#define MOZILLA_VERSION "45.8.0" -#define MOZILLA_VERSION_U 45.8.0 +#define MOZILLA_VERSION "45.9.0" +#define MOZILLA_VERSION_U 45.9.0 #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".dll" #define MOZ_MEMORY 1 diff --git a/src/third_party/mozjs-45/platform/x86_64/windows/include/js-config.h b/src/third_party/mozjs-45/platform/x86_64/windows/include/js-config.h index 1ea25b51001..a885dd4efba 100644 --- a/src/third_party/mozjs-45/platform/x86_64/windows/include/js-config.h +++ b/src/third_party/mozjs-45/platform/x86_64/windows/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 45 -#define MOZJS_MINOR_VERSION 8 +#define MOZJS_MINOR_VERSION 9 #endif /* js_config_h */ diff --git a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 index 659867fa69e..b671c45e26e 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 @@ -45,6 +45,26 @@ AC_DEFUN([AM_GCC_WARNINGS], [ [*6.[0-9].[0-9]*]) # gcc6.X w="$w -Wduplicated-cond" w="$w -Wmisleading-indentation";; + [*7.[0-9].[0-9]*]) # gcc7.X + w="$w -Walloca" + w="$w -Walloc-zero" + w="$w -Wduplicated-branches" + w="$w -Wduplicated-cond" + w="$w -Wformat-overflow=2" + w="$w -Wformat-signedness" + w="$w -Wformat-truncation=2" + w="$w -Wno-error=format-truncation" + w="$w -Wno-error=format" + w="$w -Wno-error=implicit-fallthrough" + w="$w -Wno-error=maybe-uninitialized" + w="$w -Wno-error=unsafe-loop-optimizations" + w="$w -Wjump-misses-init" + w="$w -Wlogical-op" + w="$w -Wredundant-decls" + w="$w -Wrestrict" + w="$w -Wunused-const-variable=2" + w="$w -Wunused-macros" + w="$w -Wvariadic-macros";; esac wt_cv_strict_warnings="$w" @@ -67,6 +87,9 @@ AC_DEFUN([AM_CLANG_WARNINGS], [ # w="$w -Wno-error=cast-qual" w="$w -Wno-cast-qual" + # Turn off clang unreachable-code error + w="$w -Wno-error=unreachable-code" + # On Centos 7.3.1611, system header files aren't compatible with # -Wdisabled-macro-expansion. w="$w -Wno-disabled-macro-expansion" diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index fad8c2f021a..0bfebbf8458 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -143,6 +143,7 @@ Fprintf FreeBSD FreeBSD's FreeLibrary +Fsync Fuerst GCC GIDs @@ -207,6 +208,7 @@ LSM LSN LSNs LTE +LWN LZ LZO LeafGreen @@ -285,6 +287,7 @@ Pandis Phong PlatformSDK Posix +PostgreSQL PowerPC Pre Preload diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 04b716ec1c7..d7df48cee9a 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "e99ab771b2aa837199b72c2028569e7f5d3a7815", + "commit": "65d96ccb972b239c8af5aa24a03d215eb143b0e4", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.4" diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 92be2125b88..355bc09871b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -440,17 +440,49 @@ __split_ref_move(WT_SESSION_IMPL *session, WT_PAGE *from_home, } /* + * __split_ref_final -- + * Finalize the WT_REF move. + */ +static void +__split_ref_final(WT_SESSION_IMPL *session, WT_PAGE ***lockedp) +{ + WT_PAGE **locked; + size_t i; + + /* The parent page's page index has been updated. */ + WT_WRITE_BARRIER(); + + if ((locked = *lockedp) == NULL) + return; + *lockedp = NULL; + + /* + * The moved child pages are locked to prevent them from splitting + * before the parent move completes, unlock them as the final step. + */ + for (i = 0; locked[i] != NULL; ++i) + WT_PAGE_UNLOCK(session, locked[i]); + __wt_free(session, locked); +} + +/* * __split_ref_prepare -- * Prepare a set of WT_REFs for a move. */ -static void +static int __split_ref_prepare(WT_SESSION_IMPL *session, - WT_PAGE_INDEX *pindex, uint64_t split_gen, bool skip_first) + WT_PAGE_INDEX *pindex, WT_PAGE ***lockedp, bool skip_first) { - WT_PAGE *child; + WT_DECL_RET; + WT_PAGE *child, **locked; WT_REF *child_ref, *ref; + size_t alloc, cnt; uint32_t i, j; + *lockedp = NULL; + + locked = NULL; + /* The newly created subtree is complete. */ WT_WRITE_BARRIER(); @@ -462,51 +494,44 @@ __split_ref_prepare(WT_SESSION_IMPL *session, * ascend into the created children, but eventually fail as that parent * page won't yet know about the created children pages. That's OK, we * spin there until the parent's page index is updated. - */ + * + * Lock the newly created page to ensure none of its children can split. + * First, to ensure all of the child pages are updated before any pages + * can split. Second, to ensure the original split completes before any + * of the children can split. The latter involves split generations: + * the original split page has references to these children. If they + * split immediately, they could free WT_REF structures based on split + * generations earlier than the split generation we'll eventually choose + * to protect the original split page's previous page index. + */ + alloc = cnt = 0; for (i = skip_first ? 1 : 0; i < pindex->entries; ++i) { ref = pindex->index[i]; child = ref->page; - /* - * Block eviction in newly created pages. - * - * Once the split is live, newly created internal pages might be - * evicted and their WT_REF structures freed. If that happened - * before all threads exit the index of the page that previously - * "owned" the WT_REF, a thread might see a freed WT_REF. To - * ensure that doesn't happen, the newly created page contains - * the current split generation and can't be evicted until - * all readers have left the old generation. - * - * Historic, we also blocked splits in newly created pages - * because we didn't update the WT_REF.home field until after - * the split was live, so the WT_REF.home fields being updated - * could split again before the update, there's a race between - * splits as to which would update them first. The current code - * updates the WT_REF.home fields before going live (in this - * function), this isn't an issue. - */ - child->pg_intl_split_gen = split_gen; + WT_PAGE_LOCK(session, child); - /* - * We use a page flag to prevent the child from splitting from - * underneath us, but the split-generation error checks don't - * know about that flag; use the standard macros to ensure that - * reading the child's page index structure is safe. - */ + /* Track the locked pages for cleanup. */ + WT_ERR(__wt_realloc_def(session, &alloc, cnt + 2, &locked)); + locked[cnt++] = child; + + /* Switch the WT_REF's to their new page. */ j = 0; - WT_ENTER_PAGE_INDEX(session); WT_INTL_FOREACH_BEGIN(session, child, child_ref) { child_ref->home = child; child_ref->pindex_hint = j++; } WT_INTL_FOREACH_END; - WT_LEAVE_PAGE_INDEX(session); #ifdef HAVE_DIAGNOSTIC WT_WITH_PAGE_INDEX(session, __split_verify_intl_key_order(session, child)); #endif } + *lockedp = locked; + return (0); + +err: __split_ref_final(session, &locked); + return (ret); } /* @@ -518,10 +543,9 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) { WT_BTREE *btree; WT_DECL_RET; - WT_PAGE *child; + WT_PAGE *child, **locked; WT_PAGE_INDEX *alloc_index, *child_pindex, *pindex; - WT_REF **alloc_refp; - WT_REF **child_refp, *ref, **root_refp; + WT_REF **alloc_refp, **child_refp, *ref, **root_refp; WT_SPLIT_ERROR_PHASE complete; size_t child_incr, root_decr, root_incr, size; uint64_t split_gen; @@ -536,11 +560,13 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) btree = S2BT(session); alloc_index = NULL; + locked = NULL; root_decr = root_incr = 0; complete = WT_ERR_RETURN; - /* The root page will be marked dirty, make sure that will succeed. */ + /* Mark the root page dirty. */ WT_RET(__wt_page_modify_init(session, root)); + __wt_page_modify_set(session, root); /* * Our caller is holding the root page locked to single-thread splits, @@ -583,6 +609,17 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) WT_ERR(__wt_calloc_one(session, alloc_refp)); root_incr += children * sizeof(WT_REF); + /* + * Once the split is live, newly created internal pages might be evicted + * and their WT_REF structures freed. If that happens before all threads + * exit the index of the page that previously "owned" the WT_REF, a + * thread might see a freed WT_REF. To ensure that doesn't happen, the + * created pages are set to the current split generation and so can't be + * evicted until all readers have left the old generation. + */ + split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + WT_ASSERT(session, root->pg_intl_split_gen < split_gen); + /* Allocate child pages, and connect them into the new page index. */ for (root_refp = pindex->index, alloc_refp = alloc_index->index, i = 0; i < children; ++i) { @@ -609,6 +646,7 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) /* Initialize the child page. */ child->pg_intl_parent_ref = ref; + child->pg_intl_split_gen = split_gen; /* Mark it dirty. */ WT_ERR(__wt_page_modify_init(session, child)); @@ -640,12 +678,9 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) /* Start making real changes to the tree, errors are fatal. */ complete = WT_ERR_PANIC; - /* - * Prepare the WT_REFs for the move: this requires a stable split - * generation to block splits in newly created pages, so get one. - */ + /* Prepare the WT_REFs for the move. */ WT_ENTER_PAGE_INDEX(session); - __split_ref_prepare(session, alloc_index, session->split_gen, false); + WT_ERR(__split_ref_prepare(session, alloc_index, &locked, false)); /* * Confirm the root page's index hasn't moved, then update it, which @@ -665,6 +700,9 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); root->pg_intl_split_gen = split_gen; + /* Finalize the WT_REF move. */ + __split_ref_final(session, &locked); + #ifdef HAVE_DIAGNOSTIC WT_WITH_PAGE_INDEX(session, ret = __split_verify_root(session, root)); @@ -688,12 +726,13 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) WT_TRET(__split_safe_free(session, split_gen, false, pindex, size)); root_decr += size; - /* Adjust the root's memory footprint and mark it dirty. */ + /* Adjust the root's memory footprint. */ __wt_cache_page_inmem_incr(session, root, root_incr); __wt_cache_page_inmem_decr(session, root, root_decr); - __wt_page_modify_set(session, root); -err: switch (complete) { +err: __split_ref_final(session, &locked); + + switch (complete) { case WT_ERR_RETURN: __wt_free_ref_index(session, root, alloc_index, true); break; @@ -743,8 +782,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, empty_parent = false; complete = WT_ERR_RETURN; - /* The parent page will be marked dirty, make sure that will succeed. */ + /* Mark the page dirty. */ WT_RET(__wt_page_modify_init(session, parent)); + __wt_page_modify_set(session, parent); /* * We've locked the parent, which means it cannot split (which is the @@ -972,10 +1012,9 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, WT_TRET(__split_safe_free(session, split_gen, exclusive, pindex, size)); parent_decr += size; - /* Adjust the parent's memory footprint and mark it dirty. */ + /* Adjust the parent's memory footprint. */ __wt_cache_page_inmem_incr(session, parent, parent_incr); __wt_cache_page_inmem_decr(session, parent, parent_decr); - __wt_page_modify_set(session, parent); err: __wt_scr_free(session, &scr); /* @@ -1025,10 +1064,9 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) { WT_BTREE *btree; WT_DECL_RET; - WT_PAGE *child; + WT_PAGE *child, **locked; WT_PAGE_INDEX *alloc_index, *child_pindex, *pindex, *replace_index; - WT_REF **alloc_refp; - WT_REF **child_refp, *page_ref, **page_refp, *ref; + WT_REF **alloc_refp, **child_refp, *page_ref, **page_refp, *ref; WT_SPLIT_ERROR_PHASE complete; size_t child_incr, page_decr, page_incr, parent_incr, size; uint64_t split_gen; @@ -1039,12 +1077,14 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) WT_STAT_CONN_INCR(session, cache_eviction_split_internal); WT_STAT_DATA_INCR(session, cache_eviction_split_internal); - /* The page will be marked dirty, make sure that will succeed. */ + /* Mark the page dirty. */ WT_RET(__wt_page_modify_init(session, page)); + __wt_page_modify_set(session, page); btree = S2BT(session); alloc_index = replace_index = NULL; page_ref = page->pg_intl_parent_ref; + locked = NULL; page_decr = page_incr = parent_incr = 0; complete = WT_ERR_RETURN; @@ -1111,6 +1151,17 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) WT_ERR(__wt_calloc_one(session, alloc_refp)); parent_incr += children * sizeof(WT_REF); + /* + * Once the split is live, newly created internal pages might be evicted + * and their WT_REF structures freed. If that happens before all threads + * exit the index of the page that previously "owned" the WT_REF, a + * thread might see a freed WT_REF. To ensure that doesn't happen, the + * created pages are set to the current split generation and so can't be + * evicted until all readers have left the old generation. + */ + split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + WT_ASSERT(session, page->pg_intl_split_gen < split_gen); + /* Allocate child pages, and connect them into the new page index. */ WT_ASSERT(session, page_refp == pindex->index + chunk); for (alloc_refp = alloc_index->index + 1, i = 1; i < children; ++i) { @@ -1137,6 +1188,7 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) /* Initialize the child page. */ child->pg_intl_parent_ref = ref; + child->pg_intl_split_gen = split_gen; /* Mark it dirty. */ WT_ERR(__wt_page_modify_init(session, child)); @@ -1173,7 +1225,7 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) * generation to block splits in newly created pages, so get one. */ WT_ENTER_PAGE_INDEX(session); - __split_ref_prepare(session, alloc_index, session->split_gen, true); + WT_ERR(__split_ref_prepare(session, alloc_index, &locked, true)); /* Split into the parent. */ if ((ret = __split_parent(session, page_ref, alloc_index->index, @@ -1197,6 +1249,9 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); page->pg_intl_split_gen = split_gen; + /* Finalize the WT_REF move. */ + __split_ref_final(session, &locked); + #ifdef HAVE_DIAGNOSTIC WT_WITH_PAGE_INDEX(session, __split_verify_intl_key_order(session, parent)); @@ -1228,12 +1283,13 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) WT_TRET(__split_safe_free(session, split_gen, false, pindex, size)); page_decr += size; - /* Adjust the page's memory footprint, and mark it dirty. */ + /* Adjust the page's memory footprint. */ __wt_cache_page_inmem_incr(session, page, page_incr); __wt_cache_page_inmem_decr(session, page, page_decr); - __wt_page_modify_set(session, page); -err: switch (complete) { +err: __split_ref_final(session, &locked); + + switch (complete) { case WT_ERR_RETURN: __wt_free_ref_index(session, page, alloc_index, true); __wt_free_ref_index(session, page, replace_index, false); diff --git a/src/third_party/wiredtiger/src/os_posix/os_fs.c b/src/third_party/wiredtiger/src/os_posix/os_fs.c index bc8cbf67025..3c90183caf2 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fs.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fs.c @@ -31,6 +31,23 @@ /* * __posix_sync -- * Underlying support function to flush a file descriptor. + * + * Fsync calls (or fsync-style calls, for example, fdatasync) are not retried + * on failure, and failure halts the system. + * + * Excerpted from the LWN.net article https://lwn.net/Articles/752063/: + * In short, PostgreSQL assumes that a successful call to fsync() indicates + * that all data written since the last successful call made it safely to + * persistent storage. But that is not what the kernel actually does. When + * a buffered I/O write fails due to a hardware-level error, filesystems + * will respond differently, but that behavior usually includes discarding + * the data in the affected pages and marking them as being clean. So a read + * of the blocks that were just written will likely return something other + * than the data that was written. + * + * Given the shared history of UNIX filesystems, and the difficulty of knowing + * what specific error will be returned under specific circumstances, we don't + * retry fsync-style calls and panic if a flush operation fails. */ static int __posix_sync( @@ -52,25 +69,49 @@ __posix_sync( * OS X F_FULLFSYNC fcntl documentation: * "This is currently implemented on HFS, MS-DOS (FAT), and Universal * Disk Format (UDF) file systems." + * + * See comment in __posix_sync(): sync cannot be retried or fail. */ - WT_SYSCALL_RETRY(fcntl(fd, F_FULLFSYNC, 0) == -1 ? -1 : 0, ret); - if (ret == 0) - return (0); - /* - * Assume F_FULLFSYNC failed because the file system doesn't support it - * and fallback to fsync. - */ + static enum { FF_NOTSET, FF_IGNORE, FF_OK } ff_status = FF_NOTSET; + switch (ff_status) { + case FF_NOTSET: + WT_SYSCALL(fcntl(fd, F_FULLFSYNC, 0) == -1 ? -1 : 0, ret); + if (ret == 0) { + ff_status = FF_OK; + return (0); + } + + /* + * If the first F_FULLFSYNC fails, assume the file system + * doesn't support it and fallback to fdatasync or fsync. + */ + ff_status = FF_IGNORE; + __wt_err(session, ret, + "fcntl(F_FULLFSYNC) failed, falling back to fdatasync " + "or fsync"); + break; + case FF_IGNORE: + break; + case FF_OK: + WT_SYSCALL(fcntl(fd, F_FULLFSYNC, 0) == -1 ? -1 : 0, ret); + if (ret == 0) + return (0); + WT_PANIC_RET(session, + ret, "%s: %s: fcntl(F_FULLFSYNC)", name, func); + } #endif #if defined(HAVE_FDATASYNC) - WT_SYSCALL_RETRY(fdatasync(fd), ret); + /* See comment in __posix_sync(): sync cannot be retried or fail. */ + WT_SYSCALL(fdatasync(fd), ret); if (ret == 0) return (0); - WT_RET_MSG(session, ret, "%s: %s: fdatasync", name, func); + WT_PANIC_RET(session, ret, "%s: %s: fdatasync", name, func); #else - WT_SYSCALL_RETRY(fsync(fd), ret); + /* See comment in __posix_sync(): sync cannot be retried or fail. */ + WT_SYSCALL(fsync(fd), ret); if (ret == 0) return (0); - WT_RET_MSG(session, ret, "%s: %s: fsync", name, func); + WT_PANIC_RET(session, ret, "%s: %s: fsync", name, func); #endif } @@ -108,12 +149,15 @@ __posix_directory_sync(WT_SESSION_IMPL *session, const char *path) WT_SYSCALL(close(fd), tret); if (tret != 0) { __wt_err(session, tret, "%s: directory-sync: close", dir); - if (ret == 0) - ret = tret; + WT_TRET(tret); } err: __wt_scr_free(session, &tmp); - return (ret); + if (ret == 0) + return (ret); + + /* See comment in __posix_sync(): sync cannot be retried or fail. */ + WT_PANIC_RET(session, ret, "%s: directory-sync", path); } #endif @@ -460,11 +504,13 @@ __posix_file_sync_nowait(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session) session = (WT_SESSION_IMPL *)wt_session; pfh = (WT_FILE_HANDLE_POSIX *)file_handle; - WT_SYSCALL_RETRY(sync_file_range(pfh->fd, + /* See comment in __posix_sync(): sync cannot be retried or fail. */ + WT_SYSCALL(sync_file_range(pfh->fd, (off64_t)0, (off64_t)0, SYNC_FILE_RANGE_WRITE), ret); if (ret == 0) return (0); - WT_RET_MSG(session, ret, + + WT_PANIC_RET(session, ret, "%s: handle-sync-nowait: sync_file_range", file_handle->name); } #endif diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 22b40f7164d..146861e2adc 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -33,7 +33,6 @@ static void config_checksum(void); static void config_compression(const char *); static void config_encryption(void); static const char *config_file_type(u_int); -static CONFIG *config_find(const char *, size_t); static void config_in_memory(void); static void config_in_memory_check(void); static int config_is_perm(const char *); @@ -672,6 +671,35 @@ config_reset(void) } /* + * config_find + * Find a specific configuration entry. + */ +static CONFIG * +config_find(const char *s, size_t len, bool fatal) +{ + CONFIG *cp; + + for (cp = c; cp->name != NULL; ++cp) + if (strncmp(s, cp->name, len) == 0 && cp->name[len] == '\0') + return (cp); + + /* + * Optionally ignore unknown keywords, it makes it easier to run old + * CONFIG files. + */ + if (fatal) { + fprintf(stderr, + "%s: %s: unknown required configuration keyword\n", + progname, s); + exit(EXIT_FAILURE); + } + fprintf(stderr, + "%s: %s: WARNING, ignoring unknown configuration keyword\n", + progname, s); + return (NULL); +} + +/* * config_single -- * Set a single configuration structure value. */ @@ -690,7 +718,9 @@ config_single(const char *s, int perm) exit(EXIT_FAILURE); } - cp = config_find(s, (size_t)(ep - s)); + if ((cp = config_find(s, (size_t)(ep - s), false)) == NULL) + return; + F_SET(cp, perm ? C_PERM : C_TEMP); ++ep; @@ -876,25 +906,6 @@ config_map_isolation(const char *s, u_int *vp) } /* - * config_find - * Find a specific configuration entry. - */ -static CONFIG * -config_find(const char *s, size_t len) -{ - CONFIG *cp; - - for (cp = c; cp->name != NULL; ++cp) - if (strncmp(s, cp->name, len) == 0 && cp->name[len] == '\0') - return (cp); - - fprintf(stderr, - "%s: %s: unknown configuration keyword\n", progname, s); - config_error(); - exit(EXIT_FAILURE); -} - -/* * config_is_perm * Return if a specific configuration entry was permanently set. */ @@ -903,7 +914,7 @@ config_is_perm(const char *s) { CONFIG *cp; - cp = config_find(s, strlen(s)); + cp = config_find(s, strlen(s), true); return (F_ISSET(cp, C_PERM) ? 1 : 0); } diff --git a/version.json b/version.json index d615f473529..efa53697d06 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { - "githash": "52e5b5fbaa3a2a5b1a217f5e647b5061817475f9", - "version": "3.4.15" + "githash": "0d6a9242c11b99ddadcfb6e86a850b6ba487530a", + "version": "3.4.16" }
\ No newline at end of file |