diff options
Diffstat (limited to 'buildscripts')
65 files changed, 1246 insertions, 77 deletions
diff --git a/buildscripts/packager.py b/buildscripts/packager.py index dcc95726398..d46d7e193c3 100755 --- a/buildscripts/packager.py +++ b/buildscripts/packager.py @@ -258,9 +258,7 @@ class Distro(object): else: raise Exception("unsupported build_os: %s" % build_os) elif self.n == 'debian': - if build_os == 'debian71': - return 'wheezy' - elif build_os == 'debian81': + if build_os == 'debian81': return 'jessie' else: raise Exception("unsupported build_os: %s" % build_os) @@ -277,7 +275,7 @@ class Distro(object): def build_os(self, arch): """Return the build os label in the binary package to download (e.g. "rhel55" for redhat, - "ubuntu1204" for ubuntu, "debian71" for debian, "suse11" for suse, etc.)""" + "ubuntu1204" for ubuntu, "debian81" for debian, "suse11" for suse, etc.)""" # Community builds only support amd64 if arch not in ['x86_64', 'ppc64le', 's390x', 'arm64']: raise Exception("BUG: unsupported architecture (%s)" % arch) @@ -291,7 +289,7 @@ class Distro(object): elif self.n == 'ubuntu': return [ "ubuntu1204", "ubuntu1404", "ubuntu1604", ] elif self.n == 'debian': - return [ "debian71", "debian81" ] + return [ "debian81" ] else: raise Exception("BUG: unsupported platform?") @@ -471,12 +469,8 @@ def make_deb(distro, build_os, arch, spec, srcdir): sdir=setupdir(distro, build_os, arch, spec) if re.search("debian", distro.name()): os.unlink(sdir+"debian/mongod.upstart") - if build_os == "debian71": - os.link(sdir+"debian/init.d", sdir+"debian/%s%s-server.mongod.init" % (distro.pkgbase(), suffix)) - os.unlink(sdir+"debian/mongod.service") - else: - os.link(sdir+"debian/mongod.service", sdir+"debian/%s%s-server.mongod.service" % (distro.pkgbase(), suffix)) - os.unlink(sdir+"debian/init.d") + os.link(sdir+"debian/mongod.service", sdir+"debian/%s%s-server.mongod.service" % (distro.pkgbase(), suffix)) + os.unlink(sdir+"debian/init.d") elif re.search("ubuntu", distro.name()): os.unlink(sdir+"debian/init.d") if build_os in ("ubuntu1204", "ubuntu1404", "ubuntu1410"): diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index 7aa13277384..cee9ce4e391 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -54,10 +54,18 @@ def _execute_suite(suite, logging_config): logger.info("Skipping %ss, no tests to run", group.test_kind) continue + archive = None + if resmokelib.config.ARCHIVE_FILE: + archive = resmokelib.utils.archival.Archival( + archival_json_file=resmokelib.config.ARCHIVE_FILE, + limit_size_mb=resmokelib.config.ARCHIVE_LIMIT_MB, + limit_files=resmokelib.config.ARCHIVE_LIMIT_TESTS, + logger=logger) group_config = suite.get_executor_config().get(group.test_kind, {}) executor = resmokelib.testing.executor.TestGroupExecutor(logger, group, logging_config, + archive_instance=archive, **group_config) try: @@ -73,6 +81,9 @@ def _execute_suite(suite, logging_config): group.test_kind, suite.get_name()) suite.return_code = 2 return False + finally: + if archive: + archive.exit() def _log_summary(logger, suites, time_taken): diff --git a/buildscripts/resmokeconfig/suites/aggregation.yml b/buildscripts/resmokeconfig/suites/aggregation.yml index c7a8c6e64be..0a775a2d93c 100644 --- a/buildscripts/resmokeconfig/suites/aggregation.yml +++ b/buildscripts/resmokeconfig/suites/aggregation.yml @@ -8,6 +8,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/aggregation_ese.yml b/buildscripts/resmokeconfig/suites/aggregation_ese.yml index 395a8a37898..a2d260d91d1 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_ese.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_ese.yml @@ -16,6 +16,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/aggregation_facet_unwind_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_facet_unwind_passthrough.yml index 40d248aa22d..476ba72b05d 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_facet_unwind_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_facet_unwind_passthrough.yml @@ -8,6 +8,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml index 72b838b3c07..5eea3a717bb 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml @@ -31,6 +31,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml index 1cb834a63eb..c5b76c31ef5 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml @@ -29,6 +29,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/bulk_gle_passthrough.yml b/buildscripts/resmokeconfig/suites/bulk_gle_passthrough.yml index 8e88985f671..309dd0364ea 100644 --- a/buildscripts/resmokeconfig/suites/bulk_gle_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/bulk_gle_passthrough.yml @@ -7,6 +7,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: legacy diff --git a/buildscripts/resmokeconfig/suites/concurrency.yml b/buildscripts/resmokeconfig/suites/concurrency.yml index e585902274d..4af6a5e31ee 100644 --- a/buildscripts/resmokeconfig/suites/concurrency.yml +++ b/buildscripts/resmokeconfig/suites/concurrency.yml @@ -12,6 +12,10 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections + tests: true config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_replication.yml index 8c9d272caea..9a575e63356 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication.yml @@ -6,6 +6,8 @@ selector: # Concurrency tests that run against a replica set start one themselves. executor: js_test: + archive: + tests: true config: shell_options: nodb: '' diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml index b5bd8b7169e..0e219f2f26c 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml @@ -6,6 +6,8 @@ selector: # Concurrency tests that run against a sharded cluster start one themselves. executor: js_test: + archive: + tests: true config: shell_options: nodb: '' diff --git a/buildscripts/resmokeconfig/suites/core.yml b/buildscripts/resmokeconfig/suites/core.yml index 04a6f97cb7d..9ba9ee4b742 100644 --- a/buildscripts/resmokeconfig/suites/core.yml +++ b/buildscripts/resmokeconfig/suites/core.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/core_auth.yml b/buildscripts/resmokeconfig/suites/core_auth.yml index 72c6c716927..cca107d5afa 100644 --- a/buildscripts/resmokeconfig/suites/core_auth.yml +++ b/buildscripts/resmokeconfig/suites/core_auth.yml @@ -18,6 +18,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/core_compression.yml b/buildscripts/resmokeconfig/suites/core_compression.yml index d09716735da..32e3d381bbc 100644 --- a/buildscripts/resmokeconfig/suites/core_compression.yml +++ b/buildscripts/resmokeconfig/suites/core_compression.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/core_ese.yml b/buildscripts/resmokeconfig/suites/core_ese.yml index 74ab7c9b195..86aa646bdb7 100644 --- a/buildscripts/resmokeconfig/suites/core_ese.yml +++ b/buildscripts/resmokeconfig/suites/core_ese.yml @@ -9,6 +9,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/core_minimum_batch_size.yml b/buildscripts/resmokeconfig/suites/core_minimum_batch_size.yml index 085c32c8c29..3b5a179914b 100644 --- a/buildscripts/resmokeconfig/suites/core_minimum_batch_size.yml +++ b/buildscripts/resmokeconfig/suites/core_minimum_batch_size.yml @@ -13,6 +13,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/core_op_query.yml b/buildscripts/resmokeconfig/suites/core_op_query.yml index 5ec056a57ff..45e57b65376 100644 --- a/buildscripts/resmokeconfig/suites/core_op_query.yml +++ b/buildscripts/resmokeconfig/suites/core_op_query.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: rpcProtocols: opQueryOnly diff --git a/buildscripts/resmokeconfig/suites/decimal.yml b/buildscripts/resmokeconfig/suites/decimal.yml index 62cb11ecab2..6c934554e42 100644 --- a/buildscripts/resmokeconfig/suites/decimal.yml +++ b/buildscripts/resmokeconfig/suites/decimal.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/disk.yml b/buildscripts/resmokeconfig/suites/disk.yml index fd8a5fd54b1..845ce95ea8f 100644 --- a/buildscripts/resmokeconfig/suites/disk.yml +++ b/buildscripts/resmokeconfig/suites/disk.yml @@ -7,6 +7,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/dur_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/dur_jscore_passthrough.yml index 347d23de967..f973f95586c 100644 --- a/buildscripts/resmokeconfig/suites/dur_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/dur_jscore_passthrough.yml @@ -8,6 +8,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/httpinterface.yml b/buildscripts/resmokeconfig/suites/httpinterface.yml index 1f15a873236..c74801ccdf6 100644 --- a/buildscripts/resmokeconfig/suites/httpinterface.yml +++ b/buildscripts/resmokeconfig/suites/httpinterface.yml @@ -7,6 +7,9 @@ selector: - jstests/httpinterface/httpinterface.js executor: js_test: + archive: + hooks: + - ValidateCollections hooks: - class: ValidateCollections fixture: diff --git a/buildscripts/resmokeconfig/suites/integration_tests_replset.yml b/buildscripts/resmokeconfig/suites/integration_tests_replset.yml index 1345b567eeb..f4d3e6e312e 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_replset.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_replset.yml @@ -4,6 +4,11 @@ selector: executor: cpp_integration_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: {} hooks: # The CheckReplDBHash hook waits until all operations have replicated to and have been applied diff --git a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml index a0e8c16be9b..444c66aa30d 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml @@ -6,6 +6,10 @@ selector: executor: cpp_integration_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: {} hooks: - class: CheckReplDBHash diff --git a/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml b/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml index c594fc6fae1..127a7535568 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_standalone.yml @@ -4,6 +4,9 @@ selector: executor: cpp_integration_test: + archive: + hooks: + - ValidateCollections config: {} hooks: - class: ValidateCollections diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz.yml b/buildscripts/resmokeconfig/suites/jstestfuzz.yml index 872c2a1f229..591255038f3 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml index a482038899f..f32a24e4ee8 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml @@ -5,6 +5,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml index 034f254494f..d38814a0008 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_initsync.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - BackgroundInitialSync config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_resync.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_resync.yml index 1038b99a10c..973914a3c01 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication_resync.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication_resync.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - BackgroundInitialSync config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml index 9a8a02db6b9..bf2f84fe373 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml @@ -5,6 +5,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/master_slave_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/master_slave_jscore_passthrough.yml index a46c0cf29ed..1de4c693009 100644 --- a/buildscripts/resmokeconfig/suites/master_slave_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/master_slave_jscore_passthrough.yml @@ -10,6 +10,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/mmap.yml b/buildscripts/resmokeconfig/suites/mmap.yml index 6fc951c25cb..0aefad2ee53 100644 --- a/buildscripts/resmokeconfig/suites/mmap.yml +++ b/buildscripts/resmokeconfig/suites/mmap.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml index 2afbe6ae044..8012bfaf155 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml @@ -9,6 +9,10 @@ selector: # noPassthrough tests start their own mongod's. executor: js_test: + archive: + tests: + - jstests/noPassthrough/backup*.js + - jstests/noPassthrough/wt_unclean_shutdown.js config: shell_options: nodb: '' diff --git a/buildscripts/resmokeconfig/suites/no_passthrough_with_mongod.yml b/buildscripts/resmokeconfig/suites/no_passthrough_with_mongod.yml index fa8e62813d1..4e8bfc22bab 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough_with_mongod.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough_with_mongod.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/parallel.yml b/buildscripts/resmokeconfig/suites/parallel.yml index 7f42d19c18c..9611ec55c48 100644 --- a/buildscripts/resmokeconfig/suites/parallel.yml +++ b/buildscripts/resmokeconfig/suites/parallel.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml index 22e84bc9c63..d469ff85f75 100644 --- a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml @@ -28,6 +28,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml index 114ebcab372..034c0812e90 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml @@ -73,6 +73,9 @@ selector: run_hook_interval: &run_hook_interval 20 executor: js_test: + archive: + hooks: + - BackgroundInitialSync config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_initsync_static_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_initsync_static_jscore_passthrough.yml index c15b23046a9..8fce4506393 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_initsync_static_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_initsync_static_jscore_passthrough.yml @@ -12,6 +12,9 @@ selector: run_hook_interval: &run_hook_interval 20 executor: js_test: + archive: + hooks: + - IntermediateInitialSync config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml index f487c1aa113..60da231d3bc 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml @@ -11,6 +11,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_kill_secondaries_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_kill_secondaries_jscore_passthrough.yml index a4b6c5a5feb..e62727bb606 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_kill_secondaries_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_kill_secondaries_jscore_passthrough.yml @@ -15,6 +15,9 @@ selector: executor: js_test: + archive: + hooks: + - PeriodicKillSecondaries config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml index 1e51713a475..c7d605d1184 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml @@ -73,6 +73,9 @@ selector: run_hook_interval: &run_hook_interval 20 executor: js_test: + archive: + hooks: + - BackgroundInitialSync config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/replica_sets_resync_static_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_resync_static_jscore_passthrough.yml index dc2059879d3..bd26f1c23f8 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_resync_static_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_resync_static_jscore_passthrough.yml @@ -12,6 +12,9 @@ selector: run_hook_interval: &run_hook_interval 20 executor: js_test: + archive: + hooks: + - IntermediateInitialSync config: shell_options: eval: "testingReplication = true;" diff --git a/buildscripts/resmokeconfig/suites/rlp.yml b/buildscripts/resmokeconfig/suites/rlp.yml index ef0dfeda45a..16413f1d4a8 100644 --- a/buildscripts/resmokeconfig/suites/rlp.yml +++ b/buildscripts/resmokeconfig/suites/rlp.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/serial_run.yml b/buildscripts/resmokeconfig/suites/serial_run.yml index 4ae010c7ec8..c25e61a980a 100644 --- a/buildscripts/resmokeconfig/suites/serial_run.yml +++ b/buildscripts/resmokeconfig/suites/serial_run.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml index 96674cfc20c..104707d1ac8 100644 --- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml @@ -69,6 +69,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml index 8879e39f143..5f646fd7653 100644 --- a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml @@ -19,6 +19,9 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash config: shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml index 496bd265a58..d2cb6ed55f1 100644 --- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml @@ -51,6 +51,10 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplDBHash + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/slow1.yml b/buildscripts/resmokeconfig/suites/slow1.yml index 10c399ec9e5..8a764c39d94 100644 --- a/buildscripts/resmokeconfig/suites/slow1.yml +++ b/buildscripts/resmokeconfig/suites/slow1.yml @@ -5,6 +5,9 @@ selector: executor: js_test: + archive: + hooks: + - ValidateCollections config: shell_options: readMode: commands diff --git a/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml index 388046d2782..b19989e3656 100644 --- a/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml @@ -46,6 +46,11 @@ selector: executor: js_test: + archive: + hooks: + - CheckReplOplogs + - CheckReplDBHash + - ValidateCollections config: shell_options: global_vars: diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index b75753398a5..8f636ddbadd 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -33,14 +33,20 @@ MONGO_RUNNER_SUBDIR = "mongorunner" # Names below correspond to how they are specified via the command line or in the options YAML file. DEFAULTS = { + "archiveFile": None, + "archiveLimitMb": 5000, + "archiveLimitTests": 10, "basePort": 20000, "buildloggerUrl": "https://logkeeper.mongodb.org", "continueOnFailure": False, "dbpathPrefix": None, "dbtest": None, + "distroId": None, "dryRun": None, "excludeWithAllTags": None, "excludeWithAnyTags": None, + "executionNumber": 0, + "gitRevision": None, "includeWithAllTags": None, "includeWithAnyTags": None, "jobs": 1, @@ -51,6 +57,7 @@ DEFAULTS = { "mongosSetParameters": None, "nojournal": False, "numClientsPerFixture": 1, + "projectName": "mongodb-mongo-master", "repeat": 1, "reportFile": None, "seed": long(time.time() * 256), # Taken from random.py code in Python 2.7. @@ -59,6 +66,9 @@ DEFAULTS = { "shuffle": False, "storageEngine": None, "storageEngineCacheSizeGB": None, + "taskId": None, + "taskName": None, + "variantName": None, "wiredTigerCollectionConfigString": None, "wiredTigerEngineConfigString": None, "wiredTigerIndexConfigString": None @@ -69,6 +79,15 @@ DEFAULTS = { # Variables that are set by the user at the command line or with --options. ## +# The name of the archive JSON file used to associate S3 archives to an Evergreen task. +ARCHIVE_FILE = None + +# The limit size of all archive files for an Evergreen task. +ARCHIVE_LIMIT_MB = None + +# The limit number of tests to archive for an Evergreen task. +ARCHIVE_LIMIT_TESTS = None + # The starting port number to use for mongod and mongos processes spawned by resmoke.py and the # mongo shell. BASE_PORT = None @@ -87,6 +106,28 @@ DBTEST_EXECUTABLE = None # actually running them). DRY_RUN = None +# The identifier for the Evergreen distro that resmoke.py is being run on. +EVERGREEN_DISTRO_ID = None + +# The number of the Evergreen execution that resmoke.py is being run on. +EVERGREEN_EXECUTION = None + +# The name of the Evergreen project that resmoke.py is being run on. +EVERGREEN_PROJECT_NAME = None + +# The git revision of the Evergreen task that resmoke.py is being run on. +EVERGREEN_REVISION = None + +# The identifier for the Evergreen task that resmoke.py is being run under. If set, then the +# Evergreen task id value will be transmitted to logkeeper when creating builds and tests. +EVERGREEN_TASK_ID = None + +# The name of the Evergreen task that resmoke.py is being run for. +EVERGREEN_TASK_NAME = None + +# The name of the Evergreen build variant that resmoke.py is being run on. +EVERGREEN_VARIANT_NAME = None + # If set, then any jstests that have all of the specified tags will be excluded from the suite(s). EXCLUDE_WITH_ALL_TAGS = None @@ -177,6 +218,9 @@ WT_INDEX_CONFIG = None # Internally used configuration options that aren't exposed to the user ## +# S3 Bucket to upload archive files. +ARCHIVE_BUCKET = "mongodatafiles" + # Default sort order for test execution. Will only be changed if --suites wasn't specified. ORDER_TESTS_BY_NAME = True diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 870e0e9145c..59b4585a273 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -18,14 +18,20 @@ from .. import resmokeconfig # Mapping of the attribute of the parsed arguments (dest) to its key as it appears in the options # YAML configuration file. Most should only be converting from snake_case to camelCase. DEST_TO_CONFIG = { + "archive_file": "archiveFile", + "archive_limit_mb": "archiveLimitMb", + "archive_limit_tests": "archiveLimitTests", "base_port": "basePort", "buildlogger_url": "buildloggerUrl", "continue_on_failure": "continueOnFailure", "dbpath_prefix": "dbpathPrefix", "dbtest_executable": "dbtest", + "distro_id": "distroId", "dry_run": "dryRun", "exclude_with_all_tags": "excludeWithAllTags", "exclude_with_any_tags": "excludeWithAnyTags", + "execution_number": "executionNumber", + "git_revision": "gitRevision", "include_with_all_tags": "includeWithAllTags", "include_with_any_tags": "includeWithAnyTags", "jobs": "jobs", @@ -37,6 +43,7 @@ DEST_TO_CONFIG = { "no_journal": "nojournal", "num_clients_per_fixture": "numClientsPerFixture", "prealloc_journal": "preallocJournal", + "project_name": "projectName", "repeat": "repeat", "report_file": "reportFile", "seed": "seed", @@ -45,6 +52,9 @@ DEST_TO_CONFIG = { "shuffle": "shuffle", "storage_engine": "storageEngine", "storage_engine_cache_size": "storageEngineCacheSizeGB", + "task_id": "taskId", + "task_name": "taskName", + "variant_name": "variantName", "wt_coll_config": "wiredTigerCollectionConfigString", "wt_engine_config": "wiredTigerEngineConfigString", "wt_index_config": "wiredTigerIndexConfigString" @@ -79,6 +89,23 @@ def parse_command_line(): parser.add_option("--options", dest="options_file", metavar="OPTIONS", help="A YAML file that specifies global options to resmoke.py.") + parser.add_option("--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE", + help=("Sets the archive file name for the Evergreen task running the tests." + " The archive file is JSON format containing a list of tests that were" + " successfully archived to S3. If unspecified, no data files from tests" + " will be archived in S3. Tests can be designated for archival in the" + " task suite configuration file.")) + + parser.add_option("--archiveLimitMb", type="int", dest="archive_limit_mb", + metavar="ARCHIVE_LIMIT_MB", + help=("Sets the limit (in MB) for archived files to S3. A value of 0" + " indicates there is no limit.")) + + parser.add_option("--archiveLimitTests", type="int", dest="archive_limit_tests", + metavar="ARCHIVE_LIMIT_TESTS", + help=("Sets the maximum number of tests to archive to S3. A value" + " of 0 indicates there is no limit.")) + parser.add_option("--basePort", dest="base_port", metavar="PORT", help=("The starting port number to use for mongod and mongos processes" " spawned by resmoke.py or the tests themselves. Each fixture and Job" @@ -209,6 +236,42 @@ def parse_command_line(): parser.add_option("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG", help="Set the WiredTiger index configuration setting for all mongod's.") + + evergreen_options = optparse.OptionGroup( + parser, title="Evergreen options", + description=("Options used to propagate information about the Evergreen task running this" + " script.")) + parser.add_option_group(evergreen_options) + + evergreen_options.add_option("--buildId", dest="build_id", metavar="BUILD_ID", + help="Sets the build ID of the task.") + + evergreen_options.add_option("--distroId", dest="distro_id", metavar="DISTRO_ID", + help=("Sets the identifier for the Evergreen distro running the" + " tests.")) + + evergreen_options.add_option("--executionNumber", type="int", dest="execution_number", + metavar="EXECUTION_NUMBER", + help=("Sets the number for the Evergreen execution running the" + " tests.")) + + evergreen_options.add_option("--gitRevision", dest="git_revision", metavar="GIT_REVISION", + help=("Sets the git revision for the Evergreen task running the" + " tests.")) + + evergreen_options.add_option("--projectName", dest="project_name", metavar="PROJECT_NAME", + help=("Sets the name of the Evergreen project running the tests.")) + + evergreen_options.add_option("--taskName", dest="task_name", metavar="TASK_NAME", + help="Sets the name of the Evergreen task running the tests.") + + evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID", + help="Sets the Id of the Evergreen task running the tests.") + + evergreen_options.add_option("--variantName", dest="variant_name", metavar="VARIANT_NAME", + help=("Sets the name of the Evergreen build variant running the" + " tests.")) + parser.set_defaults(executor_file="with_server", logger_file="console", dry_run="off", @@ -237,11 +300,21 @@ def update_config_vars(values): if values[dest] is not None: config[config_var] = values[dest] + _config.ARCHIVE_FILE = config.pop("archiveFile") + _config.ARCHIVE_LIMIT_MB = config.pop("archiveLimitMb") + _config.ARCHIVE_LIMIT_TESTS = config.pop("archiveLimitTests") _config.BASE_PORT = int(config.pop("basePort")) _config.BUILDLOGGER_URL = config.pop("buildloggerUrl") _config.DBPATH_PREFIX = _expand_user(config.pop("dbpathPrefix")) _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest")) _config.DRY_RUN = config.pop("dryRun") + _config.EVERGREEN_DISTRO_ID = config.pop("distroId") + _config.EVERGREEN_EXECUTION = config.pop("executionNumber") + _config.EVERGREEN_PROJECT_NAME = config.pop("projectName") + _config.EVERGREEN_REVISION = config.pop("gitRevision") + _config.EVERGREEN_TASK_ID = config.pop("taskId") + _config.EVERGREEN_TASK_NAME = config.pop("taskName") + _config.EVERGREEN_VARIANT_NAME = config.pop("variantName") _config.EXCLUDE_WITH_ALL_TAGS = config.pop("excludeWithAllTags") _config.EXCLUDE_WITH_ANY_TAGS = config.pop("excludeWithAnyTags") _config.FAIL_FAST = not config.pop("continueOnFailure") diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 438a06b9c8e..e88eed6bc02 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -8,6 +8,7 @@ import threading import time from . import fixtures +from . import hook_test_archival as archival from . import hooks as _hooks from . import job as _job from . import report as _report @@ -35,7 +36,9 @@ class TestGroupExecutor(object): logging_config, config=None, fixture=None, - hooks=None): + hooks=None, + archive_instance=None, + archive=None): """ Initializes the TestGroupExecutor with the test group to run. """ @@ -49,6 +52,11 @@ class TestGroupExecutor(object): self.hooks_config = utils.default_if_none(hooks, []) self.test_config = utils.default_if_none(config, {}) + self.archival = None + if archive_instance: + self.archival = archival.HookTestArchival(test_group, self.hooks_config, + archive_instance, archive) + self._test_group = test_group self._using_buildlogger = logging.config.using_buildlogger(logging_config) @@ -299,7 +307,7 @@ class TestGroupExecutor(object): build_id=build_id, build_config=build_config) - return _job.Job(logger, fixture, hooks, report) + return _job.Job(logger, fixture, hooks, report, self.archival) def _make_test_queue(self): """ diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index f2cb1b9c8b8..5da95175008 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -4,12 +4,16 @@ Interface of the different fixtures for executing JSTests against. from __future__ import absolute_import +import os.path import time import pymongo +import pymongo.errors +from ... import config from ... import errors from ... import logging +from ... import utils class Fixture(object): @@ -17,7 +21,7 @@ class Fixture(object): Base class for all fixtures. """ - def __init__(self, logger, job_num): + def __init__(self, logger, job_num, dbpath_prefix=None): """ Initializes the fixture with a logger instance. """ @@ -32,9 +36,12 @@ class Fixture(object): self.logger = logger self.job_num = job_num - self.port = None # Port that the mongo shell should connect to. + dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) + dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) + self._dbpath_prefix = os.path.join(dbpath_prefix, "job{}".format(self.job_num)) + def setup(self): """ Creates the fixture. @@ -79,6 +86,9 @@ class Fixture(object): """ return True + def get_dbpath_prefix(self): + return self._dbpath_prefix + def get_internal_connection_string(self): """ Returns the connection string for this fixture. This is NOT a diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py index fb444cfe097..f8bf4417451 100644 --- a/buildscripts/resmokelib/testing/fixtures/masterslave.py +++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py @@ -31,7 +31,7 @@ class MasterSlaveFixture(interface.ReplFixture): dbpath_prefix=None, preserve_dbpath=False): - interface.ReplFixture.__init__(self, logger, job_num) + interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) if "dbpath" in mongod_options: raise ValueError("Cannot specify mongod_options.dbpath") @@ -42,12 +42,7 @@ class MasterSlaveFixture(interface.ReplFixture): self.slave_options = utils.default_if_none(slave_options, {}) self.preserve_dbpath = preserve_dbpath - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self._dbpath_prefix = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.master = None self.slave = None diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index 0ec3de30280..b185ee7f8f2 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -40,7 +40,7 @@ class ReplicaSetFixture(interface.ReplFixture): voting_secondaries=False, use_replica_set_connection_string=False): - interface.ReplFixture.__init__(self, logger, job_num) + interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) self.mongod_executable = mongod_executable self.mongod_options = utils.default_if_none(mongod_options, {}) @@ -59,12 +59,7 @@ class ReplicaSetFixture(interface.ReplFixture): if "dbpath" in self.mongod_options: self._dbpath_prefix = self.mongod_options.pop("dbpath") else: - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self._dbpath_prefix = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.nodes = [] self.replset_name = None diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index 2e2db535d6d..9a90c5b31e1 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -47,7 +47,7 @@ class ShardedClusterFixture(interface.Fixture): the mongod and mongos processes. """ - interface.Fixture.__init__(self, logger, job_num) + interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) if "dbpath" in mongod_options: raise ValueError("Cannot specify mongod_options.dbpath") @@ -62,12 +62,7 @@ class ShardedClusterFixture(interface.Fixture): self.enable_sharding = utils.default_if_none(enable_sharding, []) self.auth_options = auth_options - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self._dbpath_prefix = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.configsvr = None self.mongos = None @@ -108,11 +103,7 @@ class ShardedClusterFixture(interface.Fixture): self.port = self.mongos.port client = utils.new_mongo_client(port=self.port) - if self.auth_options is not None: - auth_db = client[self.auth_options["authenticationDatabase"]] - auth_db.authenticate(self.auth_options["username"], - password=self.auth_options["password"], - mechanism=self.auth_options["authenticationMechanism"]) + self._auth_to_db(client) # Inform mongos about each of the shards for shard in self.shards: @@ -123,6 +114,20 @@ class ShardedClusterFixture(interface.Fixture): self.logger.info("Enabling sharding for '%s' database...", db_name) client.admin.command({"enablesharding": db_name}) + def _auth_to_db(self, client): + """Authenticate client for the 'authenticationDatabase'.""" + if self.auth_options is not None: + auth_db = client[self.auth_options["authenticationDatabase"]] + auth_db.authenticate(self.auth_options["username"], + password=self.auth_options["password"], + mechanism=self.auth_options["authenticationMechanism"]) + + def _stop_balancer(self, timeout_ms=60000): + """Stop the balancer.""" + client = utils.new_mongo_client(port=self.port) + self._auth_to_db(client) + client.admin.command({"balancerStop": 1}, maxTimeMS=timeout_ms) + def _do_teardown(self): """ Shuts down the sharded cluster. @@ -134,6 +139,8 @@ class ShardedClusterFixture(interface.Fixture): self.logger.info( "Sharded cluster was expected to be running in _do_teardown(), but wasn't.") + self._stop_balancer() + if self.configsvr is not None: if running_at_start: self.logger.info("Stopping config server...") @@ -154,6 +161,8 @@ class ShardedClusterFixture(interface.Fixture): if running_at_start: self.logger.info("Stopping shards...") + + for shard in self.shards: success = shard.teardown() and success if running_at_start: diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index bc69775c285..d462685a827 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -6,7 +6,6 @@ from __future__ import absolute_import import os import os.path -import shutil import time import pymongo @@ -34,7 +33,7 @@ class MongoDFixture(interface.Fixture): dbpath_prefix=None, preserve_dbpath=False): - interface.Fixture.__init__(self, logger, job_num) + interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) if "dbpath" in mongod_options and dbpath_prefix is not None: raise ValueError("Cannot specify both mongod_options.dbpath and dbpath_prefix") @@ -48,19 +47,16 @@ class MongoDFixture(interface.Fixture): # The dbpath in mongod_options takes precedence over other settings to make it easier for # users to specify a dbpath containing data to test against. if "dbpath" not in self.mongod_options: - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self.mongod_options["dbpath"] = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self.mongod_options["dbpath"] = os.path.join( + self._dbpath_prefix, config.FIXTURE_SUBDIR) self._dbpath = self.mongod_options["dbpath"] self.mongod = None def setup(self): - if not self.preserve_dbpath: - shutil.rmtree(self._dbpath, ignore_errors=True) + """Set up the mongod.""" + if not self.preserve_dbpath and os.path.lexists(self._dbpath): + utils.rmtree(self._dbpath, ignore_errors=False) try: os.makedirs(self._dbpath) @@ -145,6 +141,10 @@ class MongoDFixture(interface.Fixture): def is_running(self): return self.mongod is not None and self.mongod.poll() is None + def get_dbpath_prefix(self): + """ Returns the _dbpath, as this is the root of the data directory. """ + return self._dbpath + def get_internal_connection_string(self): if self.mongod is None: raise ValueError("Must call setup() before calling get_internal_connection_string()") diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py new file mode 100644 index 00000000000..bde562cf617 --- /dev/null +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -0,0 +1,116 @@ +""" +Enables supports for archiving tests or hooks. +""" + +from __future__ import absolute_import + +import os +import threading + +from .. import config +from .. import utils +from ..utils import globstar + + +class HookTestArchival(object): + """ + Archives hooks and tests to S3. + """ + + def __init__(self, suite, hooks, archive_instance, archive_config): + self.archive_instance = archive_instance + archive_config = utils.default_if_none(archive_config, {}) + + self.on_success = archive_config.get("on_success", False) + + self.tests = [] + if "tests" in archive_config: + # 'tests' is either a list of tests to archive or a bool (archive all if True). + if not isinstance(archive_config["tests"], bool): + for test in archive_config["tests"]: + self.tests += globstar.glob(test) + elif archive_config["tests"]: + self.tests = suite.tests + + self.hooks = [] + if "hooks" in archive_config: + # 'hooks' is either a list of hooks to archive or a bool (archive all if True). + if not isinstance(archive_config["hooks"], bool): + self.hooks = archive_config["hooks"] + elif archive_config["hooks"]: + for hook in hooks: + self.hooks.append(hook["class"]) + + self._tests_repeat = {} + self._lock = threading.Lock() + + def _should_archive(self, success): + """ Return True if failed test or 'on_success' is True. """ + return not success or self.on_success + + def _archive_hook(self, logger, hook, test, success): + """ Helper to archive hooks. """ + hook_match = hook.__class__.__name__ in self.hooks + if not hook_match or not self._should_archive(success): + return + + test_name = "{}:{}".format(test.short_name(), hook.__class__.__name__) + self._archive_hook_or_test(logger, test_name, test) + + def _archive_test(self, logger, test, success): + """ Helper to archive tests. """ + test_name = test.test_name + test_match = False + for arch_test in self.tests: + # Ensure that the test_name is in the same format as the arch_test. + if os.path.normpath(test_name) == os.path.normpath(arch_test): + test_match = True + break + if not test_match or not self._should_archive(success): + return + + self._archive_hook_or_test(logger, test_name, test) + + def archive(self, logger, test, success, hook=None): + """ Archives data files for hooks or tests. """ + if not config.ARCHIVE_FILE or not self.archive_instance: + return + if hook: + self._archive_hook(logger, hook, test, success) + else: + self._archive_test(logger, test, success) + + def _archive_hook_or_test(self, logger, test_name, test): + """ Trigger archive of data files for a test or hook. """ + + with self._lock: + # Test repeat number is how many times the particular test has been archived. + if test_name not in self._tests_repeat: + self._tests_repeat[test_name] = 0 + else: + self._tests_repeat[test_name] += 1 + # Normalize test path from a test or hook name. + test_path = \ + test_name.replace("/", "_").replace("\\", "_").replace(".", "_").replace(":", "_") + file_name = "mongo-data-{}-{}-{}-{}.tgz".format( + config.EVERGREEN_TASK_ID, + test_path, + config.EVERGREEN_EXECUTION, + self._tests_repeat[test_name]) + # Retrieve root directory for all dbPaths from fixture. + input_files = test.fixture.get_dbpath_prefix() + s3_bucket = config.ARCHIVE_BUCKET + s3_path = "{}/{}/{}/datafiles/{}".format( + config.EVERGREEN_PROJECT_NAME, + config.EVERGREEN_VARIANT_NAME, + config.EVERGREEN_REVISION, + file_name) + display_name = "Data files {} - Execution {} Repetition {}".format( + test_name, + config.EVERGREEN_EXECUTION, + self._tests_repeat[test_name]) + logger.info("Archiving data files for test %s from %s", test_name, input_files) + status, message = self.archive_instance.archive_files_to_s3( + display_name, input_files, s3_bucket, s3_path) + if status: + logger.warning("Archive failed for %s: %s", test_name, message) diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py index 4d17152b17d..c8be906dd3a 100644 --- a/buildscripts/resmokelib/testing/job.py +++ b/buildscripts/resmokelib/testing/job.py @@ -18,7 +18,7 @@ class Job(object): Runs tests from a queue. """ - def __init__(self, logger, fixture, hooks, report): + def __init__(self, logger, fixture, hooks, report, archival): """ Initializes the job with the specified fixture and custom behaviors. @@ -28,6 +28,7 @@ class Job(object): self.fixture = fixture self.hooks = hooks self.report = report + self.archival = archival def __call__(self, queue, interrupt_flag, teardown_flag=None): """ @@ -98,20 +99,38 @@ class Job(object): self._run_hooks_before_tests(test) test(self.report) - if config.FAIL_FAST and not self.report.wasSuccessful(): - self.logger.info("%s failed, so stopping..." % (test.shortDescription())) - raise errors.StopExecution("%s failed" % (test.shortDescription())) - - if not self.fixture.is_running(): - self.logger.error("%s marked as a failure because the fixture crashed during the test.", - test.shortDescription()) - self.report.setFailure(test, return_code=2) - # Always fail fast if the fixture fails. - raise errors.StopExecution("%s not running after %s" % - (self.fixture, test.shortDescription())) + try: + if config.FAIL_FAST and not self.report.wasSuccessful(): + self.logger.info("%s failed, so stopping..." % (test.shortDescription())) + raise errors.StopExecution("%s failed" % (test.shortDescription())) + + if not self.fixture.is_running(): + self.logger.error( + "%s marked as a failure because the fixture crashed during the test.", + test.shortDescription()) + self.report.setFailure(test, return_code=2) + # Always fail fast if the fixture fails. + raise errors.StopExecution("%s not running after %s" % + (self.fixture, test.shortDescription())) + + finally: + success = self.report.find_test_info(test).status == "pass" + self.archival.archive(self.logger, test, success) + if self.archival: + self.archival.archive(self.logger, test, success) self._run_hooks_after_tests(test) + def _run_hook(self, hook, hook_function, test): + """Provide helper to run hook and archival.""" + try: + success = False + hook_function(test, self.report) + success = True + finally: + if self.archival: + self.archival.archive(self.logger, test, success, hook=hook) + def _run_hooks_before_tests(self, test): """ Runs the before_test method on each of the hooks. @@ -122,7 +141,7 @@ class Job(object): try: for hook in self.hooks: - hook.before_test(test, self.report) + self._run_hook(hook, hook.before_test, test) except errors.StopExecution: raise @@ -156,7 +175,7 @@ class Job(object): """ try: for hook in self.hooks: - hook.after_test(test, self.report) + self._run_hook(hook, hook.after_test, test) except errors.StopExecution: raise diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index b9cccccc14f..e3c68393a4d 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -147,7 +147,7 @@ class TestReport(unittest.TestResult): unittest.TestResult.stopTest(self, test) with self._lock: - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.end_time = time.time() time_taken = test_info.end_time - test_info.start_time @@ -177,7 +177,7 @@ class TestReport(unittest.TestResult): with self._lock: self.num_errored += 1 - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.status = "error" test_info.return_code = test.return_code @@ -187,7 +187,7 @@ class TestReport(unittest.TestResult): """ with self._lock: - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) if test_info.end_time is None: raise ValueError("stopTest was not called on %s" % (test.basename())) @@ -211,7 +211,7 @@ class TestReport(unittest.TestResult): with self._lock: self.num_failed += 1 - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.status = "fail" test_info.return_code = test.return_code @@ -221,7 +221,7 @@ class TestReport(unittest.TestResult): """ with self._lock: - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) if test_info.end_time is None: raise ValueError("stopTest was not called on %s" % (test.basename())) @@ -244,7 +244,7 @@ class TestReport(unittest.TestResult): with self._lock: self.num_succeeded += 1 - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.status = "pass" test_info.return_code = test.return_code @@ -352,7 +352,7 @@ class TestReport(unittest.TestResult): # protecting it with the lock. self.__original_loggers = {} - def _find_test_info(self, test): + def find_test_info(self, test): """ Returns the status and timing information associated with 'test'. diff --git a/buildscripts/resmokelib/testing/testcases.py b/buildscripts/resmokelib/testing/testcases.py index 21d35215a29..78110ff680d 100644 --- a/buildscripts/resmokelib/testing/testcases.py +++ b/buildscripts/resmokelib/testing/testcases.py @@ -6,7 +6,6 @@ from __future__ import absolute_import import os import os.path -import shutil import threading import unittest @@ -236,7 +235,7 @@ class DBTestCase(TestCase): dbpath = os.path.join(dbpath_prefix, "job%d" % (self.fixture.job_num), "unittest") self.dbtest_options["dbpath"] = dbpath - shutil.rmtree(dbpath, ignore_errors=True) + utils.rmtree(dbpath, ignore_errors=True) try: os.makedirs(dbpath) @@ -352,7 +351,7 @@ class JSTestCase(TestCase): global_vars["TestData"] = test_data self.shell_options["global_vars"] = global_vars - shutil.rmtree(data_dir, ignore_errors=True) + utils.rmtree(data_dir, ignore_errors=True) self.num_clients = num_clients @@ -364,14 +363,19 @@ class JSTestCase(TestCase): process_kwargs = self.shell_options.get("process_kwargs", {}).copy() - if "KRB5_CONFIG" in process_kwargs and "KRB5CCNAME" not in process_kwargs: + if process_kwargs \ + and "env_vars" in process_kwargs \ + and "KRB5_CONFIG" in process_kwargs["env_vars"] \ + and "KRB5CCNAME" not in process_kwargs["env_vars"]: # Use a job-specific credential cache for JavaScript tests involving Kerberos. krb5_dir = os.path.join(data_dir, "krb5") + try: os.makedirs(krb5_dir) except os.error: pass - process_kwargs["KRB5CCNAME"] = "DIR:" + os.path.join(krb5_dir, ".") + + process_kwargs["env_vars"]["KRB5CCNAME"] = "DIR:" + krb5_dir self.shell_options["process_kwargs"] = process_kwargs diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py index df387cc3323..ff37fbb954a 100644 --- a/buildscripts/resmokelib/utils/__init__.py +++ b/buildscripts/resmokelib/utils/__init__.py @@ -5,15 +5,41 @@ Helper functions. from __future__ import absolute_import import os.path +import shutil +import sys import pymongo import yaml +from . import archival + def default_if_none(value, default): return value if value is not None else default +def rmtree(path, **kwargs): + """Wrap shutil.rmtreee. + + Use a UTF-8 unicode path if Windows. + See https://bugs.python.org/issue24672, where shutil.rmtree can fail with UTF-8. + Use a bytes path to rmtree, otherwise. + See https://github.com/pypa/setuptools/issues/706. + """ + if is_windows(): + if not isinstance(path, unicode): + path = unicode(path, "utf-8") + else: + if isinstance(path, unicode): + path = path.encode("utf-8") + shutil.rmtree(path, **kwargs) + + +def is_windows(): + """Return True if Windows.""" + return sys.platform.startswith("win32") or sys.platform.startswith("cygwin") + + def is_string_list(lst): """ Returns true if 'lst' is a list of strings, and false otherwise. diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py new file mode 100644 index 00000000000..1002f2ce902 --- /dev/null +++ b/buildscripts/resmokelib/utils/archival.py @@ -0,0 +1,331 @@ +""" +Archival utility. +""" + +from __future__ import absolute_import + +import Queue +import collections +import json +import math +import os +import sys +import tarfile +import tempfile +import threading +import time + +_IS_WINDOWS = sys.platform == "win32" or sys.platform == "cygwin" + +if _IS_WINDOWS: + import ctypes + +UploadArgs = collections.namedtuple( + "UploadArgs", + ["archival_file", + "display_name", + "local_file", + "content_type", + "s3_bucket", + "s3_path", + "delete_file"]) + +ArchiveArgs = collections.namedtuple( + "ArchiveArgs", ["archival_file", "display_name", "remote_file"]) + + +def file_list_size(files): + """ Return size (in bytes) of all 'files' and their subdirectories. """ + if isinstance(files, str): + files = [files] + file_bytes = 0 + for ifile in files: + if not os.path.exists(ifile): + pass + elif os.path.isdir(ifile): + file_bytes += directory_size(ifile) + else: + file_bytes += os.path.getsize(ifile) + return file_bytes + + +def directory_size(directory): + """ Return size (in bytes) of files in 'directory' tree. """ + dir_bytes = 0 + for root_dir, _, files in os.walk(unicode(directory)): + for name in files: + full_name = os.path.join(root_dir, name) + try: + dir_bytes += os.path.getsize(full_name) + except OSError: + # A file might be deleted while we are looping through the os.walk() result. + pass + return dir_bytes + + +def free_space(path): + """ Return file system free space (in bytes) for 'path'. """ + if _IS_WINDOWS: + dirname = os.path.dirname(path) + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW( + ctypes.c_wchar_p(dirname), None, None, ctypes.pointer(free_bytes)) + return free_bytes.value + else: + stat = os.statvfs(path) + return stat.f_bavail * stat.f_bsize + + +def remove_file(file_name): + """ Attempts to remove file. Returns status and message. """ + try: + # File descriptors, on Windows, are inherited by all subprocesses and file removal may fail + # because the file is still open. + # See https://www.python.org/dev/peps/pep-0446/#issues-with-inheritable-file-descriptors + os.remove(file_name) + status = 0 + message = "Successfully deleted file {}".format(file_name) + except Exception as err: + status = 1 + message = "Error deleting file {}: {}".format(file_name, err) + return status, message + + +class Archival(object): + """ Class to support file archival to S3.""" + + def __init__(self, + logger, + archival_json_file="archive.json", + limit_size_mb=0, + limit_files=0, + s3_client=None): + """ Archival init method. """ + + self.archival_json_file = archival_json_file + self.limit_size_mb = limit_size_mb + self.limit_files = limit_files + self.size_mb = 0 + self.num_files = 0 + self.archive_time = 0 + self.logger = logger + + # Lock to control access from multiple threads. + self._lock = threading.Lock() + + # Start the worker thread to update the 'archival_json_file'. + self._archive_file_queue = Queue.Queue() + self._archive_file_worker = threading.Thread( + target=self._update_archive_file_wkr, + args=(self._archive_file_queue, logger), + name="archive_file_worker") + self._archive_file_worker.setDaemon(True) + self._archive_file_worker.start() + if not s3_client: + self.s3_client = self._get_s3_client() + else: + self.s3_client = s3_client + + # Start the worker thread which uploads the archive. + self._upload_queue = Queue.Queue() + self._upload_worker = threading.Thread( + target=self._upload_to_s3_wkr, + args=(self._upload_queue, self._archive_file_queue, logger, self.s3_client), + name="upload_worker") + self._upload_worker.setDaemon(True) + self._upload_worker.start() + + @staticmethod + def _get_s3_client(): + # Since boto3 is a 3rd party module, we import locally. + import boto3 + return boto3.client("s3") + + def archive_files_to_s3(self, display_name, input_files, s3_bucket, s3_path): + """ + Archive 'input_files' to 's3_bucket' and 's3_path'. + + Archive is not done if user specified limits are reached. The size limit is + enforced after it has been exceeded, since it can only be calculated after the + tar/gzip has been done. + + Returns status and message, where message contains information if status is non-0. + """ + + start_time = time.time() + with self._lock: + if not input_files: + status = 1 + message = "No input_files specified" + elif self.limit_size_mb and self.size_mb >= self.limit_size_mb: + status = 1 + message = "Files not archived, {}MB size limit reached".format(self.limit_size_mb) + elif self.limit_files and self.num_files >= self.limit_files: + status = 1 + message = "Files not archived, {} file limit reached".format(self.limit_files) + else: + status, message, file_size_mb = self._archive_files( + display_name, + input_files, + s3_bucket, + s3_path) + + if status == 0: + self.num_files += 1 + self.size_mb += file_size_mb + self.archive_time += time.time() - start_time + + return status, message + + @staticmethod + def _update_archive_file_wkr(queue, logger): + """ Worker thread: Update the archival JSON file from 'queue'. """ + archival_json = [] + while True: + archive_args = queue.get() + # Exit worker thread when sentinel is received. + if archive_args is None: + queue.task_done() + break + archival_record = { + "name": archive_args.display_name, + "link": archive_args.remote_file, + "visibility": "private" + } + logger.debug( + "Updating archive file %s with %s", archive_args.archival_file, archival_record) + archival_json.append(archival_record) + with open(archive_args.archival_file, "w") as archival_fh: + json.dump(archival_json, archival_fh) + queue.task_done() + + @staticmethod + def _upload_to_s3_wkr(queue, archive_file_queue, logger, s3_client): + """" Worker thread: Upload to S3 from 'queue', dispatch to 'archive_file_queue'. """ + while True: + upload_args = queue.get() + # Exit worker thread when sentinel is received. + if upload_args is None: + queue.task_done() + archive_file_queue.put(None) + break + extra_args = {"ContentType": upload_args.content_type, "ACL": "public-read"} + logger.debug("Uploading to S3 %s to bucket %s path %s", + upload_args.local_file, + upload_args.s3_bucket, + upload_args.s3_path) + upload_completed = False + try: + s3_client.upload_file(upload_args.local_file, + upload_args.s3_bucket, + upload_args.s3_path, + ExtraArgs=extra_args) + upload_completed = True + logger.debug("Upload to S3 completed for %s to bucket %s path %s", + upload_args.local_file, + upload_args.s3_bucket, + upload_args.s3_path) + except Exception as err: + logger.exception("Upload to S3 error %s", err) + + if upload_args.delete_file: + status, message = remove_file(upload_args.local_file) + if status: + logger.error("Upload to S3 delete file error %s", message) + + remote_file = "https://s3.amazonaws.com/{}/{}".format( + upload_args.s3_bucket, upload_args.s3_path) + if upload_completed: + archive_file_queue.put(ArchiveArgs( + upload_args.archival_file, upload_args.display_name, remote_file)) + + queue.task_done() + + def _archive_files(self, display_name, input_files, s3_bucket, s3_path): + """ + Gather 'input_files' into a single tar/gzip and archive to 's3_path'. + + The caller waits until the list of files has been tar/gzipped to a temporary file. + The S3 upload and subsequent update to 'archival_json_file' will be done asynchronosly. + + Returns status, message and size_mb of archive. + """ + + # Parameter 'input_files' can either be a string or list of strings. + if isinstance(input_files, str): + input_files = [input_files] + + message = "Tar/gzip {} files: {}".format(display_name, input_files) + status = 0 + size_mb = 0 + + # Tar/gzip to a temporary file. + _, temp_file = tempfile.mkstemp(suffix=".tgz") + + # Check if there is sufficient space for the temporary tgz file. + if file_list_size(input_files) > free_space(temp_file): + status, message = remove_file(temp_file) + if status: + self.logger.warning("Removing tarfile due to insufficient space - %s", message) + return 1, "Insufficient space for {}".format(message), 0 + + try: + with tarfile.open(temp_file, "w:gz") as tar_handle: + for input_file in input_files: + try: + tar_handle.add(input_file) + except (IOError, OSError, tarfile.TarError) as err: + message = "{}; Unable to add {} to archive file: {}".format( + message, input_file, err) + except (IOError, OSError, tarfile.TarError) as err: + status, message = remove_file(temp_file) + if status: + self.logger.warning("Removing tarfile due to creation failure - %s", message) + return 1, str(err), 0 + + # Round up the size of the archive. + size_mb = int(math.ceil(float(file_list_size(temp_file)) / (1024 * 1024))) + self._upload_queue.put(UploadArgs( + self.archival_json_file, + display_name, + temp_file, + "application/x-gzip", + s3_bucket, + s3_path, + True)) + + return status, message, size_mb + + def check_thread(self, thread, expected_alive): + if thread.isAlive() and not expected_alive: + self.logger.warning( + "The %s thread did not complete, some files might not have been uploaded" + " to S3 or archived to %s.", thread.name, self.archival_json_file) + elif not thread.isAlive() and expected_alive: + self.logger.warning( + "The %s thread is no longer running, some files might not have been uploaded" + " to S3 or archived to %s.", thread.name, self.archival_json_file) + + def exit(self, timeout=30): + """ Waits for worker threads to finish. """ + # Put sentinel on upload queue to trigger worker thread exit. + self._upload_queue.put(None) + self.check_thread(self._upload_worker, True) + self.check_thread(self._archive_file_worker, True) + self._upload_worker.join(timeout=timeout) + self.check_thread(self._upload_worker, False) + + # Archive file worker thread exit should be triggered by upload thread worker. + self._archive_file_worker.join(timeout=timeout) + self.check_thread(self._archive_file_worker, False) + + self.logger.info("Total tar/gzip archive time is %0.2f seconds, for %d file(s) %d MB", + self.archive_time, self.num_files, self.size_mb) + + def files_archived_num(self): + """ Returns the number of the archived files. """ + return self.num_files + + def files_archived_size_mb(self): + """ Returns the size of the archived files. """ + return self.size_mb diff --git a/buildscripts/tests/resmokelib/test_archival.py b/buildscripts/tests/resmokelib/test_archival.py new file mode 100644 index 00000000000..bc36cdb4832 --- /dev/null +++ b/buildscripts/tests/resmokelib/test_archival.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python + +""" Unit tests for archival. """ + +import logging +import os +import random +import shutil +import sys +import tempfile +import unittest + + +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.getcwd()) +from buildscripts.resmokelib.utils import archival + +_BUCKET = "mongodatafiles" + + +def create_random_file(file_name, num_chars_mb): + """ Creates file with random characters, which will have minimal compression. """ + with open(file_name, "wb") as fileh: + for _ in xrange(num_chars_mb * 1024 * 1024): + fileh.write(chr(random.randint(0, 255))) + + +class MockS3Client(object): + """ Class to mock the S3 client. """ + + def __init__(self, logger): + self.logger = logger + self.logger.info("MockS3Client init") + return + + def upload_file(self, *args, **kwargs): + self.logger.info("MockS3Client upload_file %s %s", args, kwargs) + return + + def delete_object(self, *args, **kwargs): + self.logger.info("MockS3Client delete_object %s %s", args, kwargs) + return + + +class ArchivalTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + logging.basicConfig() + cls.logger = logging.getLogger() + cls.logger.setLevel(logging.INFO) + cls.bucket = _BUCKET + cls.temp_dir = tempfile.mkdtemp() + cls.remote_paths = [] + + # We can use the AWS S3 calls by setting the environment variable MOCK_CLIENT to 0. + mock_client = os.environ.get("MOCK_CLIENT", "1") != "0" + if mock_client: + cls.s3_client = MockS3Client(cls.logger) + else: + cls.s3_client = archival.Archival._get_s3_client() + cls.archive = cls.create_archival() + + @classmethod + def tearDownClass(cls): + cls.archive.exit() + cls.logger.info("Cleaning temp directory %s", cls.temp_dir) + shutil.rmtree(cls.temp_dir, ignore_errors=True) + for path in cls.remote_paths: + cls.delete_s3_file(path) + + @classmethod + def create_archival(cls): + return archival.Archival(cls.logger, s3_client=cls.s3_client) + + @classmethod + def delete_s3_file(cls, path): + cls.logger.info("Cleaning S3 bucket %s path %s", cls.bucket, path) + cls.s3_client.delete_object(Bucket=cls.bucket, Key=path) + + def s3_path(self, path, add_remote_path=True): + if add_remote_path: + self.remote_paths.append(path) + return path + + +class ArchivalFileTests(ArchivalTestCase): + def test_nofile(self): + # Invalid input_files + display_name = "Unittest invalid file" + input_files = "no_file" + s3_path = self.s3_path("unittest/no_file.tgz", False) + self.assertRaises( + OSError, + lambda: self.archive.archive_files_to_s3( + display_name, input_files, self.bucket, s3_path)) + + # Invalid input_files in a list + input_files = ["no_file", "no_file2"] + s3_path = self.s3_path("unittest/no_files.tgz", False) + self.assertRaises( + OSError, + lambda: self.archive.archive_files_to_s3( + display_name, input_files, self.bucket, s3_path)) + + # No files + display_name = "Unittest no files" + s3_path = self.s3_path("unittest/no_files.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, [], self.bucket, s3_path) + self.assertEqual(1, status, message) + + def test_files(self): + # Valid files + display_name = "Unittest valid file" + temp_file = tempfile.mkstemp(dir=self.temp_dir)[1] + s3_path = self.s3_path("unittest/valid_file.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + + # 2 valid files + display_name = "Unittest 2 valid files" + temp_file2 = tempfile.mkstemp(dir=self.temp_dir)[1] + s3_path = self.s3_path("unittest/2valid_files.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, [temp_file, temp_file2], self.bucket, s3_path) + self.assertEqual(0, status, message) + + def test_empty_directory(self): + # Valid directory + display_name = "Unittest valid directory no files" + temp_dir = tempfile.mkdtemp(dir=self.temp_dir) + s3_path = self.s3_path("unittest/valid_directory.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_dir, self.bucket, s3_path) + self.assertEqual(0, status, message) + + display_name = "Unittest valid directories no files" + temp_dir2 = tempfile.mkdtemp(dir=self.temp_dir) + s3_path = self.s3_path("unittest/valid_directories.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, [temp_dir, temp_dir2], self.bucket, s3_path) + self.assertEqual(0, status, message) + + def test_directory(self): + display_name = "Unittest directory with files" + temp_dir = tempfile.mkdtemp(dir=self.temp_dir) + s3_path = self.s3_path("unittest/directory_with_files.tgz") + # Create 10 empty files + for _ in xrange(10): + tempfile.mkstemp(dir=temp_dir) + status, message = self.archive.archive_files_to_s3( + display_name, temp_dir, self.bucket, s3_path) + self.assertEqual(0, status, message) + + display_name = "Unittest 2 valid directory files" + temp_dir2 = tempfile.mkdtemp(dir=self.temp_dir) + s3_path = self.s3_path("unittest/directories_with_files.tgz") + # Create 10 empty files + for _ in xrange(10): + tempfile.mkstemp(dir=temp_dir2) + status, message = self.archive.archive_files_to_s3( + display_name, [temp_dir, temp_dir2], self.bucket, s3_path) + self.assertEqual(0, status, message) + + +class ArchivalLimitSizeTests(ArchivalTestCase): + @classmethod + def create_archival(cls): + return archival.Archival(cls.logger, limit_size_mb=5, s3_client=cls.s3_client) + + def test_limit_size(self): + + # Files within limit size + display_name = "Unittest under limit size" + temp_file = tempfile.mkstemp(dir=self.temp_dir)[1] + create_random_file(temp_file, 3) + s3_path = self.s3_path("unittest/valid_limit_size.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + # Note the size limit is enforced after the file uploaded. Subsequent + # uploads will not be permitted, once the limit has been reached. + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + + # Files beyond limit size + display_name = "Unittest over limit size" + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(1, status, message) + + +class ArchivalLimitFileTests(ArchivalTestCase): + @classmethod + def create_archival(cls): + return archival.Archival(cls.logger, limit_files=3, s3_client=cls.s3_client) + + def test_limit_file(self): + + # Files within limit number + display_name = "Unittest under limit number" + temp_file = tempfile.mkstemp(dir=self.temp_dir)[1] + s3_path = self.s3_path("unittest/valid_limit_number.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + + # Files beyond limit number + display_name = "Unittest over limit number" + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(1, status, message) + + +class ArchivalLimitTests(ArchivalTestCase): + @classmethod + def create_archival(cls): + return archival.Archival( + cls.logger, limit_size_mb=3, limit_files=3, s3_client=cls.s3_client) + + def test_limits(self): + + # Files within limits + display_name = "Unittest under limits" + temp_file = tempfile.mkstemp(dir=self.temp_dir)[1] + create_random_file(temp_file, 1) + s3_path = self.s3_path("unittest/valid_limits.tgz") + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(0, status, message) + + # Files beyond limits + display_name = "Unittest over limits" + status, message = self.archive.archive_files_to_s3( + display_name, temp_file, self.bucket, s3_path) + self.assertEqual(1, status, message) + + +if __name__ == "__main__": + unittest.main() diff --git a/buildscripts/tests/resmokelib/utils/__init__.py b/buildscripts/tests/resmokelib/utils/__init__.py new file mode 100644 index 00000000000..4b7a2bb941b --- /dev/null +++ b/buildscripts/tests/resmokelib/utils/__init__.py @@ -0,0 +1 @@ +"""Empty.""" diff --git a/buildscripts/tests/resmokelib/utils/test_rmtree.py b/buildscripts/tests/resmokelib/utils/test_rmtree.py new file mode 100644 index 00000000000..1908395766c --- /dev/null +++ b/buildscripts/tests/resmokelib/utils/test_rmtree.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +""" Unit tests for utils.rmtree. """ + +from __future__ import absolute_import +from __future__ import print_function + +import os +import shutil +import sys +import tempfile +import unittest + +from buildscripts.resmokelib import utils + +# pylint: disable=missing-docstring,protected-access + + +def rmtree(dir_root): + """Invoke utils.rmtree(dir_root) and return True if removed.""" + utils.rmtree(dir_root) + return not os.path.exists(dir_root) + + +def create_file(path): + """Create file named 'path'.""" + with open(path, "w") as fh: + fh.write("") + + +def ascii_filesystemencoding(): + """Return True if the file system encoding is type ASCII. + + See https://www.iana.org/assignments/character-sets/character-sets.xhtml. + """ + encoding = sys.getfilesystemencoding() + return encoding.startswith("ANSI_X3.4") or encoding == "US-ASCII" + + +def string_for_ascii_filesystem_encoding(path): + """Return encoded string type for unicode on ASCII file system encoding. + + Some file system encodings are set to ASCII if LANG=C or LC_ALL=C is specified. + """ + if ascii_filesystemencoding() and isinstance(path, unicode): + return path.encode("utf-8") + return path + + +class RmtreeTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp_dir_root = tempfile.mkdtemp() + cls.cwd = os.getcwd() + + @classmethod + def tearDownClass(cls): + os.chdir(cls.cwd) + shutil.rmtree(cls.temp_dir_root, ignore_errors=True) + + def do_test(self, name): + pass + + def test_ascii(self): + # ASCII name + self.do_test("ascii") + + def test_unicode(self): + # Unicode name + self.do_test(u"unicode") + + def test_greek(self): + # Name with Greek + self.do_test(string_for_ascii_filesystem_encoding(u"ελληνικά")) + + def test_japanese(self): + # Name with Japanese + self.do_test(string_for_ascii_filesystem_encoding(u"会社案")) + + +class RmtreeFileTests(RmtreeTestCase): + def do_test(self, file_name): # pylint: disable=arguments-differ + """Execute file test.""" + temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root) + os.chdir(temp_dir) + create_file(file_name) + os.chdir(self.temp_dir_root) + self.assertTrue(rmtree(temp_dir)) + + +class RmtreeDirectoryTests(RmtreeTestCase): + def do_test(self, dir_name): # pylint: disable=arguments-differ + """Execute directory test.""" + os.chdir(self.temp_dir_root) + os.mkdir(dir_name) + self.assertTrue(rmtree(dir_name)) + + +class RmtreeDirectoryWithNonAsciiTests(RmtreeTestCase): + def do_test(self, name): + """Execute directory with non-ASCII file test.""" + os.chdir(self.temp_dir_root) + os.mkdir(name) + os.chdir(name) + create_file(name) + os.chdir(self.temp_dir_root) + self.assertTrue(rmtree(name)) + + +class ShutilWindowsRmtreeFileTests(RmtreeFileTests): + def do_test(self, file_name): + """Execute file test that are known to fail in shutil.rmtree.""" + if not utils.is_windows(): + print("Skipping ShutilWindowsRmtreeFileTests on non-Windows platforms") + return + temp_dir = tempfile.mkdtemp(dir=self.temp_dir_root) + os.chdir(temp_dir) + create_file(file_name) + os.chdir(self.temp_dir_root) + with self.assertRaises(WindowsError): # pylint: disable=undefined-variable + shutil.rmtree(temp_dir) + + def test_ascii(self): + pass + + def test_unicode(self): + pass |