summaryrefslogtreecommitdiff
path: root/buildscripts
diff options
context:
space:
mode:
authorApollon Oikonomopoulos <apoikos@debian.org>2018-03-22 12:04:55 +0200
committerApollon Oikonomopoulos <apoikos@debian.org>2018-03-22 12:04:55 +0200
commitc49e99631589113663b1a3ac691870421965a315 (patch)
treeac8127a5a6816f169a6656511bf131a35af2f74a /buildscripts
parentd982a88efa79f510c03f1c6c8c63360680ebbf88 (diff)
New upstream version 3.4.14upstream/3.4.14
Diffstat (limited to 'buildscripts')
-rw-r--r--buildscripts/burn_in_tests.py5
-rwxr-xr-xbuildscripts/hang_analyzer.py54
-rw-r--r--buildscripts/package_test/.kitchen.yml4
-rw-r--r--buildscripts/package_test/recipes/install_mongodb.rb10
-rw-r--r--buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml39
-rw-r--r--buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml99
-rw-r--r--buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml1
-rw-r--r--buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml1
-rw-r--r--buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml1
-rw-r--r--buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml2
-rw-r--r--buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml78
-rw-r--r--buildscripts/resmokelib/core/programs.py17
-rw-r--r--buildscripts/resmokelib/logging/buildlogger.py152
-rw-r--r--buildscripts/resmokelib/logging/config.py8
-rw-r--r--buildscripts/resmokelib/testing/fixtures/interface.py13
-rw-r--r--buildscripts/resmokelib/testing/fixtures/masterslave.py9
-rw-r--r--buildscripts/resmokelib/testing/fixtures/replicaset.py32
-rw-r--r--buildscripts/resmokelib/testing/fixtures/shardedcluster.py33
-rw-r--r--buildscripts/resmokelib/testing/fixtures/standalone.py10
-rw-r--r--buildscripts/resmokelib/testing/testcases.py27
-rwxr-xr-xbuildscripts/scons.py28
-rw-r--r--buildscripts/tests/resmokelib/logging/test_buildlogger.py67
22 files changed, 501 insertions, 189 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index d171a5a6bc1..6e39685ef8b 100644
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -17,7 +17,7 @@ import sys
import urlparse
import yaml
-API_SERVER_DEFAULT = "http://evergreen-api.mongodb.com:8080"
+API_SERVER_DEFAULT = "https://evergreen.mongodb.com"
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
@@ -418,6 +418,8 @@ def main():
tests_by_task = _load_tests_file(values.test_list_file)
# If there are no tests to run, carry on.
if tests_by_task is None:
+ test_results = {"failures": 0, "results": []}
+ _write_report_file(test_results, values.report_file)
sys.exit(0)
# Run the executor finder.
@@ -438,6 +440,7 @@ def main():
# If there are no changed tests, exit cleanly.
if not changed_tests:
print "No new or modified tests found."
+ _write_report_file({}, values.test_list_outfile)
sys.exit(0)
suites = resmokelib.parser.get_suites(values, changed_tests)
tests_by_executor = create_executor_list(suites, exclude_suites)
diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py
index 73e1cbe6d41..09245904a7f 100755
--- a/buildscripts/hang_analyzer.py
+++ b/buildscripts/hang_analyzer.py
@@ -24,6 +24,7 @@ import signal
import subprocess
import sys
import tempfile
+import traceback
import time
from distutils import spawn
from optparse import OptionParser
@@ -53,7 +54,7 @@ def call(a, logger):
if ret != 0:
logger.error("Bad exit code %d" % (ret))
- raise Exception()
+ raise Exception("Bad exit code %d from %s" % (ret, " ".join(a)))
# Copied from python 2.7 version of subprocess.py
@@ -375,14 +376,14 @@ class GDBDumper(object):
base, ext = os.path.splitext(logger.mongo_process_filename)
raw_stacks_filename = base + '_raw_stacks' + ext
raw_stacks_commands = [
- 'echo \\nWriting raw stacks to %s.\\n' % raw_stacks_filename,
- # This sends output to log file rather than stdout until we turn logging off.
- 'set logging redirect on',
- 'set logging file ' + raw_stacks_filename,
- 'set logging on',
- 'thread apply all bt',
- 'set logging off',
- ]
+ 'echo \\nWriting raw stacks to %s.\\n' % raw_stacks_filename,
+ # This sends output to log file rather than stdout until we turn logging off.
+ 'set logging redirect on',
+ 'set logging file ' + raw_stacks_filename,
+ 'set logging on',
+ 'thread apply all bt',
+ 'set logging off',
+ ]
cmds = [
"set interactive-mode off",
@@ -732,28 +733,38 @@ def main():
# a signal handler to wait for the signal since it supports POSIX signals.
if _is_windows:
root_logger.info("Calling SetEvent to signal python process %s with PID %d" %
- (process_name, pid))
+ (process_name, pid))
signal_event_object(root_logger, pid)
else:
root_logger.info("Sending signal SIGUSR1 to python process %s with PID %d" %
- (process_name, pid))
+ (process_name, pid))
signal_process(root_logger, pid, signal.SIGUSR1)
+ trapped_exceptions = []
+
# Dump all processes, except python & java.
for (pid, process_name) in [(p, pn) for (p, pn) in processes
if not re.match("^(java|python)", pn)]:
process_logger = get_process_logger(options.debugger_output, pid, process_name)
- dbg.dump_info(
- root_logger,
- process_logger,
- pid,
- process_name,
- options.dump_core and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext()))
+ try:
+ dbg.dump_info(
+ root_logger,
+ process_logger,
+ pid,
+ process_name,
+ options.dump_core and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext()))
+ except Exception as err:
+ root_logger.info("Error encountered when invoking debugger %s" % err)
+ trapped_exceptions.append(traceback.format_exc())
# Dump java processes using jstack.
for (pid, process_name) in [(p, pn) for (p, pn) in processes if pn.startswith("java")]:
process_logger = get_process_logger(options.debugger_output, pid, process_name)
- jstack.dump_info(root_logger, process_logger, pid, process_name)
+ try:
+ jstack.dump_info(root_logger, process_logger, pid, process_name)
+ except Exception as err:
+ root_logger.info("Error encountered when invoking debugger %s" % err)
+ trapped_exceptions.append(traceback.format_exc())
# Signal go processes to ensure they print out stack traces, and die on POSIX OSes.
# On Windows, this will simply kill the process since python emulates SIGABRT as
@@ -761,10 +772,15 @@ def main():
# Note: The stacktrace output may be captured elsewhere (i.e. resmoke).
for (pid, process_name) in [(p, pn) for (p, pn) in processes if pn in go_processes]:
root_logger.info("Sending signal SIGABRT to go process %s with PID %d" %
- (process_name, pid))
+ (process_name, pid))
signal_process(root_logger, pid, signal.SIGABRT)
root_logger.info("Done analyzing all processes for hangs")
+ for exception in trapped_exceptions:
+ root_logger.info(exception)
+ if trapped_exceptions:
+ sys.exit(1)
+
if __name__ == "__main__":
main()
diff --git a/buildscripts/package_test/.kitchen.yml b/buildscripts/package_test/.kitchen.yml
index e9b6989d221..3b4a476551c 100644
--- a/buildscripts/package_test/.kitchen.yml
+++ b/buildscripts/package_test/.kitchen.yml
@@ -2,11 +2,13 @@
driver:
name: ec2
region: us-east-1
+ vpc_mode: true
+ vpc_id: <%= ENV['KITCHEN_VPC'] %>
subnet_id: <%= ENV['KITCHEN_SUBNET'] %>
security_group_ids:
- <%= ENV['KITCHEN_SECURITY_GROUP'] %>
aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %>
- interface: dns
+ interface: private
associate_public_ip: true
verifier:
diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb
index dcf7499289e..9b4ea703b2b 100644
--- a/buildscripts/package_test/recipes/install_mongodb.rb
+++ b/buildscripts/package_test/recipes/install_mongodb.rb
@@ -84,6 +84,16 @@ if platform_family? 'suse'
EOD
end
+ %w(
+ SLES12-Pool
+ SLES12-Updates
+ ).each do |repo|
+ execute "add #{repo}" do
+ command "zypper addrepo --check --refresh --name \"#{repo}\" http://smt-ec2.susecloud.net/repo/SUSE/Products/SLE-SERVER/12/x86_64/product?credentials=SMT-http_smt-ec2_susecloud_net 'SMT-http_smt-ec2_susecloud_net:#{repo}'"
+ not_if "zypper lr | grep #{repo}"
+ end
+ end
+
execute 'install mongod' do
command 'zypper -n install `find . -name "*server*.rpm"`'
cwd homedir
diff --git a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml
index bc227b09279..72b838b3c07 100644
--- a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml
@@ -6,11 +6,28 @@ selector:
- jstests/aggregation/expressions/*.js
- jstests/aggregation/sources/*/*.js
exclude_files:
- - jstests/aggregation/bugs/server18198.js # Uses a mocked mongo client to test read preference.
- - jstests/aggregation/mongos_slaveok.js # Majority read on secondary requires afterOpTime.
- - jstests/aggregation/sources/facet/use_cases.js # Cannot specify write concern when
- # secondaryThrottle is not set.
- - jstests/aggregation/testSlave.js # Majority read on secondary requires afterOpTime.
+ # These tests fail due to the inability to specify a writeConcern when secondaryThrottle is not
+ # set as part of the moveChunk command.
+ - jstests/aggregation/sources/facet/use_cases.js
+ # These test fail because afterOpTime is required to guarantee a secondary has advanced its
+ # majority-committed snapshot.
+ - jstests/aggregation/mongos_slaveok.js
+ - jstests/aggregation/testSlave.js
+ exclude_with_any_tags:
+ ##
+ # The next three tags correspond to the special errors thrown by the
+ # set_read_and_write_concerns.js override when it refuses to replace the readConcern or
+ # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to
+ # be warranted.
+ ##
+ # "Cowardly refusing to override read concern of command: ..."
+ - assumes_read_concern_unchanged
+ # "Cowardly refusing to override write concern of command: ..."
+ - assumes_write_concern_unchanged
+ # "Cowardly refusing to run test with overridden write concern when it uses a command that can
+ # only perform w=1 writes: ..."
+ - requires_collmod_command
+ - requires_eval_command
executor:
js_test:
@@ -18,13 +35,19 @@ executor:
shell_options:
global_vars:
TestData:
+ defaultReadConcernLevel: majority
enableMajorityReadConcern: ''
- eval: "var testingReplication = true; load('jstests/libs/override_methods/set_majority_read_and_write_concerns.js');"
+ eval: >-
+ var testingReplication = true;
+ load('jstests/libs/override_methods/set_read_and_write_concerns.js');
readMode: commands
hooks:
- - class: ValidateCollections
+ # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
+ # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
+ # validating the entire contents of the collection.
- class: CheckReplOplogs
- class: CheckReplDBHash
+ - class: ValidateCollections
fixture:
class: ReplicaSetFixture
mongod_options:
@@ -33,5 +56,3 @@ executor:
enableTestCommands: 1
numInitialSyncAttempts: 1
num_nodes: 2
- # Needs to be set for any ephemeral or no-journaling storage engine
- write_concern_majority_journal_default: false
diff --git a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml
index f47e424e1cf..22e84bc9c63 100644
--- a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml
@@ -3,79 +3,47 @@ selector:
roots:
- jstests/core/**/*.js
exclude_files:
- # Tests that won't work with an injected 'majority' readConcern
- # and/or an injected 'majority' writeConcern. Where a function is
- # listed the reason is we don't have a reliable solution to override
- # the write concern for that function.
- - jstests/core/batch_write_command*.js # these tests use various write concerns
- - jstests/core/bench_test*.js # benchRun() used for writes
- - jstests/core/capped_update.js # uses godinsert and can't run under replication.
- - jstests/core/crud_api.js # has specific w:0 tests
- - jstests/core/error2.js # db.eval() used
- - jstests/core/eval0.js # db.eval() used
- - jstests/core/eval1.js # db.eval() used
- - jstests/core/eval3.js # db.eval() used
- - jstests/core/eval4.js # db.eval() used
- - jstests/core/eval5.js # db.eval() used
- - jstests/core/eval6.js # db.eval() used
- - jstests/core/eval7.js # db.eval() used
- - jstests/core/eval9.js # db.eval() used
- - jstests/core/evala.js # db.eval() used
- - jstests/core/evalb.js # db.eval() used
- - jstests/core/evald.js # db.eval() used
- - jstests/core/evale.js # db.eval() used
- - jstests/core/evalg.js # db.eval() used
- - jstests/core/eval_mr.js # db.eval() used
- - jstests/core/eval_nolock.js # db.eval() used
- - jstests/core/geo_s2cursorlimitskip.js # drops system.profile collection and counts ops.
- - jstests/core/js3.js # db.dbEval() used
- - jstests/core/js7.js # db.eval() used
- - jstests/core/js9.js # db.eval() used
- - jstests/core/mr_merge.js # mr temp tables aren't replicated
- - jstests/core/mr_merge2.js # mr temp tables aren't replicated
- - jstests/core/mr_outreduce.js # mr temp tables aren't replicated
- - jstests/core/mr_outreduce2.js # mr temp tables aren't replicated
- - jstests/core/opcounters_active.js # off by n problem with opcounters
- - jstests/core/opcounters_write_cmd.js # off by n problem with opcounters
- - jstests/core/profile_agg.js # system.profile not replicated
- - jstests/core/profile_count.js # system.profile not replicated
- - jstests/core/profile_delete.js # system.profile not replicated
- - jstests/core/profile_distinct.js # system.profile not replicated
- - jstests/core/profile_find.js # system.profile not replicated
- - jstests/core/profile_findandmodify.js # system.profile not replicated
- - jstests/core/profile_geonear.js # system.profile not replicated
- - jstests/core/profile_getmore.js # system.profile not replicated
- - jstests/core/profile_group.js # system.profile not replicated
- - jstests/core/profile_insert.js # system.profile not replicated
- - jstests/core/profile_mapreduce.js # system.profile not replicated
- - jstests/core/profile_update.js # system.profile not replicated
- - jstests/core/profile1.js # system.profile not replicated
- - jstests/core/profile2.js # system.profile not replicated
- - jstests/core/profile3.js # system.profile not replicated
- - jstests/core/read_after_optime.js # verifies read after optime fails on standalone
- - jstests/core/remove8.js # db.eval() used
- - jstests/core/rename4.js # db.eval() used
- - jstests/core/shell1.js # tests setSlaveOk() variations on standalone mongod
- - jstests/core/shellkillop.js # db.eval() used
- - jstests/core/shell_writeconcern.js # checks write concern shell helpers
- - jstests/core/storefunc.js # db.eval() used
- - jstests/core/write_result.js # Tests invalid writeConcern, we shouldn't override.
- # Tests that need triaging & remediation | blacklist decision
- # Comments list possible problem point under review.
- - jstests/core/capped6.js # Uses captrunc test command.
- - jstests/core/convert_to_capped_nonexistant.js # Uses convertToCapped and captrunc command.
- - jstests/core/stages_delete.js # Uses stageDebug command for deletes.
+ # These tests are not expected to pass with replica-sets:
+ - jstests/core/dbadmin.js
+ - jstests/core/opcounters_write_cmd.js
+ - jstests/core/read_after_optime.js
+ - jstests/core/capped_update.js
+ # These tests use benchRun(), which isn't configured to use the overridden writeConcern.
+ - jstests/core/bench_test*.js
+ exclude_with_any_tags:
+ ##
+ # The next three tags correspond to the special errors thrown by the
+ # set_read_and_write_concerns.js override when it refuses to replace the readConcern or
+ # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to
+ # be warranted.
+ ##
+ # "Cowardly refusing to override read concern of command: ..."
+ - assumes_read_concern_unchanged
+ # "Cowardly refusing to override write concern of command: ..."
+ - assumes_write_concern_unchanged
+ # "Cowardly refusing to run test with overridden write concern when it uses a command that can
+ # only perform w=1 writes: ..."
+ - requires_collmod_command
+ - requires_eval_command
executor:
js_test:
config:
shell_options:
- eval: "var testingReplication = true; load('jstests/libs/override_methods/set_majority_read_and_write_concerns.js');"
+ global_vars:
+ TestData:
+ defaultReadConcernLevel: majority
+ eval: >-
+ var testingReplication = true;
+ load('jstests/libs/override_methods/set_read_and_write_concerns.js');
readMode: commands
hooks:
- - class: ValidateCollections
+ # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
+ # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
+ # validating the entire contents of the collection.
- class: CheckReplOplogs
- class: CheckReplDBHash
+ - class: ValidateCollections
- class: CleanEveryN
n: 20
fixture:
@@ -84,7 +52,6 @@ executor:
set_parameters:
enableTestCommands: 1
numInitialSyncAttempts: 1
+ writePeriodicNoops: 1
enableMajorityReadConcern: ''
num_nodes: 2
- # Needs to be set for any ephemeral or no-journaling storage engine
- write_concern_majority_journal_default: false
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml
index 832e0ab9bfc..114ebcab372 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml
@@ -14,6 +14,7 @@ selector:
- jstests/core/commands_that_do_not_write_do_not_accept_wc.js
- jstests/core/constructors.js
- jstests/core/eval_mr.js
+ - jstests/core/function_string_representations.js
- jstests/core/geo_big_polygon3.js
- jstests/core/geo_mapreduce.js
- jstests/core/geo_mapreduce2.js
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml
index 3b48456c92e..1e51713a475 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml
@@ -14,6 +14,7 @@ selector:
- jstests/core/commands_that_do_not_write_do_not_accept_wc.js
- jstests/core/constructors.js
- jstests/core/eval_mr.js
+ - jstests/core/function_string_representations.js
- jstests/core/geo_big_polygon3.js
- jstests/core/geo_mapreduce.js
- jstests/core/geo_mapreduce2.js
diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
index 0bafa32a1e5..acf8ae53e96 100644
--- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
@@ -50,6 +50,7 @@ selector:
- jstests/sharding/shard2.js
- jstests/sharding/shard3.js
- jstests/sharding/shard_collection_basic.js
+ - jstests/sharding/shard_existing_coll_chunk_count.js
- jstests/sharding/sharding_balance1.js
- jstests/sharding/sharding_balance2.js
- jstests/sharding/sharding_balance3.js
diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index f4c741badd4..c7ff1b24395 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -95,6 +95,8 @@ selector:
# TODO (SERVER-27379): Cannot create views when featureCompatibilityVersion is 3.2.
# Unblacklist when the featureCompatibilityVersion is 3.4 or higher.
- jstests/sharding/views.js
+ # SERVER-20392: Reenable when backported to 3.4 and released as last-stable.
+ - jstests/sharding/shard_existing_coll_chunk_count.js
executor:
js_test:
diff --git a/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml
new file mode 100644
index 00000000000..388046d2782
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml
@@ -0,0 +1,78 @@
+test_kind: js_test
+
+selector:
+ js_test:
+ roots:
+ - jstests/core/**/*.js
+ exclude_files:
+ # These tests are not expected to pass with replica-sets:
+ - jstests/core/dbadmin.js
+ - jstests/core/opcounters_write_cmd.js
+ - jstests/core/read_after_optime.js
+ - jstests/core/capped_update.js
+ # The connection_string_validation.js test does not expect the mongo shell to be using a replica
+ # set connection string.
+ - jstests/core/connection_string_validation.js
+ # These tests attempt to read from the "system.profile" collection, which may be missing entries
+ # if a write was performed on the primary of the replica set instead.
+ - jstests/core/*profile*.js
+ # The shellkillop.js test spawns a parallel shell without using startParallelShell() and
+ # therefore doesn't inherit the w="majority" write concern when performing its writes.
+ - jstests/core/shellkillop.js
+ exclude_with_any_tags:
+ ##
+ # The next three tags correspond to the special errors thrown by the
+ # set_read_and_write_concerns.js override when it refuses to replace the readConcern or
+ # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to
+ # be warranted.
+ ##
+ # "Cowardly refusing to override read concern of command: ..."
+ - assumes_read_concern_unchanged
+ # "Cowardly refusing to override write concern of command: ..."
+ - assumes_write_concern_unchanged
+ # "Cowardly refusing to run test with overridden write concern when it uses a command that can
+ # only perform w=1 writes: ..."
+ - requires_collmod_command
+ - requires_eval_command
+ ##
+ # The next tag corresponds to the special error thrown by the set_read_preference_secondary.js
+ # override when it refuses to replace the readPreference of a particular command. Above each tag
+ # are the message(s) that cause the tag to be warranted.
+ ##
+ # "Cowardly refusing to override read preference of command: ..."
+ # "Cowardly refusing to run test with overridden read preference when it reads from a
+ # non-replicated collection: ..."
+ - assumes_read_preference_unchanged
+
+executor:
+ js_test:
+ config:
+ shell_options:
+ global_vars:
+ TestData:
+ defaultReadConcernLevel: local
+ eval: >-
+ testingReplication = true;
+ load('jstests/libs/override_methods/set_read_and_write_concerns.js');
+ load('jstests/libs/override_methods/set_read_preference_secondary.js');
+ readMode: commands
+ hooks:
+ # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
+ # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
+ # validating the entire contents of the collection.
+ - class: CheckReplOplogs
+ - class: CheckReplDBHash
+ - class: ValidateCollections
+ - class: CleanEveryN
+ n: 20
+ fixture:
+ class: ReplicaSetFixture
+ mongod_options:
+ set_parameters:
+ enableTestCommands: 1
+ numInitialSyncAttempts: 1
+ # This suite requires w="majority" writes to be applied on all secondaries. By using a 2-node
+ # replica set and having secondaries vote, the majority of the replica set is all nodes.
+ num_nodes: 2
+ voting_secondaries: true
+ use_replica_set_connection_string: true
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index e4cadc8e006..d94cd438ee0 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -115,8 +115,8 @@ def mongos_program(logger, executable=None, process_kwargs=None, **kwargs):
return _process.Process(logger, args, **process_kwargs)
-def mongo_shell_program(logger, executable=None, filename=None, process_kwargs=None,
- isMainTest=True, **kwargs):
+def mongo_shell_program(logger, executable=None, connection_string=None, filename=None,
+ process_kwargs=None, isMainTest=True, **kwargs):
"""
Returns a Process instance that starts a mongo shell with arguments
constructed from 'kwargs'.
@@ -184,9 +184,22 @@ def mongo_shell_program(logger, executable=None, filename=None, process_kwargs=N
if config.SHELL_WRITE_MODE is not None:
kwargs["writeMode"] = config.SHELL_WRITE_MODE
+ if connection_string is not None:
+ # The --host and --port options are ignored by the mongo shell when an explicit connection
+ # string is specified. We remove these options to avoid any ambiguity with what server the
+ # logged mongo shell invocation will connect to.
+ if "port" in kwargs:
+ kwargs.pop("port")
+
+ if "host" in kwargs:
+ kwargs.pop("host")
+
# Apply the rest of the command line arguments.
_apply_kwargs(args, kwargs)
+ if connection_string is not None:
+ args.append(connection_string)
+
# Have the mongos shell run the specified file.
args.append(filename)
diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py
index f058372df8c..e8e37d2c004 100644
--- a/buildscripts/resmokelib/logging/buildlogger.py
+++ b/buildscripts/resmokelib/logging/buildlogger.py
@@ -5,7 +5,9 @@ Defines handlers for communicating with a buildlogger server.
from __future__ import absolute_import
import functools
+import httplib
import urllib2
+import json
from . import handlers
from . import loggers
@@ -131,6 +133,47 @@ def new_test_id(build_id, build_config, test_filename, test_command):
return response["id"]
+class _LogsSplitter(object):
+ """Class with static methods used to split list of log lines into smaller batches."""
+
+ @staticmethod
+ def split_logs(log_lines, max_size):
+ """
+ Splits the log lines into batches of size less than or equal to max_size.
+
+ Args:
+ log_lines: A list of log lines.
+ max_size: The maximum size in bytes a batch of log lines can have in JSON.
+ Returns:
+ A list of list of log lines. Each item is a list is a list of log lines
+ satisfying the size requirement.
+ """
+ if not max_size:
+ return [log_lines]
+
+ def line_size(line):
+ """
+ Computes the encoded JSON size of a log line as part of an array.
+ 2 is added to each string size to account for the array representation of the logs,
+ as each line is preceded by a '[' or a space and followed by a ',' or a ']'.
+ """
+ return len(json.dumps(line, encoding="utf-8")) + 2
+
+ curr_logs = []
+ curr_logs_size = 0
+ split_logs = []
+ for line in log_lines:
+ size = line_size(line)
+ if curr_logs_size + size > max_size:
+ split_logs.append(curr_logs)
+ curr_logs = []
+ curr_logs_size = 0
+ curr_logs.append(line)
+ curr_logs_size += size
+ split_logs.append(curr_logs)
+ return split_logs
+
+
class _BaseBuildloggerHandler(handlers.BufferedHandler):
"""
Base class of the buildlogger handler for the global logs and the
@@ -138,8 +181,8 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler):
"""
def __init__(self,
- build_id,
build_config,
+ endpoint,
capacity=_SEND_AFTER_LINES,
interval_secs=_SEND_AFTER_SECS):
"""
@@ -157,8 +200,9 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler):
username,
password)
- self.build_id = build_id
+ self.endpoint = endpoint
self.retry_buffer = []
+ self.max_size = None
def process_record(self, record):
"""
@@ -177,12 +221,55 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler):
"""
Convenience method for subclasses to use when making POST requests.
"""
-
return self.http_handler.post(*args, **kwargs)
def _append_logs(self, log_lines):
- raise NotImplementedError("_append_logs must be implemented by _BaseBuildloggerHandler"
- " subclasses")
+ """
+ Sends a POST request to the handlers endpoint with the logs that have been captured.
+
+ Returns:
+ The number of log lines that have been successfully sent.
+ """
+ lines_sent = 0
+ for chunk in _LogsSplitter.split_logs(log_lines, self.max_size):
+ chunk_lines_sent = self.__append_logs_chunk(chunk)
+ lines_sent += chunk_lines_sent
+ if chunk_lines_sent < len(chunk):
+ # Not all lines have been sent. We stop here.
+ break
+ return lines_sent
+
+ def __append_logs_chunk(self, log_lines_chunk):
+ """
+ Sends a log lines chunk, handles 413 Request Entity Too Large errors and retries
+ if necessary.
+
+ Returns:
+ The number of log lines that have been successfully sent.
+ """
+ try:
+ self.post(self.endpoint, data=log_lines_chunk)
+ return len(log_lines_chunk)
+ except urllib2.HTTPError as err:
+ # Handle the "Request Entity Too Large" error, set the max size and retry.
+ if err.code == httplib.REQUEST_ENTITY_TOO_LARGE:
+ response_data = json.load(err, "utf-8")
+ if isinstance(response_data, dict) and "max_size" in response_data:
+ new_max_size = response_data["max_size"]
+ if self.max_size and new_max_size >= self.max_size:
+ loggers._BUILDLOGGER_FALLBACK.exception(
+ "Received an HTTP 413 code, but already had max_size set")
+ return 0
+ loggers._BUILDLOGGER_FALLBACK.warning(
+ "Received an HTTP 413 code, updating the request max_size to %s",
+ new_max_size)
+ self.max_size = new_max_size
+ return self._append_logs(log_lines_chunk)
+ loggers._BUILDLOGGER_FALLBACK.exception("Encountered an error.")
+ return 0
+ except:
+ loggers._BUILDLOGGER_FALLBACK.exception("Encountered an error.")
+ return 0
def _flush_buffer_with_lock(self, buf, close_called):
"""
@@ -196,9 +283,10 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler):
self.retry_buffer.extend(buf)
- if self._append_logs(self.retry_buffer):
- self.retry_buffer = []
- elif close_called:
+ nb_sent = self._append_logs(self.retry_buffer)
+ if nb_sent:
+ self.retry_buffer = self.retry_buffer[nb_sent:]
+ if close_called and self.retry_buffer:
# Request to the buildlogger server returned an error, so use the fallback logger to
# avoid losing the log messages entirely.
for (_, message) in self.retry_buffer:
@@ -215,29 +303,14 @@ class BuildloggerTestHandler(_BaseBuildloggerHandler):
Buildlogger handler for the test logs.
"""
- def __init__(self, build_id, build_config, test_id, **kwargs):
- """
- Initializes the buildlogger handler with the build id, test id,
- and credentials.
- """
-
- _BaseBuildloggerHandler.__init__(self, build_id, build_config, **kwargs)
-
- self.test_id = test_id
-
- @_log_on_error
- def _append_logs(self, log_lines):
- """
- Sends a POST request to the APPEND_TEST_LOGS_ENDPOINT with the
- logs that have been captured.
- """
+ def __init__(self, build_config, build_id, test_id,
+ capacity=_SEND_AFTER_LINES, interval_secs=_SEND_AFTER_SECS):
+ """Initializes the buildlogger handler with the credentials, build id, and test id."""
endpoint = APPEND_TEST_LOGS_ENDPOINT % {
- "build_id": self.build_id,
- "test_id": self.test_id,
+ "build_id": build_id,
+ "test_id": test_id,
}
-
- response = self.post(endpoint, data=log_lines)
- return response is not None
+ _BaseBuildloggerHandler.__init__(self, build_config, endpoint, capacity, interval_secs)
@_log_on_error
def _finish_test(self, failed=False):
@@ -245,12 +318,7 @@ class BuildloggerTestHandler(_BaseBuildloggerHandler):
Sends a POST request to the APPEND_TEST_LOGS_ENDPOINT with the
test status.
"""
- endpoint = APPEND_TEST_LOGS_ENDPOINT % {
- "build_id": self.build_id,
- "test_id": self.test_id,
- }
-
- self.post(endpoint, headers={
+ self.post(self.endpoint, headers={
"X-Sendlogs-Test-Done": "true",
"X-Sendlogs-Test-Failed": "true" if failed else "false",
})
@@ -271,12 +339,8 @@ class BuildloggerGlobalHandler(_BaseBuildloggerHandler):
Buildlogger handler for the global logs.
"""
- @_log_on_error
- def _append_logs(self, log_lines):
- """
- Sends a POST request to the APPEND_GLOBAL_LOGS_ENDPOINT with
- the logs that have been captured.
- """
- endpoint = APPEND_GLOBAL_LOGS_ENDPOINT % {"build_id": self.build_id}
- response = self.post(endpoint, data=log_lines)
- return response is not None
+ def __init__(self, build_config, build_id,
+ capacity=_SEND_AFTER_LINES, interval_secs=_SEND_AFTER_SECS):
+ """Initializes the buildlogger handler with the credentials and build id."""
+ endpoint = APPEND_GLOBAL_LOGS_ENDPOINT % {"build_id": build_id}
+ _BaseBuildloggerHandler.__init__(self, build_config, endpoint, capacity, interval_secs)
diff --git a/buildscripts/resmokelib/logging/config.py b/buildscripts/resmokelib/logging/config.py
index c3960bbafd3..c3216b2d879 100644
--- a/buildscripts/resmokelib/logging/config.py
+++ b/buildscripts/resmokelib/logging/config.py
@@ -67,8 +67,8 @@ def apply_buildlogger_global_handler(logger, logging_config, build_id=None, buil
log_format = logger_info.get("format", _DEFAULT_FORMAT)
formatter = formatters.ISO8601Formatter(fmt=log_format)
- handler = buildlogger.BuildloggerGlobalHandler(build_id,
- build_config,
+ handler = buildlogger.BuildloggerGlobalHandler(build_config,
+ build_id,
**handler_info)
handler.setFormatter(formatter)
else:
@@ -98,8 +98,8 @@ def apply_buildlogger_test_handler(logger,
log_format = logger_info.get("format", _DEFAULT_FORMAT)
formatter = formatters.ISO8601Formatter(fmt=log_format)
- handler = buildlogger.BuildloggerTestHandler(build_id,
- build_config,
+ handler = buildlogger.BuildloggerTestHandler(build_config,
+ build_id,
test_id,
**handler_info)
handler.setFormatter(formatter)
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index b4b0066a5aa..83418502ead 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -79,13 +79,22 @@ class Fixture(object):
"""
return True
- def get_connection_string(self):
+ def get_internal_connection_string(self):
"""
Returns the connection string for this fixture. This is NOT a
driver connection string, but a connection string of the format
expected by the mongo::ConnectionString class.
"""
- raise NotImplementedError("get_connection_string must be implemented by Fixture subclasses")
+ raise NotImplementedError(
+ "get_internal_connection_string must be implemented by Fixture subclasses")
+
+ def get_driver_connection_url(self):
+ """
+ Return the mongodb connection string as defined here:
+ https://docs.mongodb.com/manual/reference/connection-string/
+ """
+ raise NotImplementedError(
+ "get_driver_connection_url must be implemented by Fixture subclasses")
def __str__(self):
return "%s (Job #%d)" % (self.__class__.__name__, self.job_num)
diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py
index 469c7ac0816..fb444cfe097 100644
--- a/buildscripts/resmokelib/testing/fixtures/masterslave.py
+++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py
@@ -5,7 +5,6 @@ Master/slave fixture for executing JSTests against.
from __future__ import absolute_import
import os.path
-import socket
import pymongo
@@ -160,6 +159,12 @@ class MasterSlaveFixture(interface.ReplFixture):
mongod_options = self.mongod_options.copy()
mongod_options.update(self.slave_options)
mongod_options["slave"] = ""
- mongod_options["source"] = "%s:%d" % (socket.gethostname(), self.port)
+ mongod_options["source"] = self.master.get_internal_connection_string()
mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "slave")
return self._new_mongod(mongod_logger, mongod_options)
+
+ def get_internal_connection_string(self):
+ return self.master.get_internal_connection_string()
+
+ def get_driver_connection_url(self):
+ return self.master.get_driver_connection_url()
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 141bbd60ea1..0ec3de30280 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -37,7 +37,8 @@ class ReplicaSetFixture(interface.ReplFixture):
write_concern_majority_journal_default=None,
auth_options=None,
replset_config_options=None,
- voting_secondaries=True):
+ voting_secondaries=False,
+ use_replica_set_connection_string=False):
interface.ReplFixture.__init__(self, logger, job_num)
@@ -50,6 +51,7 @@ class ReplicaSetFixture(interface.ReplFixture):
self.auth_options = auth_options
self.replset_config_options = utils.default_if_none(replset_config_options, {})
self.voting_secondaries = voting_secondaries
+ self.use_replica_set_connection_string = use_replica_set_connection_string
# The dbpath in mongod_options is used as the dbpath prefix for replica set members and
# takes precedence over other settings. The ShardedClusterFixture uses this parameter to
@@ -97,7 +99,7 @@ class ReplicaSetFixture(interface.ReplFixture):
# Initiate the replica set.
members = []
for (i, node) in enumerate(self.nodes):
- member_info = {"_id": i, "host": node.get_connection_string()}
+ member_info = {"_id": i, "host": node.get_internal_connection_string()}
if i > 0:
member_info["priority"] = 0
if i >= 7 or not self.voting_secondaries:
@@ -107,7 +109,7 @@ class ReplicaSetFixture(interface.ReplFixture):
members.append(member_info)
if self.initial_sync_node:
members.append({"_id": self.initial_sync_node_idx,
- "host": self.initial_sync_node.get_connection_string(),
+ "host": self.initial_sync_node.get_internal_connection_string(),
"priority": 0,
"hidden": 1,
"votes": 0})
@@ -291,11 +293,27 @@ class ReplicaSetFixture(interface.ReplFixture):
return logging.loggers.new_logger(logger_name, parent=self.logger)
- def get_connection_string(self):
+ def get_internal_connection_string(self):
if self.replset_name is None:
- raise ValueError("Must call setup() before calling get_connection_string()")
+ raise ValueError("Must call setup() before calling get_internal_connection_string()")
- conn_strs = [node.get_connection_string() for node in self.nodes]
+ conn_strs = [node.get_internal_connection_string() for node in self.nodes]
if self.initial_sync_node:
- conn_strs.append(self.initial_sync_node.get_connection_string())
+ conn_strs.append(self.initial_sync_node.get_internal_connection_string())
return self.replset_name + "/" + ",".join(conn_strs)
+
+ def get_driver_connection_url(self):
+ if self.replset_name is None:
+ raise ValueError("Must call setup() before calling get_driver_connection_url()")
+
+ if self.use_replica_set_connection_string:
+ # We use a replica set connection string when all nodes are electable because we
+ # anticipate the client will want to gracefully handle any failovers.
+ conn_strs = [node.get_internal_connection_string() for node in self.nodes]
+ if self.initial_sync_node:
+ conn_strs.append(self.initial_sync_node.get_internal_connection_string())
+ return "mongodb://" + ",".join(conn_strs) + "/?replicaSet=" + self.replset_name
+ else:
+ # We return a direct connection to the expected pimary when only the first node is
+ # electable because we want the client to error out if a stepdown occurs.
+ return self.nodes[0].get_driver_connection_url()
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index ac7e597f24b..2e2db535d6d 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -170,11 +170,14 @@ class ShardedClusterFixture(interface.Fixture):
all(shard.is_running() for shard in self.shards) and
self.mongos is not None and self.mongos.is_running())
- def get_connection_string(self):
+ def get_internal_connection_string(self):
if self.mongos is None:
- raise ValueError("Must call setup() before calling get_connection_string()")
+ raise ValueError("Must call setup() before calling get_internal_connection_string()")
- return "%s:%d" % (socket.gethostname(), self.mongos.port)
+ return self.mongos.get_internal_connection_string()
+
+ def get_driver_connection_url(self):
+ return "mongodb://" + self.get_internal_connection_string()
def _new_configsvr(self):
"""
@@ -229,16 +232,11 @@ class ShardedClusterFixture(interface.Fixture):
mongos_logger = logging.loggers.new_logger(logger_name, parent=self.logger)
mongos_options = copy.deepcopy(self.mongos_options)
- configdb_hostname = socket.gethostname()
if self.separate_configsvr:
- configdb_replset = ShardedClusterFixture._CONFIGSVR_REPLSET_NAME
- configdb_port = self.configsvr.port
- mongos_options["configdb"] = "%s/%s:%d" % (configdb_replset,
- configdb_hostname,
- configdb_port)
+ mongos_options["configdb"] = self.configsvr.get_internal_connection_string()
else:
- mongos_options["configdb"] = "%s:%d" % (configdb_hostname, self.shards[0].port)
+ mongos_options["configdb"] = "localhost:%d" % (self.shards[0].port)
return _MongoSFixture(mongos_logger,
self.job_num,
@@ -254,9 +252,9 @@ class ShardedClusterFixture(interface.Fixture):
for more details.
"""
- hostname = socket.gethostname()
- self.logger.info("Adding %s:%d as a shard..." % (hostname, shard.port))
- client.admin.command({"addShard": "%s:%d" % (hostname, shard.port)})
+ connection_string = shard.get_internal_connection_string()
+ self.logger.info("Adding %s as a shard...", connection_string)
+ client.admin.command({"addShard": connection_string})
class _MongoSFixture(interface.Fixture):
@@ -356,3 +354,12 @@ class _MongoSFixture(interface.Fixture):
def is_running(self):
return self.mongos is not None and self.mongos.poll() is None
+
+ def get_internal_connection_string(self):
+ if self.mongos is None:
+ raise ValueError("Must call setup() before calling get_internal_connection_string()")
+
+ return "localhost:%d" % self.port
+
+ def get_driver_connection_url(self):
+ return "mongodb://" + self.get_internal_connection_string()
diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py
index ba62b3d2b8c..bc69775c285 100644
--- a/buildscripts/resmokelib/testing/fixtures/standalone.py
+++ b/buildscripts/resmokelib/testing/fixtures/standalone.py
@@ -7,7 +7,6 @@ from __future__ import absolute_import
import os
import os.path
import shutil
-import socket
import time
import pymongo
@@ -146,8 +145,11 @@ class MongoDFixture(interface.Fixture):
def is_running(self):
return self.mongod is not None and self.mongod.poll() is None
- def get_connection_string(self):
+ def get_internal_connection_string(self):
if self.mongod is None:
- raise ValueError("Must call setup() before calling get_connection_string()")
+ raise ValueError("Must call setup() before calling get_internal_connection_string()")
- return "%s:%d" % (socket.gethostname(), self.port)
+ return "localhost:%d" % self.port
+
+ def get_driver_connection_url(self):
+ return "mongodb://" + self.get_internal_connection_string()
diff --git a/buildscripts/resmokelib/testing/testcases.py b/buildscripts/resmokelib/testing/testcases.py
index b4029fc6ea8..21d35215a29 100644
--- a/buildscripts/resmokelib/testing/testcases.py
+++ b/buildscripts/resmokelib/testing/testcases.py
@@ -187,7 +187,7 @@ class CPPIntegrationTestCase(TestCase):
def configure(self, fixture, *args, **kwargs):
TestCase.configure(self, fixture, *args, **kwargs)
- self.program_options["connectionString"] = self.fixture.get_connection_string()
+ self.program_options["connectionString"] = self.fixture.get_internal_connection_string()
def run_test(self):
try:
@@ -362,6 +362,19 @@ class JSTestCase(TestCase):
# Directory already exists.
pass
+ process_kwargs = self.shell_options.get("process_kwargs", {}).copy()
+
+ if "KRB5_CONFIG" in process_kwargs and "KRB5CCNAME" not in process_kwargs:
+ # Use a job-specific credential cache for JavaScript tests involving Kerberos.
+ krb5_dir = os.path.join(data_dir, "krb5")
+ try:
+ os.makedirs(krb5_dir)
+ except os.error:
+ pass
+ process_kwargs["KRB5CCNAME"] = "DIR:" + os.path.join(krb5_dir, ".")
+
+ self.shell_options["process_kwargs"] = process_kwargs
+
def _get_data_dir(self, global_vars):
"""
Returns the value that the mongo shell should set for the
@@ -408,11 +421,13 @@ class JSTestCase(TestCase):
is_main_test = True
if thread_id > 0:
is_main_test = False
- return core.programs.mongo_shell_program(logger,
- executable=self.shell_executable,
- filename=self.js_filename,
- isMainTest=is_main_test,
- **self.shell_options)
+ return core.programs.mongo_shell_program(
+ logger,
+ executable=self.shell_executable,
+ filename=self.js_filename,
+ connection_string=self.fixture.get_driver_connection_url(),
+ isMainTest=is_main_test,
+ **self.shell_options)
def _run_test_in_thread(self, thread_id):
# Make a logger for each thread.
diff --git a/buildscripts/scons.py b/buildscripts/scons.py
index e7860162eb6..b0b9cfa834e 100755
--- a/buildscripts/scons.py
+++ b/buildscripts/scons.py
@@ -1,16 +1,26 @@
-#!/usr/bin/python
+#!/usr/bin/env python2
+
+from __future__ import print_function
import os
-import subprocess
import sys
SCONS_VERSION = os.environ.get('SCONS_VERSION', "2.5.0")
-mongodb_root = os.path.dirname(os.path.dirname(__file__))
-scons_dir = os.path.join(mongodb_root, 'src', 'third_party', 'scons-' + SCONS_VERSION)
+mongodb_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+scons_dir = os.path.join(mongodb_root, 'src', 'third_party','scons-' + SCONS_VERSION,
+ 'scons-local-' + SCONS_VERSION)
+
+if not os.path.exists(scons_dir):
+ print("Could not find SCons in '%s'" % (scons_dir))
+ sys.exit(1)
+
+sys.path = [scons_dir] + sys.path
+
+try:
+ import SCons.Script
+except ImportError:
+ print("Could not find SCons in '%s'" % (scons_dir))
+ sys.exit(1)
-if sys.platform == 'win32':
- args = [sys.executable, os.path.join(scons_dir, 'scons.py')] + sys.argv[1:]
- sys.exit(subprocess.call(args))
-else:
- os.execv(os.path.join(scons_dir, 'scons.py'), sys.argv)
+SCons.Script.main()
diff --git a/buildscripts/tests/resmokelib/logging/test_buildlogger.py b/buildscripts/tests/resmokelib/logging/test_buildlogger.py
new file mode 100644
index 00000000000..7bc705948db
--- /dev/null
+++ b/buildscripts/tests/resmokelib/logging/test_buildlogger.py
@@ -0,0 +1,67 @@
+"""Unit tests for the buildscripts.resmokelib.logging.buildlogger module."""
+
+from __future__ import absolute_import
+
+import json
+import unittest
+
+from buildscripts.resmokelib.logging import buildlogger
+
+
+class TestLogsSplitter(unittest.TestCase):
+ """Unit tests for the _LogsSplitter class."""
+
+ def test_split_no_logs(self):
+ logs = []
+ max_size = 10
+ self.assertEqual([[]], buildlogger._LogsSplitter.split_logs(logs, max_size))
+
+ def test_split_no_max_size(self):
+ logs = self.__generate_logs(size=30)
+ max_size = None
+ self.assertEqual([logs], buildlogger._LogsSplitter.split_logs(logs, max_size))
+
+ def test_split_max_size_smaller(self):
+ logs = self.__generate_logs(size=20)
+ max_size = 30
+ self.assertEqual([logs], buildlogger._LogsSplitter.split_logs(logs, max_size))
+
+ def test_split_max_size_equal(self):
+ logs = self.__generate_logs(size=30)
+ max_size = 30
+ self.assertEqual([logs], buildlogger._LogsSplitter.split_logs(logs, max_size))
+
+ def test_split_max_size_larger(self):
+ logs = self.__generate_logs(size=31)
+ max_size = 30
+ self.assertEqual(
+ [logs[0:-1], logs[-1:]],
+ buildlogger._LogsSplitter.split_logs(logs, max_size))
+
+ logs = self.__generate_logs(size=149)
+ max_size = 19
+ self.assertEqual(
+ [logs[0:3], logs[3:6], logs[6:9], logs[9:12], logs[12:15],
+ logs[15:18], logs[18:21], logs[21:24], logs[24:27], logs[27:]],
+ buildlogger._LogsSplitter.split_logs(logs, max_size))
+
+ def check_split_sizes(self, splits, max_size):
+ for split in splits:
+ self.assertTrue(TestLogsSplitter.size(split) <= max_size)
+
+ def __generate_logs(self, size):
+ # The size of [ "x" ] is 5. This is the minimum size we generate.
+ self.assertTrue(size >= 5)
+ # Each new "x" adds 5 to the size.
+ nb_lines = size / 5
+ # Each additional "x" on a line adds 1 to the size.
+ last_line_extra = size % 5
+ logs = ["x"] * nb_lines
+ logs[-1] += "x" * last_line_extra
+ self.assertEqual(size, TestLogsSplitter.size(logs))
+ return logs
+
+ @staticmethod
+ def size(logs):
+ """Returns the size of the log lines when represented in JSON."""
+ return len(json.dumps(logs, encoding="utf-8"))