diff options
| author | Apollon Oikonomopoulos <apoikos@debian.org> | 2018-03-22 12:04:55 +0200 |
|---|---|---|
| committer | Apollon Oikonomopoulos <apoikos@debian.org> | 2018-03-22 12:04:55 +0200 |
| commit | c49e99631589113663b1a3ac691870421965a315 (patch) | |
| tree | ac8127a5a6816f169a6656511bf131a35af2f74a /buildscripts | |
| parent | d982a88efa79f510c03f1c6c8c63360680ebbf88 (diff) | |
New upstream version 3.4.14upstream/3.4.14
Diffstat (limited to 'buildscripts')
22 files changed, 501 insertions, 189 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index d171a5a6bc1..6e39685ef8b 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -17,7 +17,7 @@ import sys import urlparse import yaml -API_SERVER_DEFAULT = "http://evergreen-api.mongodb.com:8080" +API_SERVER_DEFAULT = "https://evergreen.mongodb.com" # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: @@ -418,6 +418,8 @@ def main(): tests_by_task = _load_tests_file(values.test_list_file) # If there are no tests to run, carry on. if tests_by_task is None: + test_results = {"failures": 0, "results": []} + _write_report_file(test_results, values.report_file) sys.exit(0) # Run the executor finder. @@ -438,6 +440,7 @@ def main(): # If there are no changed tests, exit cleanly. if not changed_tests: print "No new or modified tests found." + _write_report_file({}, values.test_list_outfile) sys.exit(0) suites = resmokelib.parser.get_suites(values, changed_tests) tests_by_executor = create_executor_list(suites, exclude_suites) diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py index 73e1cbe6d41..09245904a7f 100755 --- a/buildscripts/hang_analyzer.py +++ b/buildscripts/hang_analyzer.py @@ -24,6 +24,7 @@ import signal import subprocess import sys import tempfile +import traceback import time from distutils import spawn from optparse import OptionParser @@ -53,7 +54,7 @@ def call(a, logger): if ret != 0: logger.error("Bad exit code %d" % (ret)) - raise Exception() + raise Exception("Bad exit code %d from %s" % (ret, " ".join(a))) # Copied from python 2.7 version of subprocess.py @@ -375,14 +376,14 @@ class GDBDumper(object): base, ext = os.path.splitext(logger.mongo_process_filename) raw_stacks_filename = base + '_raw_stacks' + ext raw_stacks_commands = [ - 'echo \\nWriting raw stacks to %s.\\n' % raw_stacks_filename, - # This sends output to log file rather than stdout until we turn logging off. - 'set logging redirect on', - 'set logging file ' + raw_stacks_filename, - 'set logging on', - 'thread apply all bt', - 'set logging off', - ] + 'echo \\nWriting raw stacks to %s.\\n' % raw_stacks_filename, + # This sends output to log file rather than stdout until we turn logging off. + 'set logging redirect on', + 'set logging file ' + raw_stacks_filename, + 'set logging on', + 'thread apply all bt', + 'set logging off', + ] cmds = [ "set interactive-mode off", @@ -732,28 +733,38 @@ def main(): # a signal handler to wait for the signal since it supports POSIX signals. if _is_windows: root_logger.info("Calling SetEvent to signal python process %s with PID %d" % - (process_name, pid)) + (process_name, pid)) signal_event_object(root_logger, pid) else: root_logger.info("Sending signal SIGUSR1 to python process %s with PID %d" % - (process_name, pid)) + (process_name, pid)) signal_process(root_logger, pid, signal.SIGUSR1) + trapped_exceptions = [] + # Dump all processes, except python & java. for (pid, process_name) in [(p, pn) for (p, pn) in processes if not re.match("^(java|python)", pn)]: process_logger = get_process_logger(options.debugger_output, pid, process_name) - dbg.dump_info( - root_logger, - process_logger, - pid, - process_name, - options.dump_core and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext())) + try: + dbg.dump_info( + root_logger, + process_logger, + pid, + process_name, + options.dump_core and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext())) + except Exception as err: + root_logger.info("Error encountered when invoking debugger %s" % err) + trapped_exceptions.append(traceback.format_exc()) # Dump java processes using jstack. for (pid, process_name) in [(p, pn) for (p, pn) in processes if pn.startswith("java")]: process_logger = get_process_logger(options.debugger_output, pid, process_name) - jstack.dump_info(root_logger, process_logger, pid, process_name) + try: + jstack.dump_info(root_logger, process_logger, pid, process_name) + except Exception as err: + root_logger.info("Error encountered when invoking debugger %s" % err) + trapped_exceptions.append(traceback.format_exc()) # Signal go processes to ensure they print out stack traces, and die on POSIX OSes. # On Windows, this will simply kill the process since python emulates SIGABRT as @@ -761,10 +772,15 @@ def main(): # Note: The stacktrace output may be captured elsewhere (i.e. resmoke). for (pid, process_name) in [(p, pn) for (p, pn) in processes if pn in go_processes]: root_logger.info("Sending signal SIGABRT to go process %s with PID %d" % - (process_name, pid)) + (process_name, pid)) signal_process(root_logger, pid, signal.SIGABRT) root_logger.info("Done analyzing all processes for hangs") + for exception in trapped_exceptions: + root_logger.info(exception) + if trapped_exceptions: + sys.exit(1) + if __name__ == "__main__": main() diff --git a/buildscripts/package_test/.kitchen.yml b/buildscripts/package_test/.kitchen.yml index e9b6989d221..3b4a476551c 100644 --- a/buildscripts/package_test/.kitchen.yml +++ b/buildscripts/package_test/.kitchen.yml @@ -2,11 +2,13 @@ driver: name: ec2 region: us-east-1 + vpc_mode: true + vpc_id: <%= ENV['KITCHEN_VPC'] %> subnet_id: <%= ENV['KITCHEN_SUBNET'] %> security_group_ids: - <%= ENV['KITCHEN_SECURITY_GROUP'] %> aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %> - interface: dns + interface: private associate_public_ip: true verifier: diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb index dcf7499289e..9b4ea703b2b 100644 --- a/buildscripts/package_test/recipes/install_mongodb.rb +++ b/buildscripts/package_test/recipes/install_mongodb.rb @@ -84,6 +84,16 @@ if platform_family? 'suse' EOD end + %w( + SLES12-Pool + SLES12-Updates + ).each do |repo| + execute "add #{repo}" do + command "zypper addrepo --check --refresh --name \"#{repo}\" http://smt-ec2.susecloud.net/repo/SUSE/Products/SLE-SERVER/12/x86_64/product?credentials=SMT-http_smt-ec2_susecloud_net 'SMT-http_smt-ec2_susecloud_net:#{repo}'" + not_if "zypper lr | grep #{repo}" + end + end + execute 'install mongod' do command 'zypper -n install `find . -name "*server*.rpm"`' cwd homedir diff --git a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml index bc227b09279..72b838b3c07 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml @@ -6,11 +6,28 @@ selector: - jstests/aggregation/expressions/*.js - jstests/aggregation/sources/*/*.js exclude_files: - - jstests/aggregation/bugs/server18198.js # Uses a mocked mongo client to test read preference. - - jstests/aggregation/mongos_slaveok.js # Majority read on secondary requires afterOpTime. - - jstests/aggregation/sources/facet/use_cases.js # Cannot specify write concern when - # secondaryThrottle is not set. - - jstests/aggregation/testSlave.js # Majority read on secondary requires afterOpTime. + # These tests fail due to the inability to specify a writeConcern when secondaryThrottle is not + # set as part of the moveChunk command. + - jstests/aggregation/sources/facet/use_cases.js + # These test fail because afterOpTime is required to guarantee a secondary has advanced its + # majority-committed snapshot. + - jstests/aggregation/mongos_slaveok.js + - jstests/aggregation/testSlave.js + exclude_with_any_tags: + ## + # The next three tags correspond to the special errors thrown by the + # set_read_and_write_concerns.js override when it refuses to replace the readConcern or + # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to + # be warranted. + ## + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "Cowardly refusing to override write concern of command: ..." + - assumes_write_concern_unchanged + # "Cowardly refusing to run test with overridden write concern when it uses a command that can + # only perform w=1 writes: ..." + - requires_collmod_command + - requires_eval_command executor: js_test: @@ -18,13 +35,19 @@ executor: shell_options: global_vars: TestData: + defaultReadConcernLevel: majority enableMajorityReadConcern: '' - eval: "var testingReplication = true; load('jstests/libs/override_methods/set_majority_read_and_write_concerns.js');" + eval: >- + var testingReplication = true; + load('jstests/libs/override_methods/set_read_and_write_concerns.js'); readMode: commands hooks: - - class: ValidateCollections + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. - class: CheckReplOplogs - class: CheckReplDBHash + - class: ValidateCollections fixture: class: ReplicaSetFixture mongod_options: @@ -33,5 +56,3 @@ executor: enableTestCommands: 1 numInitialSyncAttempts: 1 num_nodes: 2 - # Needs to be set for any ephemeral or no-journaling storage engine - write_concern_majority_journal_default: false diff --git a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml index f47e424e1cf..22e84bc9c63 100644 --- a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml @@ -3,79 +3,47 @@ selector: roots: - jstests/core/**/*.js exclude_files: - # Tests that won't work with an injected 'majority' readConcern - # and/or an injected 'majority' writeConcern. Where a function is - # listed the reason is we don't have a reliable solution to override - # the write concern for that function. - - jstests/core/batch_write_command*.js # these tests use various write concerns - - jstests/core/bench_test*.js # benchRun() used for writes - - jstests/core/capped_update.js # uses godinsert and can't run under replication. - - jstests/core/crud_api.js # has specific w:0 tests - - jstests/core/error2.js # db.eval() used - - jstests/core/eval0.js # db.eval() used - - jstests/core/eval1.js # db.eval() used - - jstests/core/eval3.js # db.eval() used - - jstests/core/eval4.js # db.eval() used - - jstests/core/eval5.js # db.eval() used - - jstests/core/eval6.js # db.eval() used - - jstests/core/eval7.js # db.eval() used - - jstests/core/eval9.js # db.eval() used - - jstests/core/evala.js # db.eval() used - - jstests/core/evalb.js # db.eval() used - - jstests/core/evald.js # db.eval() used - - jstests/core/evale.js # db.eval() used - - jstests/core/evalg.js # db.eval() used - - jstests/core/eval_mr.js # db.eval() used - - jstests/core/eval_nolock.js # db.eval() used - - jstests/core/geo_s2cursorlimitskip.js # drops system.profile collection and counts ops. - - jstests/core/js3.js # db.dbEval() used - - jstests/core/js7.js # db.eval() used - - jstests/core/js9.js # db.eval() used - - jstests/core/mr_merge.js # mr temp tables aren't replicated - - jstests/core/mr_merge2.js # mr temp tables aren't replicated - - jstests/core/mr_outreduce.js # mr temp tables aren't replicated - - jstests/core/mr_outreduce2.js # mr temp tables aren't replicated - - jstests/core/opcounters_active.js # off by n problem with opcounters - - jstests/core/opcounters_write_cmd.js # off by n problem with opcounters - - jstests/core/profile_agg.js # system.profile not replicated - - jstests/core/profile_count.js # system.profile not replicated - - jstests/core/profile_delete.js # system.profile not replicated - - jstests/core/profile_distinct.js # system.profile not replicated - - jstests/core/profile_find.js # system.profile not replicated - - jstests/core/profile_findandmodify.js # system.profile not replicated - - jstests/core/profile_geonear.js # system.profile not replicated - - jstests/core/profile_getmore.js # system.profile not replicated - - jstests/core/profile_group.js # system.profile not replicated - - jstests/core/profile_insert.js # system.profile not replicated - - jstests/core/profile_mapreduce.js # system.profile not replicated - - jstests/core/profile_update.js # system.profile not replicated - - jstests/core/profile1.js # system.profile not replicated - - jstests/core/profile2.js # system.profile not replicated - - jstests/core/profile3.js # system.profile not replicated - - jstests/core/read_after_optime.js # verifies read after optime fails on standalone - - jstests/core/remove8.js # db.eval() used - - jstests/core/rename4.js # db.eval() used - - jstests/core/shell1.js # tests setSlaveOk() variations on standalone mongod - - jstests/core/shellkillop.js # db.eval() used - - jstests/core/shell_writeconcern.js # checks write concern shell helpers - - jstests/core/storefunc.js # db.eval() used - - jstests/core/write_result.js # Tests invalid writeConcern, we shouldn't override. - # Tests that need triaging & remediation | blacklist decision - # Comments list possible problem point under review. - - jstests/core/capped6.js # Uses captrunc test command. - - jstests/core/convert_to_capped_nonexistant.js # Uses convertToCapped and captrunc command. - - jstests/core/stages_delete.js # Uses stageDebug command for deletes. + # These tests are not expected to pass with replica-sets: + - jstests/core/dbadmin.js + - jstests/core/opcounters_write_cmd.js + - jstests/core/read_after_optime.js + - jstests/core/capped_update.js + # These tests use benchRun(), which isn't configured to use the overridden writeConcern. + - jstests/core/bench_test*.js + exclude_with_any_tags: + ## + # The next three tags correspond to the special errors thrown by the + # set_read_and_write_concerns.js override when it refuses to replace the readConcern or + # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to + # be warranted. + ## + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "Cowardly refusing to override write concern of command: ..." + - assumes_write_concern_unchanged + # "Cowardly refusing to run test with overridden write concern when it uses a command that can + # only perform w=1 writes: ..." + - requires_collmod_command + - requires_eval_command executor: js_test: config: shell_options: - eval: "var testingReplication = true; load('jstests/libs/override_methods/set_majority_read_and_write_concerns.js');" + global_vars: + TestData: + defaultReadConcernLevel: majority + eval: >- + var testingReplication = true; + load('jstests/libs/override_methods/set_read_and_write_concerns.js'); readMode: commands hooks: - - class: ValidateCollections + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. - class: CheckReplOplogs - class: CheckReplDBHash + - class: ValidateCollections - class: CleanEveryN n: 20 fixture: @@ -84,7 +52,6 @@ executor: set_parameters: enableTestCommands: 1 numInitialSyncAttempts: 1 + writePeriodicNoops: 1 enableMajorityReadConcern: '' num_nodes: 2 - # Needs to be set for any ephemeral or no-journaling storage engine - write_concern_majority_journal_default: false diff --git a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml index 832e0ab9bfc..114ebcab372 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml @@ -14,6 +14,7 @@ selector: - jstests/core/commands_that_do_not_write_do_not_accept_wc.js - jstests/core/constructors.js - jstests/core/eval_mr.js + - jstests/core/function_string_representations.js - jstests/core/geo_big_polygon3.js - jstests/core/geo_mapreduce.js - jstests/core/geo_mapreduce2.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml index 3b48456c92e..1e51713a475 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml @@ -14,6 +14,7 @@ selector: - jstests/core/commands_that_do_not_write_do_not_accept_wc.js - jstests/core/constructors.js - jstests/core/eval_mr.js + - jstests/core/function_string_representations.js - jstests/core/geo_big_polygon3.js - jstests/core/geo_mapreduce.js - jstests/core/geo_mapreduce2.js diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml index 0bafa32a1e5..acf8ae53e96 100644 --- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml +++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml @@ -50,6 +50,7 @@ selector: - jstests/sharding/shard2.js - jstests/sharding/shard3.js - jstests/sharding/shard_collection_basic.js + - jstests/sharding/shard_existing_coll_chunk_count.js - jstests/sharding/sharding_balance1.js - jstests/sharding/sharding_balance2.js - jstests/sharding/sharding_balance3.js diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml index f4c741badd4..c7ff1b24395 100644 --- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml +++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml @@ -95,6 +95,8 @@ selector: # TODO (SERVER-27379): Cannot create views when featureCompatibilityVersion is 3.2. # Unblacklist when the featureCompatibilityVersion is 3.4 or higher. - jstests/sharding/views.js + # SERVER-20392: Reenable when backported to 3.4 and released as last-stable. + - jstests/sharding/shard_existing_coll_chunk_count.js executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml new file mode 100644 index 00000000000..388046d2782 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml @@ -0,0 +1,78 @@ +test_kind: js_test + +selector: + js_test: + roots: + - jstests/core/**/*.js + exclude_files: + # These tests are not expected to pass with replica-sets: + - jstests/core/dbadmin.js + - jstests/core/opcounters_write_cmd.js + - jstests/core/read_after_optime.js + - jstests/core/capped_update.js + # The connection_string_validation.js test does not expect the mongo shell to be using a replica + # set connection string. + - jstests/core/connection_string_validation.js + # These tests attempt to read from the "system.profile" collection, which may be missing entries + # if a write was performed on the primary of the replica set instead. + - jstests/core/*profile*.js + # The shellkillop.js test spawns a parallel shell without using startParallelShell() and + # therefore doesn't inherit the w="majority" write concern when performing its writes. + - jstests/core/shellkillop.js + exclude_with_any_tags: + ## + # The next three tags correspond to the special errors thrown by the + # set_read_and_write_concerns.js override when it refuses to replace the readConcern or + # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to + # be warranted. + ## + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "Cowardly refusing to override write concern of command: ..." + - assumes_write_concern_unchanged + # "Cowardly refusing to run test with overridden write concern when it uses a command that can + # only perform w=1 writes: ..." + - requires_collmod_command + - requires_eval_command + ## + # The next tag corresponds to the special error thrown by the set_read_preference_secondary.js + # override when it refuses to replace the readPreference of a particular command. Above each tag + # are the message(s) that cause the tag to be warranted. + ## + # "Cowardly refusing to override read preference of command: ..." + # "Cowardly refusing to run test with overridden read preference when it reads from a + # non-replicated collection: ..." + - assumes_read_preference_unchanged + +executor: + js_test: + config: + shell_options: + global_vars: + TestData: + defaultReadConcernLevel: local + eval: >- + testingReplication = true; + load('jstests/libs/override_methods/set_read_and_write_concerns.js'); + load('jstests/libs/override_methods/set_read_preference_secondary.js'); + readMode: commands + hooks: + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. + - class: CheckReplOplogs + - class: CheckReplDBHash + - class: ValidateCollections + - class: CleanEveryN + n: 20 + fixture: + class: ReplicaSetFixture + mongod_options: + set_parameters: + enableTestCommands: 1 + numInitialSyncAttempts: 1 + # This suite requires w="majority" writes to be applied on all secondaries. By using a 2-node + # replica set and having secondaries vote, the majority of the replica set is all nodes. + num_nodes: 2 + voting_secondaries: true + use_replica_set_connection_string: true diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index e4cadc8e006..d94cd438ee0 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -115,8 +115,8 @@ def mongos_program(logger, executable=None, process_kwargs=None, **kwargs): return _process.Process(logger, args, **process_kwargs) -def mongo_shell_program(logger, executable=None, filename=None, process_kwargs=None, - isMainTest=True, **kwargs): +def mongo_shell_program(logger, executable=None, connection_string=None, filename=None, + process_kwargs=None, isMainTest=True, **kwargs): """ Returns a Process instance that starts a mongo shell with arguments constructed from 'kwargs'. @@ -184,9 +184,22 @@ def mongo_shell_program(logger, executable=None, filename=None, process_kwargs=N if config.SHELL_WRITE_MODE is not None: kwargs["writeMode"] = config.SHELL_WRITE_MODE + if connection_string is not None: + # The --host and --port options are ignored by the mongo shell when an explicit connection + # string is specified. We remove these options to avoid any ambiguity with what server the + # logged mongo shell invocation will connect to. + if "port" in kwargs: + kwargs.pop("port") + + if "host" in kwargs: + kwargs.pop("host") + # Apply the rest of the command line arguments. _apply_kwargs(args, kwargs) + if connection_string is not None: + args.append(connection_string) + # Have the mongos shell run the specified file. args.append(filename) diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index f058372df8c..e8e37d2c004 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -5,7 +5,9 @@ Defines handlers for communicating with a buildlogger server. from __future__ import absolute_import import functools +import httplib import urllib2 +import json from . import handlers from . import loggers @@ -131,6 +133,47 @@ def new_test_id(build_id, build_config, test_filename, test_command): return response["id"] +class _LogsSplitter(object): + """Class with static methods used to split list of log lines into smaller batches.""" + + @staticmethod + def split_logs(log_lines, max_size): + """ + Splits the log lines into batches of size less than or equal to max_size. + + Args: + log_lines: A list of log lines. + max_size: The maximum size in bytes a batch of log lines can have in JSON. + Returns: + A list of list of log lines. Each item is a list is a list of log lines + satisfying the size requirement. + """ + if not max_size: + return [log_lines] + + def line_size(line): + """ + Computes the encoded JSON size of a log line as part of an array. + 2 is added to each string size to account for the array representation of the logs, + as each line is preceded by a '[' or a space and followed by a ',' or a ']'. + """ + return len(json.dumps(line, encoding="utf-8")) + 2 + + curr_logs = [] + curr_logs_size = 0 + split_logs = [] + for line in log_lines: + size = line_size(line) + if curr_logs_size + size > max_size: + split_logs.append(curr_logs) + curr_logs = [] + curr_logs_size = 0 + curr_logs.append(line) + curr_logs_size += size + split_logs.append(curr_logs) + return split_logs + + class _BaseBuildloggerHandler(handlers.BufferedHandler): """ Base class of the buildlogger handler for the global logs and the @@ -138,8 +181,8 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): """ def __init__(self, - build_id, build_config, + endpoint, capacity=_SEND_AFTER_LINES, interval_secs=_SEND_AFTER_SECS): """ @@ -157,8 +200,9 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): username, password) - self.build_id = build_id + self.endpoint = endpoint self.retry_buffer = [] + self.max_size = None def process_record(self, record): """ @@ -177,12 +221,55 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): """ Convenience method for subclasses to use when making POST requests. """ - return self.http_handler.post(*args, **kwargs) def _append_logs(self, log_lines): - raise NotImplementedError("_append_logs must be implemented by _BaseBuildloggerHandler" - " subclasses") + """ + Sends a POST request to the handlers endpoint with the logs that have been captured. + + Returns: + The number of log lines that have been successfully sent. + """ + lines_sent = 0 + for chunk in _LogsSplitter.split_logs(log_lines, self.max_size): + chunk_lines_sent = self.__append_logs_chunk(chunk) + lines_sent += chunk_lines_sent + if chunk_lines_sent < len(chunk): + # Not all lines have been sent. We stop here. + break + return lines_sent + + def __append_logs_chunk(self, log_lines_chunk): + """ + Sends a log lines chunk, handles 413 Request Entity Too Large errors and retries + if necessary. + + Returns: + The number of log lines that have been successfully sent. + """ + try: + self.post(self.endpoint, data=log_lines_chunk) + return len(log_lines_chunk) + except urllib2.HTTPError as err: + # Handle the "Request Entity Too Large" error, set the max size and retry. + if err.code == httplib.REQUEST_ENTITY_TOO_LARGE: + response_data = json.load(err, "utf-8") + if isinstance(response_data, dict) and "max_size" in response_data: + new_max_size = response_data["max_size"] + if self.max_size and new_max_size >= self.max_size: + loggers._BUILDLOGGER_FALLBACK.exception( + "Received an HTTP 413 code, but already had max_size set") + return 0 + loggers._BUILDLOGGER_FALLBACK.warning( + "Received an HTTP 413 code, updating the request max_size to %s", + new_max_size) + self.max_size = new_max_size + return self._append_logs(log_lines_chunk) + loggers._BUILDLOGGER_FALLBACK.exception("Encountered an error.") + return 0 + except: + loggers._BUILDLOGGER_FALLBACK.exception("Encountered an error.") + return 0 def _flush_buffer_with_lock(self, buf, close_called): """ @@ -196,9 +283,10 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): self.retry_buffer.extend(buf) - if self._append_logs(self.retry_buffer): - self.retry_buffer = [] - elif close_called: + nb_sent = self._append_logs(self.retry_buffer) + if nb_sent: + self.retry_buffer = self.retry_buffer[nb_sent:] + if close_called and self.retry_buffer: # Request to the buildlogger server returned an error, so use the fallback logger to # avoid losing the log messages entirely. for (_, message) in self.retry_buffer: @@ -215,29 +303,14 @@ class BuildloggerTestHandler(_BaseBuildloggerHandler): Buildlogger handler for the test logs. """ - def __init__(self, build_id, build_config, test_id, **kwargs): - """ - Initializes the buildlogger handler with the build id, test id, - and credentials. - """ - - _BaseBuildloggerHandler.__init__(self, build_id, build_config, **kwargs) - - self.test_id = test_id - - @_log_on_error - def _append_logs(self, log_lines): - """ - Sends a POST request to the APPEND_TEST_LOGS_ENDPOINT with the - logs that have been captured. - """ + def __init__(self, build_config, build_id, test_id, + capacity=_SEND_AFTER_LINES, interval_secs=_SEND_AFTER_SECS): + """Initializes the buildlogger handler with the credentials, build id, and test id.""" endpoint = APPEND_TEST_LOGS_ENDPOINT % { - "build_id": self.build_id, - "test_id": self.test_id, + "build_id": build_id, + "test_id": test_id, } - - response = self.post(endpoint, data=log_lines) - return response is not None + _BaseBuildloggerHandler.__init__(self, build_config, endpoint, capacity, interval_secs) @_log_on_error def _finish_test(self, failed=False): @@ -245,12 +318,7 @@ class BuildloggerTestHandler(_BaseBuildloggerHandler): Sends a POST request to the APPEND_TEST_LOGS_ENDPOINT with the test status. """ - endpoint = APPEND_TEST_LOGS_ENDPOINT % { - "build_id": self.build_id, - "test_id": self.test_id, - } - - self.post(endpoint, headers={ + self.post(self.endpoint, headers={ "X-Sendlogs-Test-Done": "true", "X-Sendlogs-Test-Failed": "true" if failed else "false", }) @@ -271,12 +339,8 @@ class BuildloggerGlobalHandler(_BaseBuildloggerHandler): Buildlogger handler for the global logs. """ - @_log_on_error - def _append_logs(self, log_lines): - """ - Sends a POST request to the APPEND_GLOBAL_LOGS_ENDPOINT with - the logs that have been captured. - """ - endpoint = APPEND_GLOBAL_LOGS_ENDPOINT % {"build_id": self.build_id} - response = self.post(endpoint, data=log_lines) - return response is not None + def __init__(self, build_config, build_id, + capacity=_SEND_AFTER_LINES, interval_secs=_SEND_AFTER_SECS): + """Initializes the buildlogger handler with the credentials and build id.""" + endpoint = APPEND_GLOBAL_LOGS_ENDPOINT % {"build_id": build_id} + _BaseBuildloggerHandler.__init__(self, build_config, endpoint, capacity, interval_secs) diff --git a/buildscripts/resmokelib/logging/config.py b/buildscripts/resmokelib/logging/config.py index c3960bbafd3..c3216b2d879 100644 --- a/buildscripts/resmokelib/logging/config.py +++ b/buildscripts/resmokelib/logging/config.py @@ -67,8 +67,8 @@ def apply_buildlogger_global_handler(logger, logging_config, build_id=None, buil log_format = logger_info.get("format", _DEFAULT_FORMAT) formatter = formatters.ISO8601Formatter(fmt=log_format) - handler = buildlogger.BuildloggerGlobalHandler(build_id, - build_config, + handler = buildlogger.BuildloggerGlobalHandler(build_config, + build_id, **handler_info) handler.setFormatter(formatter) else: @@ -98,8 +98,8 @@ def apply_buildlogger_test_handler(logger, log_format = logger_info.get("format", _DEFAULT_FORMAT) formatter = formatters.ISO8601Formatter(fmt=log_format) - handler = buildlogger.BuildloggerTestHandler(build_id, - build_config, + handler = buildlogger.BuildloggerTestHandler(build_config, + build_id, test_id, **handler_info) handler.setFormatter(formatter) diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index b4b0066a5aa..83418502ead 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -79,13 +79,22 @@ class Fixture(object): """ return True - def get_connection_string(self): + def get_internal_connection_string(self): """ Returns the connection string for this fixture. This is NOT a driver connection string, but a connection string of the format expected by the mongo::ConnectionString class. """ - raise NotImplementedError("get_connection_string must be implemented by Fixture subclasses") + raise NotImplementedError( + "get_internal_connection_string must be implemented by Fixture subclasses") + + def get_driver_connection_url(self): + """ + Return the mongodb connection string as defined here: + https://docs.mongodb.com/manual/reference/connection-string/ + """ + raise NotImplementedError( + "get_driver_connection_url must be implemented by Fixture subclasses") def __str__(self): return "%s (Job #%d)" % (self.__class__.__name__, self.job_num) diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py index 469c7ac0816..fb444cfe097 100644 --- a/buildscripts/resmokelib/testing/fixtures/masterslave.py +++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py @@ -5,7 +5,6 @@ Master/slave fixture for executing JSTests against. from __future__ import absolute_import import os.path -import socket import pymongo @@ -160,6 +159,12 @@ class MasterSlaveFixture(interface.ReplFixture): mongod_options = self.mongod_options.copy() mongod_options.update(self.slave_options) mongod_options["slave"] = "" - mongod_options["source"] = "%s:%d" % (socket.gethostname(), self.port) + mongod_options["source"] = self.master.get_internal_connection_string() mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "slave") return self._new_mongod(mongod_logger, mongod_options) + + def get_internal_connection_string(self): + return self.master.get_internal_connection_string() + + def get_driver_connection_url(self): + return self.master.get_driver_connection_url() diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index 141bbd60ea1..0ec3de30280 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -37,7 +37,8 @@ class ReplicaSetFixture(interface.ReplFixture): write_concern_majority_journal_default=None, auth_options=None, replset_config_options=None, - voting_secondaries=True): + voting_secondaries=False, + use_replica_set_connection_string=False): interface.ReplFixture.__init__(self, logger, job_num) @@ -50,6 +51,7 @@ class ReplicaSetFixture(interface.ReplFixture): self.auth_options = auth_options self.replset_config_options = utils.default_if_none(replset_config_options, {}) self.voting_secondaries = voting_secondaries + self.use_replica_set_connection_string = use_replica_set_connection_string # The dbpath in mongod_options is used as the dbpath prefix for replica set members and # takes precedence over other settings. The ShardedClusterFixture uses this parameter to @@ -97,7 +99,7 @@ class ReplicaSetFixture(interface.ReplFixture): # Initiate the replica set. members = [] for (i, node) in enumerate(self.nodes): - member_info = {"_id": i, "host": node.get_connection_string()} + member_info = {"_id": i, "host": node.get_internal_connection_string()} if i > 0: member_info["priority"] = 0 if i >= 7 or not self.voting_secondaries: @@ -107,7 +109,7 @@ class ReplicaSetFixture(interface.ReplFixture): members.append(member_info) if self.initial_sync_node: members.append({"_id": self.initial_sync_node_idx, - "host": self.initial_sync_node.get_connection_string(), + "host": self.initial_sync_node.get_internal_connection_string(), "priority": 0, "hidden": 1, "votes": 0}) @@ -291,11 +293,27 @@ class ReplicaSetFixture(interface.ReplFixture): return logging.loggers.new_logger(logger_name, parent=self.logger) - def get_connection_string(self): + def get_internal_connection_string(self): if self.replset_name is None: - raise ValueError("Must call setup() before calling get_connection_string()") + raise ValueError("Must call setup() before calling get_internal_connection_string()") - conn_strs = [node.get_connection_string() for node in self.nodes] + conn_strs = [node.get_internal_connection_string() for node in self.nodes] if self.initial_sync_node: - conn_strs.append(self.initial_sync_node.get_connection_string()) + conn_strs.append(self.initial_sync_node.get_internal_connection_string()) return self.replset_name + "/" + ",".join(conn_strs) + + def get_driver_connection_url(self): + if self.replset_name is None: + raise ValueError("Must call setup() before calling get_driver_connection_url()") + + if self.use_replica_set_connection_string: + # We use a replica set connection string when all nodes are electable because we + # anticipate the client will want to gracefully handle any failovers. + conn_strs = [node.get_internal_connection_string() for node in self.nodes] + if self.initial_sync_node: + conn_strs.append(self.initial_sync_node.get_internal_connection_string()) + return "mongodb://" + ",".join(conn_strs) + "/?replicaSet=" + self.replset_name + else: + # We return a direct connection to the expected pimary when only the first node is + # electable because we want the client to error out if a stepdown occurs. + return self.nodes[0].get_driver_connection_url() diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index ac7e597f24b..2e2db535d6d 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -170,11 +170,14 @@ class ShardedClusterFixture(interface.Fixture): all(shard.is_running() for shard in self.shards) and self.mongos is not None and self.mongos.is_running()) - def get_connection_string(self): + def get_internal_connection_string(self): if self.mongos is None: - raise ValueError("Must call setup() before calling get_connection_string()") + raise ValueError("Must call setup() before calling get_internal_connection_string()") - return "%s:%d" % (socket.gethostname(), self.mongos.port) + return self.mongos.get_internal_connection_string() + + def get_driver_connection_url(self): + return "mongodb://" + self.get_internal_connection_string() def _new_configsvr(self): """ @@ -229,16 +232,11 @@ class ShardedClusterFixture(interface.Fixture): mongos_logger = logging.loggers.new_logger(logger_name, parent=self.logger) mongos_options = copy.deepcopy(self.mongos_options) - configdb_hostname = socket.gethostname() if self.separate_configsvr: - configdb_replset = ShardedClusterFixture._CONFIGSVR_REPLSET_NAME - configdb_port = self.configsvr.port - mongos_options["configdb"] = "%s/%s:%d" % (configdb_replset, - configdb_hostname, - configdb_port) + mongos_options["configdb"] = self.configsvr.get_internal_connection_string() else: - mongos_options["configdb"] = "%s:%d" % (configdb_hostname, self.shards[0].port) + mongos_options["configdb"] = "localhost:%d" % (self.shards[0].port) return _MongoSFixture(mongos_logger, self.job_num, @@ -254,9 +252,9 @@ class ShardedClusterFixture(interface.Fixture): for more details. """ - hostname = socket.gethostname() - self.logger.info("Adding %s:%d as a shard..." % (hostname, shard.port)) - client.admin.command({"addShard": "%s:%d" % (hostname, shard.port)}) + connection_string = shard.get_internal_connection_string() + self.logger.info("Adding %s as a shard...", connection_string) + client.admin.command({"addShard": connection_string}) class _MongoSFixture(interface.Fixture): @@ -356,3 +354,12 @@ class _MongoSFixture(interface.Fixture): def is_running(self): return self.mongos is not None and self.mongos.poll() is None + + def get_internal_connection_string(self): + if self.mongos is None: + raise ValueError("Must call setup() before calling get_internal_connection_string()") + + return "localhost:%d" % self.port + + def get_driver_connection_url(self): + return "mongodb://" + self.get_internal_connection_string() diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index ba62b3d2b8c..bc69775c285 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -7,7 +7,6 @@ from __future__ import absolute_import import os import os.path import shutil -import socket import time import pymongo @@ -146,8 +145,11 @@ class MongoDFixture(interface.Fixture): def is_running(self): return self.mongod is not None and self.mongod.poll() is None - def get_connection_string(self): + def get_internal_connection_string(self): if self.mongod is None: - raise ValueError("Must call setup() before calling get_connection_string()") + raise ValueError("Must call setup() before calling get_internal_connection_string()") - return "%s:%d" % (socket.gethostname(), self.port) + return "localhost:%d" % self.port + + def get_driver_connection_url(self): + return "mongodb://" + self.get_internal_connection_string() diff --git a/buildscripts/resmokelib/testing/testcases.py b/buildscripts/resmokelib/testing/testcases.py index b4029fc6ea8..21d35215a29 100644 --- a/buildscripts/resmokelib/testing/testcases.py +++ b/buildscripts/resmokelib/testing/testcases.py @@ -187,7 +187,7 @@ class CPPIntegrationTestCase(TestCase): def configure(self, fixture, *args, **kwargs): TestCase.configure(self, fixture, *args, **kwargs) - self.program_options["connectionString"] = self.fixture.get_connection_string() + self.program_options["connectionString"] = self.fixture.get_internal_connection_string() def run_test(self): try: @@ -362,6 +362,19 @@ class JSTestCase(TestCase): # Directory already exists. pass + process_kwargs = self.shell_options.get("process_kwargs", {}).copy() + + if "KRB5_CONFIG" in process_kwargs and "KRB5CCNAME" not in process_kwargs: + # Use a job-specific credential cache for JavaScript tests involving Kerberos. + krb5_dir = os.path.join(data_dir, "krb5") + try: + os.makedirs(krb5_dir) + except os.error: + pass + process_kwargs["KRB5CCNAME"] = "DIR:" + os.path.join(krb5_dir, ".") + + self.shell_options["process_kwargs"] = process_kwargs + def _get_data_dir(self, global_vars): """ Returns the value that the mongo shell should set for the @@ -408,11 +421,13 @@ class JSTestCase(TestCase): is_main_test = True if thread_id > 0: is_main_test = False - return core.programs.mongo_shell_program(logger, - executable=self.shell_executable, - filename=self.js_filename, - isMainTest=is_main_test, - **self.shell_options) + return core.programs.mongo_shell_program( + logger, + executable=self.shell_executable, + filename=self.js_filename, + connection_string=self.fixture.get_driver_connection_url(), + isMainTest=is_main_test, + **self.shell_options) def _run_test_in_thread(self, thread_id): # Make a logger for each thread. diff --git a/buildscripts/scons.py b/buildscripts/scons.py index e7860162eb6..b0b9cfa834e 100755 --- a/buildscripts/scons.py +++ b/buildscripts/scons.py @@ -1,16 +1,26 @@ -#!/usr/bin/python +#!/usr/bin/env python2 + +from __future__ import print_function import os -import subprocess import sys SCONS_VERSION = os.environ.get('SCONS_VERSION', "2.5.0") -mongodb_root = os.path.dirname(os.path.dirname(__file__)) -scons_dir = os.path.join(mongodb_root, 'src', 'third_party', 'scons-' + SCONS_VERSION) +mongodb_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +scons_dir = os.path.join(mongodb_root, 'src', 'third_party','scons-' + SCONS_VERSION, + 'scons-local-' + SCONS_VERSION) + +if not os.path.exists(scons_dir): + print("Could not find SCons in '%s'" % (scons_dir)) + sys.exit(1) + +sys.path = [scons_dir] + sys.path + +try: + import SCons.Script +except ImportError: + print("Could not find SCons in '%s'" % (scons_dir)) + sys.exit(1) -if sys.platform == 'win32': - args = [sys.executable, os.path.join(scons_dir, 'scons.py')] + sys.argv[1:] - sys.exit(subprocess.call(args)) -else: - os.execv(os.path.join(scons_dir, 'scons.py'), sys.argv) +SCons.Script.main() diff --git a/buildscripts/tests/resmokelib/logging/test_buildlogger.py b/buildscripts/tests/resmokelib/logging/test_buildlogger.py new file mode 100644 index 00000000000..7bc705948db --- /dev/null +++ b/buildscripts/tests/resmokelib/logging/test_buildlogger.py @@ -0,0 +1,67 @@ +"""Unit tests for the buildscripts.resmokelib.logging.buildlogger module.""" + +from __future__ import absolute_import + +import json +import unittest + +from buildscripts.resmokelib.logging import buildlogger + + +class TestLogsSplitter(unittest.TestCase): + """Unit tests for the _LogsSplitter class.""" + + def test_split_no_logs(self): + logs = [] + max_size = 10 + self.assertEqual([[]], buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def test_split_no_max_size(self): + logs = self.__generate_logs(size=30) + max_size = None + self.assertEqual([logs], buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def test_split_max_size_smaller(self): + logs = self.__generate_logs(size=20) + max_size = 30 + self.assertEqual([logs], buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def test_split_max_size_equal(self): + logs = self.__generate_logs(size=30) + max_size = 30 + self.assertEqual([logs], buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def test_split_max_size_larger(self): + logs = self.__generate_logs(size=31) + max_size = 30 + self.assertEqual( + [logs[0:-1], logs[-1:]], + buildlogger._LogsSplitter.split_logs(logs, max_size)) + + logs = self.__generate_logs(size=149) + max_size = 19 + self.assertEqual( + [logs[0:3], logs[3:6], logs[6:9], logs[9:12], logs[12:15], + logs[15:18], logs[18:21], logs[21:24], logs[24:27], logs[27:]], + buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def check_split_sizes(self, splits, max_size): + for split in splits: + self.assertTrue(TestLogsSplitter.size(split) <= max_size) + + def __generate_logs(self, size): + # The size of [ "x" ] is 5. This is the minimum size we generate. + self.assertTrue(size >= 5) + # Each new "x" adds 5 to the size. + nb_lines = size / 5 + # Each additional "x" on a line adds 1 to the size. + last_line_extra = size % 5 + logs = ["x"] * nb_lines + logs[-1] += "x" * last_line_extra + self.assertEqual(size, TestLogsSplitter.size(logs)) + return logs + + @staticmethod + def size(logs): + """Returns the size of the log lines when represented in JSON.""" + return len(json.dumps(logs, encoding="utf-8")) |
