diff options
| author | Apollon Oikonomopoulos <apoikos@debian.org> | 2018-12-03 20:45:58 +0200 |
|---|---|---|
| committer | Apollon Oikonomopoulos <apoikos@debian.org> | 2018-12-03 20:45:58 +0200 |
| commit | 239aabeb53a8dcd45eac7d069e0cb0180b4bc412 (patch) | |
| tree | c1642d9d026783a026d41a2ae3e1920fa31edb71 | |
| parent | 3896a4a134ae19f25de14727862898cee19aa2e3 (diff) | |
New upstream version 3.4.18upstream/3.4.18
225 files changed, 5967 insertions, 1958 deletions
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index d94cd438ee0..63e2b12de87 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -116,7 +116,7 @@ def mongos_program(logger, executable=None, process_kwargs=None, **kwargs): def mongo_shell_program(logger, executable=None, connection_string=None, filename=None, - process_kwargs=None, isMainTest=True, **kwargs): + process_kwargs=None, **kwargs): """ Returns a Process instance that starts a mongo shell with arguments constructed from 'kwargs'. @@ -148,7 +148,6 @@ def mongo_shell_program(logger, executable=None, connection_string=None, filenam # Only use 'opt_default' if the property wasn't set in the YAML configuration. test_data[opt_name] = opt_default - test_data["isMainTest"] = isMainTest global_vars["TestData"] = test_data # Pass setParameters for mongos and mongod through TestData. The setParameter parsing in diff --git a/buildscripts/resmokelib/testing/hooks.py b/buildscripts/resmokelib/testing/hooks.py index f6773c1d682..05a24117232 100644 --- a/buildscripts/resmokelib/testing/hooks.py +++ b/buildscripts/resmokelib/testing/hooks.py @@ -172,7 +172,7 @@ class JsCustomBehavior(CustomBehavior): test_report.addFailure(self.hook_test_case, sys.exc_info()) raise errors.StopExecution(err.args[0]) except self.hook_test_case.failureException as err: - self.hook_test_case.logger.exception("{0} failed".format(description)) + self.hook_test_case.logger.error("{0} failed".format(description)) test_report.addFailure(self.hook_test_case, sys.exc_info()) raise errors.StopExecution(err.args[0]) else: diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py index c8be906dd3a..ad324bd0d1d 100644 --- a/buildscripts/resmokelib/testing/job.py +++ b/buildscripts/resmokelib/testing/job.py @@ -115,7 +115,6 @@ class Job(object): finally: success = self.report.find_test_info(test).status == "pass" - self.archival.archive(self.logger, test, success) if self.archival: self.archival.archive(self.logger, test, success) diff --git a/buildscripts/resmokelib/testing/testcases.py b/buildscripts/resmokelib/testing/testcases.py index 78110ff680d..44bb0d767ea 100644 --- a/buildscripts/resmokelib/testing/testcases.py +++ b/buildscripts/resmokelib/testing/testcases.py @@ -344,9 +344,6 @@ class JSTestCase(TestCase): test_data = global_vars.get("TestData", {}).copy() test_data["minPort"] = core.network.PortAllocator.min_test_port(fixture.job_num) test_data["maxPort"] = core.network.PortAllocator.max_test_port(fixture.job_num) - # Marks the main test when multiple test clients are run concurrently, to notify the test - # of any code that should only be run once. If there is only one client, it is the main one. - test_data["isMainTest"] = True global_vars["TestData"] = test_data self.shell_options["global_vars"] = global_vars @@ -419,19 +416,32 @@ class JSTestCase(TestCase): raise t._get_exception() def _make_process(self, logger=None, thread_id=0): + # Since _make_process() is called by each thread, we make a shallow copy of the mongo shell + # options to avoid modifying the shared options for the JSTestCase. + shell_options = self.shell_options.copy() + global_vars = shell_options["global_vars"].copy() + test_data = global_vars["TestData"].copy() + + # We set a property on TestData to mark the main test when multiple clients are going to run + # concurrently in case there is logic within the test that must execute only once. We also + # set a property on TestData to indicate how many clients are going to run the test so they + # can avoid executing certain logic when there may be other operations running concurrently. + is_main_test = thread_id == 0 + test_data["isMainTest"] = is_main_test + test_data["numTestClients"] = self.num_clients + + global_vars["TestData"] = test_data + shell_options["global_vars"] = global_vars + # If logger is none, it means that it's not running in a thread and thus logger should be # set to self.logger. logger = utils.default_if_none(logger, self.logger) - is_main_test = True - if thread_id > 0: - is_main_test = False return core.programs.mongo_shell_program( logger, executable=self.shell_executable, filename=self.js_filename, connection_string=self.fixture.get_driver_connection_url(), - isMainTest=is_main_test, - **self.shell_options) + **shell_options) def _run_test_in_thread(self, thread_id): # Make a logger for each thread. diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 5992bb0f57b..b558f478e62 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -402,6 +402,31 @@ functions: https_validate_certificates = False EOF + "call BF Suggestion service": + command: shell.exec + params: + working_dir: src + shell: bash + silent: true + script: | + report_file="report.json" + # Check if the report file exists and has failures. + if [ -f $report_file ] && grep -Eq "\"failures\": [1-9]" $report_file; then + # Calling the BF Suggestion server endpoint to start feature extraction. + payload="{\"task_id\": \"${task_id}\", \"execution\": ${execution}}" + echo "Sending task info to the BF suggestion service" + # The --user option is passed through stdin to avoid showing in process list. + user_option="--user ${bfsuggestion_user}:${bfsuggestion_password}" + curl --header "Content-Type: application/json" \ + --data "$payload" \ + --max-time 10 \ + --silent \ + --show-error \ + --config - \ + https://bfsuggestion.corp.mongodb.com/tasks <<< $user_option + echo "Request to BF Suggestion service status: $?" + fi + "upload debugsymbols" : &upload_debugsymbols command: s3.put params: @@ -767,6 +792,10 @@ functions: set +o errexit npm test > npm_test-${task_id}.log 2>&1 if [ $? -ne 0 ]; then + echo "jstestfuzz self-tests failure" + branch=$(git symbolic-ref --short HEAD) + commit=$(git show -s --pretty=format:"%h - %an, %ar: %s") + echo "Git branch: $branch, commit: $commit" which node node --version which npm @@ -955,8 +984,6 @@ pre: params: system_log: true script: | - set -o errexit - set -o verbose ulimit -a # Clear the dmesg ring buffer. The "post" phase will check dmesg for OOM messages. @@ -969,7 +996,6 @@ pre: fi - post: - command: attach.results params: @@ -979,6 +1005,7 @@ post: ignore_artifacts_for_spawn: false files: - src/archive.json + - func: "call BF Suggestion service" - func: "kill processes" # Print out any Out of Memory killed process messages. - command: shell.exec @@ -2457,7 +2484,7 @@ tasks: - func: "do setup" - func: "run tests" vars: - resmoke_args: --suites=no_passthrough --storageEngine=mmapv1 + resmoke_args: --suites=no_passthrough --storageEngine=mmapv1 --excludeWithAnyTags=requires_document_locking run_multiple_jobs: true - <<: *task_template @@ -3137,7 +3164,7 @@ tasks: /usr/local/bin/notary-client.py --key-name "server-3.4" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} mongodb-win32-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${ext|tar.gz} if [ "${has_packages|}" = "true" ]; then - CURATOR_RELEASE=${curator_release|"ea8d75dcc1a587111e7418e2428fb67e267af9fe"} + CURATOR_RELEASE=${curator_release|"37d930ea2eeb27acb3782f1aa4d69da1e9dac223"} curl -L -O http://boxes.10gen.com/build/curator/curator-dist-rhel70-$CURATOR_RELEASE.tar.gz tar -zxvf curator-dist-rhel70-$CURATOR_RELEASE.tar.gz ./curator repo --config ./etc/repo_config.yaml --distro ${packager_distro} --edition ${repo_edition} --version ${version} --arch ${packager_arch} --packages repo @@ -4216,20 +4243,9 @@ buildvariants: - name: parallel_WT - name: parallel_compatibility_WT - name: concurrency_WT - distros: - # There is a known performance issue with the ubuntu1604-arm64-1.linaro.build.10gen.cc host - # that's part of the ubuntu1604-arm64-large pool. We run the concurrency tests only on the - # hosts from the ubuntu1604-arm64-small pool to work around this issue. - - ubuntu1604-arm64-small - name: concurrency_replication_WT - distros: - - ubuntu1604-arm64-small - name: concurrency_sharded_WT - distros: - - ubuntu1604-arm64-small - name: concurrency_simultaneous_WT - distros: - - ubuntu1604-arm64-small - name: replica_sets_WT - name: replica_sets_auth - name: replica_sets_jscore_passthrough_WT @@ -7415,7 +7431,7 @@ buildvariants: test_flags: --storageEngine=ephemeralForTest compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --dbg=off --opt=on CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy num_jobs_available: $(grep -c ^processor /proc/cpuinfo) - variant_excluded_flags: requires_persistence,requires_fsync,SERVER-21420,SERVER-21658,requires_journaling + variant_excluded_flags: requires_persistence,requires_fsync,SERVER-21420,SERVER-21658,requires_journaling,requires_document_locking use_scons_cache: true gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' tooltags: "" diff --git a/etc/longevity.yml b/etc/longevity.yml index c045bc83f62..79fe5364df2 100644 --- a/etc/longevity.yml +++ b/etc/longevity.yml @@ -299,7 +299,9 @@ modules: buildvariants: - name: linux-wt-shard display_name: Linux WT Shard - batchtime: 40320 # 4 weeks + # We set an exceptionally large batchtime because the intention is for builds on this Evergreen + # project to only be triggered manually. + batchtime: 524160 # 52 weeks modules: &modules - dsi expansions: @@ -310,6 +312,7 @@ buildvariants: storageEngine: wiredTiger use_scons_cache: true project: &project longevity-v3.4 + platform: linux run_on: - "rhel70-perf-longevity" tasks: @@ -320,7 +323,9 @@ buildvariants: - name: linux-mmapv1-shard display_name: Linux MMAPv1 Shard - batchtime: 40320 # 4 week + # We set an exceptionally large batchtime because the intention is for builds on this Evergreen + # project to only be triggered manually. + batchtime: 524160 # 52 weeks modules: *modules expansions: compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy diff --git a/etc/perf.yml b/etc/perf.yml index 1d485290d82..47962eba3f8 100644 --- a/etc/perf.yml +++ b/etc/perf.yml @@ -58,6 +58,11 @@ functions: rm -rf ./dsi mkdir -p ./src git clone git@github.com:10gen/dsi.git + # get the mongo source, note the s3.get calls put the + # exe files in their respective make locations. + - command: git.get_project + params: + directory: src "start server": - command: shell.exec params: @@ -135,9 +140,43 @@ functions: script: | set -e set -v - virtualenv ./venv - source ./venv/bin/activate - pip install -r ../dsi/requirements/analysis.txt + ../dsi/run-dsi setup + - command: shell.exec + params: + working_dir: src + script: | + cat > bootstrap.yml <<EOF + infrastructure_provisioning: ${cluster} + platform: ${platform} + mongodb_setup: ${setup} + storageEngine: ${storageEngine} + test_control: ${test} + production: true + EOF + + cat > runtime.yml <<EOF + # evergreen default expansions + is_patch: ${is_patch} + task_id: ${task_id} + EOF + + cp ../dsi/configurations/analysis/analysis.common.yml analysis.yml + - command: shell.exec + params: + working_dir: src + silent: true + script: | + cat > runtime_secret.yml <<EOF + dsi_analysis_atlas_user: "${dsi_analysis_atlas_user}" + dsi_analysis_atlas_pw: "${dsi_analysis_atlas_pw}" + EOF + chmod 400 runtime_secret.yml + - command: shell.exec + params: + working_dir: src + script: | + set -v + ../dsi/run-dsi detect-changes - command: json.get_history params: task: ${task_name} @@ -150,23 +189,6 @@ functions: file: "src/tags.json" name: "perf" - command: shell.exec - # generate dashboard data - type : test - params: - working_dir: src - silent: true - script: | - set -o errexit - source ./venv/bin/activate - REFTAGS="3.4.13-Baseline 3.2.17-Baseline 3.0.15-Baseline" # These are project specific - # Project Opts are project specific, but don't match expansion from elsewhere. - PROJECT_OPTS="--overrideFile ../dsi/analysis/${branch_name}/perf_override.json --project_id performance" - python -u ../dsi/analysis/dashboard_gen.py --rev ${revision} -f history.json -t tags.json --refTag $REFTAGS $PROJECT_OPTS --variant ${build_variant} --task ${task_name} --jira-user ${perf_jira_user} --jira-password ${perf_jira_pw} || true - - command: "json.send" - params: - name: "dashboard" - file: "src/dashboard.json" - - command: shell.exec type : test params: working_dir: src @@ -453,7 +475,17 @@ buildvariants: compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --release --variables-files=etc/scons/mongodbtoolchain_gcc.vars mongod_exec_wrapper: &exec_wrapper "numactl --physcpubind=4,5,6,7 -i 1" perf_exec_wrapper: &perf_wrapper "numactl --physcpubind=1,2,3 -i 0" - mongod_flags: "--storageEngine=inMemory --logpath ./mongod.log --fork --syncdelay 0 --setParameter ttlMonitorEnabled=false --setParameter diagnosticDataCollectionEnabled=false --inMemoryEngineConfigString 'eviction=(threads_min=1),' --inMemorySizeGB 60 --auth" + mongod_flags: >- + --auth + --fork + --inMemoryEngineConfigString 'eviction=(threads_min=1),' + --inMemorySizeGB 60 + --logpath ./mongod.log + --setParameter diagnosticDataCollectionEnabled=false + --setParameter enableTestCommands=1 + --setParameter ttlMonitorEnabled=false + --storageEngine inMemory + --syncdelay 0 use_scons_cache: true project: &project perf-3.4 run_on: @@ -479,7 +511,16 @@ buildvariants: expansions: mongod_exec_wrapper: *exec_wrapper perf_exec_wrapper: *perf_wrapper - mongod_flags: "--storageEngine=mmapv1 --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --setParameter diagnosticDataCollectionEnabled=false --auth" + mongod_flags: >- + --auth + --fork + --logpath ./mongod.log + --nojournal + --setParameter diagnosticDataCollectionEnabled=false + --setParameter enableTestCommands=1 + --setParameter ttlMonitorEnabled=false + --storageEngine mmapv1 + --syncdelay 0 project: *project run_on: - "centos6-perf" @@ -501,7 +542,19 @@ buildvariants: expansions: mongod_exec_wrapper: *exec_wrapper perf_exec_wrapper: *perf_wrapper - mongod_flags: "--replSet=test --storageEngine=inMemory --logpath ./mongod.log --fork --syncdelay 0 --setParameter ttlMonitorEnabled=false --setParameter diagnosticDataCollectionEnabled=false --inMemoryEngineConfigString 'eviction=(threads_min=1),' --inMemorySizeGB 60 --auth --oplogSize 30000" + mongod_flags: >- + --auth + --fork + --inMemoryEngineConfigString 'eviction=(threads_min=1),' + --inMemorySizeGB 60 + --logpath ./mongod.log + --oplogSize 30000 + --replSet test + --setParameter diagnosticDataCollectionEnabled=false + --setParameter enableTestCommands=1 + --setParameter ttlMonitorEnabled=false + --storageEngine inMemory + --syncdelay 0 project: *project run_on: - "centos6-perf" @@ -517,7 +570,18 @@ buildvariants: expansions: mongod_exec_wrapper: *exec_wrapper perf_exec_wrapper: *perf_wrapper - mongod_flags: "--replSet=test --storageEngine=mmapv1 --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --setParameter diagnosticDataCollectionEnabled=false --auth --oplogSize 100000" + mongod_flags: >- + --auth + --fork + --logpath ./mongod.log + --nojournal + --oplogSize 100000 + --replSet test + --setParameter diagnosticDataCollectionEnabled=false + --setParameter enableTestCommands=1 + --setParameter ttlMonitorEnabled=false + --storageEngine mmapv1 + --syncdelay 0 project: *project run_on: - "centos6-perf" diff --git a/etc/system_perf.yml b/etc/system_perf.yml index 930ced96aba..7a6de6f99c0 100644 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -207,6 +207,7 @@ functions: aws_secret_key: "${terraform_secret}" perf_jira_user: "${perf_jira_user}" perf_jira_pw: "${perf_jira_pw}" + dsi_analysis_atlas_user: "${dsi_analysis_atlas_user}" dsi_analysis_atlas_pw: "${dsi_analysis_atlas_pw}" EOF chmod 400 runtime_secret.yml @@ -267,7 +268,7 @@ functions: script: | set -o errexit set -o verbose - TAG="3.2.20-Baseline" + TAG="3.4.17-Baseline" OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json" python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} @@ -456,7 +457,7 @@ buildvariants: # proper artifacts directly from that project, we should do that and remove these tasks. - name: compile-rhel70 display_name: Compile on rhel70 - batchtime: 1440 # 24 hours + batchtime: 10080 # 7 days modules: - wtdevelop expansions: diff --git a/jstests/core/apply_ops_invalid_index_spec.js b/jstests/core/apply_ops_invalid_index_spec.js new file mode 100644 index 00000000000..2ca88081fa7 --- /dev/null +++ b/jstests/core/apply_ops_invalid_index_spec.js @@ -0,0 +1,107 @@ +/** + * Tests how applyOps handles index specs with unknown fields. + * + * We subject index specs with version 2 or later to stricter validation than version 1 index specs. + * When given an index spec with an unrecognized field, applyOps will reject v:2 indexes with an + * InvalidIndexSpecificationOption error while v:1 indexes are accepted as-is. + * + * @tags: [ + * requires_non_retryable_commands, + * requires_fastcount, + * + * # applyOps uses the oplog which requires replication support. + * requires_replication, + * ] + */ + +(function() { + 'use strict'; + + const t = db.apply_ops_invalid_index_spec; + t.drop(); + + const collNs = t.getFullName(); + const cmdNs = db.getName() + '.$cmd'; + const systemIndexesNs = db.getCollection('system.indexes').getFullName(); + + assert.commandWorked(db.createCollection(t.getName())); + assert.writeOK(t.save({_id: 100, a: 100})); + + // Tests that db.collection.createIndex() fails when given an index spec containing an unknown + // field. + assert.commandFailedWithCode(t.createIndex({a: 1}, {v: 2, name: 'a_1_base_v2', unknown: 1}), + ErrorCodes.InvalidIndexSpecificationOption); + assert.commandFailedWithCode(t.createIndex({a: 1}, {v: 1, name: 'a_1_base_v1', unknown: 1}), + ErrorCodes.InvalidIndexSpecificationOption); + + // Inserting a v:2 index directly into system.indexes with an unknown field in the index + // spec should return an error. + assert.commandFailedWithCode(db.adminCommand({ + applyOps: [{ + op: 'i', + ns: systemIndexesNs, + o: {v: 2, key: {a: 1}, name: 'a_1_system_v2', ns: collNs, unknown: 1}, + }], + }), + ErrorCodes.InvalidIndexSpecificationOption); + + // Inserting a v:1 index directly into system.indexes with an unknown field in the index spec + // should ignore the unrecognized field and create the index. + assert.commandWorked(db.adminCommand({ + applyOps: [{ + op: 'i', + ns: systemIndexesNs, + o: {v: 1, key: {a: 1}, name: 'a_1_system_v1', ns: collNs, unknown: 1}, + }], + })); + + // + // Background indexes should be subject to the same level of validation as foreground indexes. + // + + // Inserting a background index directly into system.indexes with a bad index key pattern should + // return an error. + assert.commandFailedWithCode(db.adminCommand({ + applyOps: [{ + op: 'i', + ns: systemIndexesNs, + o: {key: {b: 'sideways'}, name: 'b_1_bg_system_v2', ns: collNs, background: true}, + }], + }), + ErrorCodes.CannotCreateIndex); + + // Inserting a v:2 background index directly into system.indexes with an unknown field in the + // index spec should return an error. + assert.commandFailedWithCode(db.adminCommand({ + applyOps: [{ + op: 'i', + ns: systemIndexesNs, + o: { + v: 2, + key: {b: 1}, + name: 'b_1_bg_system_v2', + ns: collNs, + background: true, + unknown: true, + }, + }], + }), + ErrorCodes.InvalidIndexSpecificationOption); + + // Inserting a background v:1 index directly into system.indexes with an unknown field in the + // index spec should work. + assert.commandWorked(db.adminCommand({ + applyOps: [{ + op: 'i', + ns: systemIndexesNs, + o: { + v: 1, + key: {b: 1}, + name: 'b_1_bg_system_v1', + ns: collNs, + background: true, + unknown: true, + }, + }], + })); +})(); diff --git a/jstests/core/collation_with_reverse_index.js b/jstests/core/collation_with_reverse_index.js new file mode 100644 index 00000000000..af246187348 --- /dev/null +++ b/jstests/core/collation_with_reverse_index.js @@ -0,0 +1,12 @@ +// Regression test for SERVER-34846. +(function() { + const coll = db.collation_with_reverse_index; + coll.drop(); + + coll.insertOne({int: 1, text: "hello world"}); + coll.createIndex({int: -1, text: -1}, {collation: {locale: "en", strength: 1}}); + const res = coll.find({int: 1}, {_id: 0, int: 1, text: 1}).toArray(); + + assert.eq(res.length, 1); + assert.eq(res[0].text, "hello world"); +})(); diff --git a/jstests/core/set_param1.js b/jstests/core/set_param1.js index 51b13ae87cc..4faa312808b 100644 --- a/jstests/core/set_param1.js +++ b/jstests/core/set_param1.js @@ -115,29 +115,59 @@ assert.commandFailed( assert.commandWorked( db.adminCommand({"setParameter": 1, logComponentVerbosity: old.logComponentVerbosity})); -// -// oplogFetcherMaxFetcherRestarts -// - var isMongos = (db.isMaster().msg === 'isdbgrid'); if (!isMongos) { - var origRestarts = - assert.commandWorked(db.adminCommand({getParameter: 1, oplogFetcherMaxFetcherRestarts: 1})) - .oplogFetcherMaxFetcherRestarts; - assert.gte( - origRestarts, 0, 'default value of oplogFetcherMaxFetcherRestarts cannot be negative'); + // + // oplogFetcherSteadyStateMaxFetcherRestarts + // + var origRestarts = assert + .commandWorked(db.adminCommand( + {getParameter: 1, oplogFetcherSteadyStateMaxFetcherRestarts: 1})) + .oplogFetcherSteadyStateMaxFetcherRestarts; + assert.gte(origRestarts, + 0, + 'default value of oplogFetcherSteadyStateMaxFetcherRestarts cannot be negative'); assert.commandFailedWithCode( - db.adminCommand({setParameter: 1, oplogFetcherMaxFetcherRestarts: -1}), + db.adminCommand({setParameter: 1, oplogFetcherSteadyStateMaxFetcherRestarts: -1}), ErrorCodes.BadValue, - 'server should reject negative values for oplogFetcherMaxFetcherRestarts'); - assert.commandWorked(db.adminCommand({setParameter: 1, oplogFetcherMaxFetcherRestarts: 0})); + 'server should reject negative values for oplogFetcherSteadyStateMaxFetcherRestarts'); assert.commandWorked( - db.adminCommand({setParameter: 1, oplogFetcherMaxFetcherRestarts: origRestarts + 20})); - assert.eq( - origRestarts + 20, - assert.commandWorked(db.adminCommand({getParameter: 1, oplogFetcherMaxFetcherRestarts: 1})) - .oplogFetcherMaxFetcherRestarts); + db.adminCommand({setParameter: 1, oplogFetcherSteadyStateMaxFetcherRestarts: 0})); + assert.commandWorked(db.adminCommand( + {setParameter: 1, oplogFetcherSteadyStateMaxFetcherRestarts: origRestarts + 20})); + assert.eq(origRestarts + 20, + assert + .commandWorked(db.adminCommand( + {getParameter: 1, oplogFetcherSteadyStateMaxFetcherRestarts: 1})) + .oplogFetcherSteadyStateMaxFetcherRestarts); // Restore original value. + assert.commandWorked(db.adminCommand( + {setParameter: 1, oplogFetcherSteadyStateMaxFetcherRestarts: origRestarts})); + + // + // oplogFetcherInitialSyncStateMaxFetcherRestarts + // + origRestarts = assert + .commandWorked(db.adminCommand( + {getParameter: 1, oplogFetcherInitialSyncMaxFetcherRestarts: 1})) + .oplogFetcherInitialSyncMaxFetcherRestarts; + assert.gte(origRestarts, + 0, + 'default value of oplogFetcherInitialSyncMaxFetcherRestarts cannot be negative'); + assert.commandFailedWithCode( + db.adminCommand({setParameter: 1, oplogFetcherInitialSyncMaxFetcherRestarts: -1}), + ErrorCodes.BadValue, + 'server should reject negative values for oplogFetcherInitialSyncMaxFetcherRestarts'); assert.commandWorked( - db.adminCommand({setParameter: 1, oplogFetcherMaxFetcherRestarts: origRestarts})); + db.adminCommand({setParameter: 1, oplogFetcherInitialSyncMaxFetcherRestarts: 0})); + assert.commandWorked(db.adminCommand( + {setParameter: 1, oplogFetcherInitialSyncMaxFetcherRestarts: origRestarts + 20})); + assert.eq(origRestarts + 20, + assert + .commandWorked(db.adminCommand( + {getParameter: 1, oplogFetcherInitialSyncMaxFetcherRestarts: 1})) + .oplogFetcherInitialSyncMaxFetcherRestarts); + // Restore original value. + assert.commandWorked(db.adminCommand( + {setParameter: 1, oplogFetcherInitialSyncMaxFetcherRestarts: origRestarts})); } diff --git a/jstests/core/update_numeric_field_name.js b/jstests/core/update_numeric_field_name.js new file mode 100644 index 00000000000..4adb7eee8be --- /dev/null +++ b/jstests/core/update_numeric_field_name.js @@ -0,0 +1,29 @@ +// Test that update operations correctly fail if they violate the "ambiguous field name in array" +// constraint for indexes. This is designed to reproduce SERVER-37058. +(function() { + "use strict"; + + const coll = db.update_numeric_field_name; + coll.drop(); + + assert.writeOK(coll.insert({_id: 0, 'a': [{}]})); + assert.commandWorked(coll.createIndex({'a.0.c': 1})); + + // Attempt to insert a field name '0'. The first '0' refers to the first element of the array + // 'a'. + assert.writeErrorWithCode(coll.update({_id: 0}, {$set: {'a.0.0': 1}}), 16746); + + // Verify that the indexes were not affected. + let res = assert.commandWorked(coll.validate(true)); + assert(res.valid, tojson(res)); + + assert.writeErrorWithCode(coll.update({_id: 0}, {$set: {'a.0.0.b': 1}}), 16746); + res = assert.commandWorked(coll.validate(true)); + assert(res.valid, tojson(res)); + + // An update which does not violate the ambiguous field name in array constraint should succeed. + assert.writeOK(coll.update({_id: 0}, {$set: {'a.1.b.0.0': 1}})); + + res = assert.commandWorked(coll.validate(true)); + assert(res.valid, tojson(res)); +})(); diff --git a/jstests/hooks/run_validate_collections.js b/jstests/hooks/run_validate_collections.js index 525003148e1..e0292740b97 100644 --- a/jstests/hooks/run_validate_collections.js +++ b/jstests/hooks/run_validate_collections.js @@ -10,11 +10,14 @@ const topology = DiscoverTopology.findConnectedNodes(db.getMongo()); const hostList = []; + let setFCVHost; if (topology.type === Topology.kStandalone) { hostList.push(topology.mongod); + setFCVHost = topology.mongod; } else if (topology.type === Topology.kReplicaSet) { hostList.push(...topology.nodes); + setFCVHost = topology.primary; } else if (topology.type === Topology.kShardedCluster) { hostList.push(...topology.configsvr.nodes); @@ -29,6 +32,8 @@ throw new Error('Unrecognized topology format: ' + tojson(topology)); } } + // Any of the mongos instances can be used for setting FCV. + setFCVHost = topology.mongos.nodes[0]; } else { throw new Error('Unrecognized topology format: ' + tojson(topology)); } @@ -44,6 +49,18 @@ conn.setSlaveOk(); jsTest.authenticate(conn); + if (jsTest.options().forceValidationWithFeatureCompatibilityVersion) { + let adminDB = conn.getDB('admin'); + // Make sure this node has the desired FCV. + assert.soon(() => { + const res = + adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}); + return res !== null && + res.version === + jsTest.options().forceValidationWithFeatureCompatibilityVersion; + }); + } + const dbNames = conn.getDBNames(); for (let dbName of dbNames) { if (!validateCollections(conn.getDB(dbName), {full: true})) { @@ -61,6 +78,43 @@ // We run the scoped threads in a try/finally block in case any thread throws an exception, in // which case we want to still join all the threads. let threads = []; + let adminDB; + let originalFCV; + + function getFeatureCompatibilityVersion(adminDB) { + const res = adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}); + if (res === null) { + return "3.2"; + } + return res.version; + } + + if (jsTest.options().forceValidationWithFeatureCompatibilityVersion) { + let conn = new Mongo(setFCVHost); + adminDB = conn.getDB('admin'); + try { + originalFCV = getFeatureCompatibilityVersion(adminDB); + } catch (e) { + if (jsTest.options().skipValidationOnInvalidViewDefinitions && + e.code === ErrorCodes.InvalidViewDefinition) { + print("Reading the featureCompatibilityVersion from the admin.system.version" + + " collection failed due to an invalid view definition on the admin database"); + // The view catalog would only have been resolved if the namespace doesn't exist as + // a collection. The absence of the admin.system.version collection is equivalent to + // having featureCompatibilityVersion=3.2. + originalFCV = "3.2"; + } else { + throw e; + } + } + + if (originalFCV !== jsTest.options().forceValidationWithFeatureCompatibilityVersion) { + assert.commandWorked(adminDB.adminCommand({ + setFeatureCompatibilityVersion: + jsTest.options().forceValidationWithFeatureCompatibilityVersion + })); + } + } try { hostList.forEach(host => { @@ -79,4 +133,8 @@ assert.commandWorked(res, 'Collection validation failed'); }); } + + if (jsTest.options().forceValidationWithFeatureCompatibilityVersion !== originalFCV) { + assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: originalFCV})); + } })(); diff --git a/jstests/hooks/validate_collections.js b/jstests/hooks/validate_collections.js index c5be1a230e8..8287388b6c9 100644 --- a/jstests/hooks/validate_collections.js +++ b/jstests/hooks/validate_collections.js @@ -13,19 +13,6 @@ function validateCollections(db, obj) { } } - function getFeatureCompatibilityVersion(adminDB) { - var res = adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}); - if (res === null) { - return "3.2"; - } - return res.version; - } - - function setFeatureCompatibilityVersion(adminDB, version) { - assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: version})); - assert.eq(version, getFeatureCompatibilityVersion(adminDB)); - } - assert.eq(typeof db, 'object', 'Invalid `db` object, is the shell connected to a mongod?'); assert.eq(typeof obj, 'object', 'The `obj` argument must be an object'); assert(obj.hasOwnProperty('full'), 'Please specify whether to use full validation'); @@ -34,42 +21,6 @@ function validateCollections(db, obj) { var success = true; - var adminDB = db.getSiblingDB("admin"); - - // Set the featureCompatibilityVersion to its required value for performing validation. Save the - // original value. - var originalFeatureCompatibilityVersion; - if (jsTest.options().forceValidationWithFeatureCompatibilityVersion) { - try { - originalFeatureCompatibilityVersion = getFeatureCompatibilityVersion(adminDB); - } catch (e) { - if (jsTest.options().skipValidationOnInvalidViewDefinitions && - e.code === ErrorCodes.InvalidViewDefinition) { - print("Reading the featureCompatibilityVersion from the admin.system.version" + - " collection failed due to an invalid view definition on the admin database"); - // The view catalog would only have been resolved if the namespace doesn't exist as - // a collection. The absence of the admin.system.version collection is equivalent to - // having featureCompatibilityVersion=3.2. - originalFeatureCompatibilityVersion = "3.2"; - } else { - throw e; - } - } - - try { - setFeatureCompatibilityVersion( - adminDB, jsTest.options().forceValidationWithFeatureCompatibilityVersion); - } catch (e) { - if (e.code === ErrorCodes.NotMaster) { - print('Skipping collection validation on ' + db.getMongo() + ' because the' + - ' featureCompatibilityVersion cannot be changed while connected to a' + - ' secondary'); - return true; - } - throw e; - } - } - // Don't run validate on view namespaces. let filter = {type: "collection"}; if (jsTest.options().skipValidationOnInvalidViewDefinitions) { @@ -108,10 +59,5 @@ function validateCollections(db, obj) { } } - // Restore the original value for featureCompatibilityVersion. - if (jsTest.options().forceValidationWithFeatureCompatibilityVersion) { - setFeatureCompatibilityVersion(adminDB, originalFeatureCompatibilityVersion); - } - return success; } diff --git a/jstests/libs/client-custom-oids.csr.in b/jstests/libs/client-custom-oids.csr.in new file mode 100644 index 00000000000..3a2125b5fea --- /dev/null +++ b/jstests/libs/client-custom-oids.csr.in @@ -0,0 +1,23 @@ +# Create certificate using: +# openssl req -new -config client-custom-oids.csr.in -keyout client-custom-oids.key -out client-custom-oids.csr +# openssl rsa -in client-custom-oids.key -out client-custom-oids.rsa +# openssl x509 -in client-custom-oids.csr -out client-custom-oids.pem -req -CA ca.pem -days 3650 -CAcreateserial +# cat client-custom-oids.rsa >> client-custom-oids.pem +# rm ca.srl client-custom-oids.key client-custom-oids.rsa client-custom-oids.csr + +[ req ] +default_bits=2048 +prompt=no +encrypt_key=no +default_md=sha1 +distinguished_name=dn + +[ dn ] +0.1.2.3.45=Value,Rando +0.1.2.3.56=RandoValue +CN=client +OU=KernelUser +O=MongoDB +L=New York City +ST=New York +C=US diff --git a/jstests/libs/client-custom-oids.pem b/jstests/libs/client-custom-oids.pem new file mode 100644 index 00000000000..0cd72eb99c6 --- /dev/null +++ b/jstests/libs/client-custom-oids.pem @@ -0,0 +1,48 @@ +-----BEGIN CERTIFICATE----- +MIIDjDCCAnQCCQCWaybHN+kU+zANBgkqhkiG9w0BAQsFADB0MRcwFQYDVQQDEw5L +ZXJuZWwgVGVzdCBDQTEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RC +MRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkG +A1UEBhMCVVMwHhcNMTgwNTExMTc1ODA3WhcNMjgwNTA4MTc1ODA3WjCBmzEUMBIG +AyoDLQwLVmFsdWUsUmFuZG8xEzARBgMqAzgMClJhbmRvVmFsdWUxDzANBgNVBAMM +BmNsaWVudDETMBEGA1UECwwKS2VybmVsVXNlcjEQMA4GA1UECgwHTW9uZ29EQjEW +MBQGA1UEBwwNTmV3IFlvcmsgQ2l0eTERMA8GA1UECAwITmV3IFlvcmsxCzAJBgNV +BAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxs/74Lm1axfZ +cua9jT9mIhM2CY1YGSutsaVca47PJ+H3j2JXhKln6WhUpMAKJRVInU+XoIDq7HP8 +4CMO+K2ztKdikjg3M5lhgEsy0/538jJA3TtxiENAugSSpAxpnX7bSQ4HjUPMNOGu +nd0gXbZ4YDpDAtofC3yaSzJKuCYSFMlh5qDWIoGOce3z475GX3UQZOYusRCI6c7v +Ss8JgjhaTRS32hVknBpTnf6cRzxoegSzhRag+3w4gy/nj9ne3vuvXe9dEVLxyGao +4//Ir7x6RzlE8kxrvaUYAsF4kXIEaunNF0phKGvPC9CvlupBJ9uY2lCCsDHoxt72 +dfvSnah4RQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAukmX07p8k040tTHFX3fgC +nLWyVeCFDZO1jfR0DTYQ7DW4BqgRCUx/TROu9Tl1cxihYCd0svXw/Cr/isvnhQNC +gMd9EvDw+E5sh3W1GPR8jaFR8QsESsAgY2IOpwRXLapayCZXoAxTFocLydy3Vxy/ +KwYELdXtZTzdy4D91mBZqVuYc5UxlJrP1vH425htmkKDvFrCLSSRpJ9E/DiEtFVk +dFW1YtHlIHEPoNe7+kqjlb9b5oTuYVtrUgWerJld9Wy7Jcumq0TGgavNyPEed6B7 +UMKd00FcgumGFrYVqAiYZ/ve0XpU0HZJh98YSQx+zYtza33PyQWSKUP+ajqunGID +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAxs/74Lm1axfZcua9jT9mIhM2CY1YGSutsaVca47PJ+H3j2JX +hKln6WhUpMAKJRVInU+XoIDq7HP84CMO+K2ztKdikjg3M5lhgEsy0/538jJA3Ttx +iENAugSSpAxpnX7bSQ4HjUPMNOGund0gXbZ4YDpDAtofC3yaSzJKuCYSFMlh5qDW +IoGOce3z475GX3UQZOYusRCI6c7vSs8JgjhaTRS32hVknBpTnf6cRzxoegSzhRag ++3w4gy/nj9ne3vuvXe9dEVLxyGao4//Ir7x6RzlE8kxrvaUYAsF4kXIEaunNF0ph +KGvPC9CvlupBJ9uY2lCCsDHoxt72dfvSnah4RQIDAQABAoIBAA5qZkkVLiJlyE6c +jIIBZtZxriljJlAs6ptG8jyaTksGB31OFCp6Xh5+giSwCAxITsuZSdVJg79BacSW +xi8h6TXgLTWM/fOv23ICAd19RhU5r5pbBRBsT/Us/7UFcWMKH7xcWA/WKqhq9M5d +iktBP9k6YEGEr1uV3Vo2e1R+7jCziLKFBAPnpIv4SsOTFktJD4MHqGXZxk2GN9TW +KaYyaBZjMVCs1BbIoG7FynjagQOei/GEJntlDWX5+C8rxb8rpiAoZxQysZqV7m9X +JRAFqUh25732duZM1orrOj/ItwThRptl8q/MHk2a7L+ajlKqwdXM1KGUaa8a4a6h +iL8spgECgYEA7eNx6gSCkS+3Z/YSTVCuZpUfyUclPOvLyiewWasueuo37izpIdiV +RbAzZD34zPjz2tUIBHBTO8YR35GXSvi+d1Oaq7g1U9O0wCPsqCuzPWCjntt29e5F +UWsvim5DKJgGRyeuV5KVkWaSi4Lb0Eu6jvwjzs/I98P5tsDasi5DdT0CgYEA1fLt +7H+hcW17I9rTF1BRt9BALYVidBxeOTqAK0udW+VFh/WtTAz2cYRM4DClVqYnR+BS +Rl9w6ufRaqAYE72m9CwGuiaYkE+n8RX4K1XGXcGCrLj6QyEO7BmhKr4sxK49aM8K +eHe7n2JgD6+tJuGsbq+zd6AjYOAr4vdTlZ9fj6kCgYEA4qLO8zmqvvFr1VK9KwWT +sc2ew97RHlIzC/x16WfJ13ZvJK2KyiESTZtJytWzxGvlvvU4NypgUdEpVArbzaRf +qUVwVOsheyR1zpwrKijAEdiQ6ZaCpliDl8b7OvJDb1gumvm2Al53ulezg9B+5NpQ ++UpEPxL8jvgZXsArWpfy0q0CgYEAz1QUFov0UiwLGcrTpxMRrbQkjThmGSjockLr +s4kaG9SJVeDRKBKju4u+x768asSm6jNK56FTloBBYzdzPaYapSII/tmqHKbkk297 +x9reWTrOPD0hYG4nvMp1cStLzOkg33FLr0QwUJsPhgPzIuusorKnkvRfdGCohtCw +ch04iMkCgYBUKnBh0gZWNgp5pjP2OnOVYr7FWwn6xou7W4fihqWDJkkD4THGIGlI +YvfeipEPrXIuBv70xC1jTc1jrQuQa/wvxpClA+qhcJ7twSKNsrRYgj7If+9d2foA +lLlI1CgBxGc4ClFME0XfFEPVhXLqpZclJ3uQ5o6DBH3N2apqh9EQjw== +-----END RSA PRIVATE KEY----- diff --git a/jstests/libs/client-multivalue-rdn.pem b/jstests/libs/client-multivalue-rdn.pem new file mode 100644 index 00000000000..69bc10cd05d --- /dev/null +++ b/jstests/libs/client-multivalue-rdn.pem @@ -0,0 +1,47 @@ +-----BEGIN CERTIFICATE----- +MIIDWDCCAkACCQC/RF6aMdzdETANBgkqhkiG9w0BAQsFADB0MRcwFQYDVQQDEw5L +ZXJuZWwgVGVzdCBDQTEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RC +MRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkG +A1UEBhMCVVMwHhcNMTgwNTE0MjMyODA5WhcNMjgwNTExMjMyODA5WjBoMTIwDQYD +VQQDDAZjbGllbnQwDgYDVQQKDAdNb25nb0RCMBEGA1UECwwKS2VybmVsVXNlcjEy +MAkGA1UEBhMCVVMwDwYDVQQIDAhOZXcgWW9yazAUBgNVBAcMDU5ldyBZb3JrIENp +dHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC1XoQEyPdpy+icEuSK +/9QDNVfyaAQF9//Et78uPxeTWDGzYyIBRx6Z29SR72pB78UKDrH+sw9QBiw9jg8E +D3SNg1M+ueB1jUcCgtOfpnJnF3ImfgsKnC1prWkJCtkuTrD5hEy6UXimX8sQCb6v +E8FPItzKXWPyzcn/AVUVpa8yC3A0qfOM01VxDegwjsN4PEdnDgzmwCcbJg5KN/3R +iYW6oPC3NA6tjRKpCSG0NiYJA0sj9ojYcybZCPaxBMD+ToYBujwcnp61ltZ9PFSV +swQPfh5Lniy1pBrj84dyoSAOwsQxER4bbyiyplEhNEDpTHmPwfcLR4UgOF2NHU0i +4Jq7AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAHcP7Wjl90a33oumgWlw6iOuEOS1 +L2pmMGV/b0MYEU9lifp4iGh9NzMp4Iq4OFaJXqj2c7ouPegYnP8Ga0vZj7x5xIoJ +eXcrLmSJATPKCWAauG3ouw/RcrjtoWXPkuYvwDWANc3qwBr+EziMLrhFA3GzIFrQ +duyZLxMweYVi7zNzm7L6yxKmBBFmcSGfuvMOrotly/8r+rfcgmcVV2IagHIQmGrX +0Fu2dkqU1aldChD0WhovnO5dbd1QnQ9rPlvFd8N3EfVUwGvwymLNWUTQh+lNZCmj +jbihLFlTnxb5d6EQuCu2czpNGy/XnOfYiUCbvtck9z3ecaDp1aPWe+Ibt2k= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAtV6EBMj3acvonBLkiv/UAzVX8mgEBff/xLe/Lj8Xk1gxs2Mi +AUcemdvUke9qQe/FCg6x/rMPUAYsPY4PBA90jYNTPrngdY1HAoLTn6ZyZxdyJn4L +Cpwtaa1pCQrZLk6w+YRMulF4pl/LEAm+rxPBTyLcyl1j8s3J/wFVFaWvMgtwNKnz +jNNVcQ3oMI7DeDxHZw4M5sAnGyYOSjf90YmFuqDwtzQOrY0SqQkhtDYmCQNLI/aI +2HMm2Qj2sQTA/k6GAbo8HJ6etZbWfTxUlbMED34eS54staQa4/OHcqEgDsLEMREe +G28osqZRITRA6Ux5j8H3C0eFIDhdjR1NIuCauwIDAQABAoIBACuXtBOSWiVLpXUd +9WCTbNn4kFLrPUxR91/I6mKrN3dq00dSpHG6Lli8xbLt4JRIlQt9zLpzP1L0qN2C +r4lCTblVv0RsWM7oThEEcOkGkKjGseEemnjKYP/tdjl+KgU/VLokWmzN+DnE/IG2 +Veau0N+8eWUKXWOWot5v64twb1OPlKDCcgUcP3kobM1eGFkMsK3a40lnGMnBmYD5 +EcOjAqcboTY+wSwsHiVur34qUr2ojyyJ578qq1gLkEd0mzYBBRwdfchuBbXNq80v +dQuv66tx8AmYgniuTJ/rbNcWQdHEClRuP5F9kGsYPHBU8oq5dHfLioMk5ukyUmS/ +n8zJWsECgYEA3apRczGf7/a2BiiGOC8OL2/L8JltFuhE2W1YuiduVI9pQ7lzsOxG +3Q0zsd61+Yzc4dJTrep4QPyG3ddGCuERb+U20vse0UsfkouE7OQD4j5aOhCTty/u +dsbfHcbQDQZMuT9lGPywAX33Cvzyg0zDEbf4y0fJ1tn0Er6Os0X+yvUCgYEA0XZW +otFIYmZSleDaCSeN4wIfHi67bg5Ofw9GtkLtMcjS6KNC6Ta42FzT7ThIYevo1e3E +VFiu7Z4/p22fduJnKSPhxVhvD4gN7U7BHrRVbpnsMDtsCM9qZ/ILO/63UQzJUOJz +ZfxW2ZN0g8J2eutC0guIIDaxp0jY115SMzWIoO8CgYAXc+vGOlvnsby0jhkVXLUt +g7CaNkF6iPyv3IfkukNMIxhKGBcLgxREUx9DFINAcgz0v6Im5oXuqklCs4IvqHyF +0ESqsfMixcYDZUudHMrkZyB4wERAv4uFeaklp2Ff32jCuNhjemjM350Bxp2KEtcY +ytRB/ch3OPw/93V1tlQs+QKBgQCvnz4Ss9CyGELkQavugwiXLn8yylICe+Ja7E8I +oGLKHCyiO8jtSyF+344dVtRxreACBqJXaif5OXb7hZFvl3KRbvFsirJL3nJ251JK +0T7URiBfbGMvm+EGmE3NFstTHJpqxAOnngSf1t+ZXeIDO+BBxsNy5wjbTtvo77+S +Ci+tZwKBgAvUh4UBLxebS0G9RhTMXr59lJ8hfXvofLDXxy83xvuNEczzekar9Rqz +2bGBgVz4EcggjMvP1MYJlvyJOA+k4OJZftkgaY0DJgv+T77KzGy5VGS/ApPvLNe7 +1+wgJ6G//Be9IWH2apoLAt/5wZhe/s0Yd/RGcXXS07KEJ+VCbvh0 +-----END RSA PRIVATE KEY----- diff --git a/jstests/libs/discover_topology.js b/jstests/libs/discover_topology.js index 95b072894b0..175f5831eaf 100644 --- a/jstests/libs/discover_topology.js +++ b/jstests/libs/discover_topology.js @@ -4,6 +4,7 @@ // Symbol type, so we just use unique string values instead. var Topology = { kStandalone: 'stand-alone', + kRouter: 'mongos router', kReplicaSet: 'replica set', kShardedCluster: 'sharded cluster', }; @@ -22,7 +23,11 @@ var DiscoverTopology = (function() { // The "passives" field contains the list of unelectable (priority=0) secondaries // and is omitted from the server's response when there are none. res.passives = res.passives || []; - return {type: Topology.kReplicaSet, nodes: [...res.hosts, ...res.passives]}; + return { + type: Topology.kReplicaSet, + primary: res.primary, + nodes: [...res.hosts, ...res.passives] + }; } function findConnectedNodesViaMongos(conn, options) { @@ -58,7 +63,22 @@ var DiscoverTopology = (function() { shardHosts[shardInfo._id] = getDataMemberConnectionStrings(shardConn); } - return {type: Topology.kShardedCluster, configsvr: configsvrHosts, shards: shardHosts}; + // Discover mongos URIs from the connection string. If a mongos is not passed in explicitly, + // it will not be discovered. Prior to the changes from SERVER-28560 and SERVER-31061, only + // one mongos URI could be present in the connection string. + const mongosUris = new MongoURI("mongodb://" + conn.host); + + const mongos = { + type: Topology.kRouter, + nodes: mongosUris.servers.map(uriObj => uriObj.server), + }; + + return { + type: Topology.kShardedCluster, + configsvr: configsvrHosts, + shards: shardHosts, + mongos: mongos, + }; } return { @@ -72,7 +92,11 @@ var DiscoverTopology = (function() { * is returned. * * For a replica set, an object of the form - * {type: Topology.kReplicaSet, nodes: [<conn-string1>, <conn-string2>, ...]} + * { + * type: Topology.kReplicaSet, + * primary: <primary-conn-string>, + * nodes: [<conn-string1>, <conn-string2>, ...], + * } * is returned. * * For a sharded cluster, an object of the form @@ -81,7 +105,9 @@ var DiscoverTopology = (function() { * configsvr: {nodes: [...]}, * shards: { * <shard-name1>: {type: Topology.kStandalone, mongod: ...}, - * <shard-name2>: {type: Topology.kReplicaSet, nodes: [...]}, + * <shard-name2>: {type: Topology.kReplicaSet, + * primary: <primary-conn-string>, + * nodes: [...]}, * ... * } * } diff --git a/jstests/libs/parallelTester.js b/jstests/libs/parallelTester.js index db784ab66c1..56de95c2b6d 100644 --- a/jstests/libs/parallelTester.js +++ b/jstests/libs/parallelTester.js @@ -167,8 +167,6 @@ if (typeof _threadInject != "undefined") { "mr_drop.js", "mr3.js", "indexh.js", - "evald.js", - "evalf.js", "run_program1.js", "notablescan.js", "dropdb_race.js", @@ -191,6 +189,35 @@ if (typeof _threadInject != "undefined") { // Assumes that other tests are not creating cursors. "kill_cursors.js", + + // Use eval command and potentially cause deadlock. + "constructors.js", + "error2.js", + "eval0.js", + "eval1.js", + "eval3.js", + "eval4.js", + "eval5.js", + "eval6.js", + "eval7.js", + "eval9.js", + "evala.js", + "evalb.js", + "evald.js", + "evale.js", + "evalg.js", + "evalh.js", + "evalj.js", + "eval_mr.js", + "eval_nolock.js", + "fsync.js", + "js3.js", + "js7.js", + "js9.js", + "recursion.js", + "remove8.js", + "rename4.js", + "storefunc.js", ]); var parallelFilesDir = "jstests/core"; @@ -267,6 +294,10 @@ if (typeof _threadInject != "undefined") { args.forEach(function(x) { print(" S" + suite + " Test : " + x + " ..."); var time = Date.timeFunc(function() { + // Create a new connection to the db for each file. If tests share the same + // connection it can create difficult to debug issues. + db = new Mongo(db.getMongo().host).getDB(db.getName()); + gc(); load(x); }, 1); print(" S" + suite + " Test : " + x + " " + time + "ms"); diff --git a/jstests/noPassthrough/index_killop.js b/jstests/noPassthrough/index_killop.js index 734053ce06a..42a5270a28a 100644 --- a/jstests/noPassthrough/index_killop.js +++ b/jstests/noPassthrough/index_killop.js @@ -47,9 +47,6 @@ // Kill the index build. assert.commandWorked(testDB.killOp(opId)); - assert.commandWorked( - testDB.adminCommand({configureFailPoint: 'hangAfterStartingIndexBuild', mode: 'off'})); - // Wait for the index build to stop. assert.soon(function() { return getIndexBuildOpId() == -1; @@ -62,6 +59,9 @@ // Check that no new index has been created. This verifies that the index build was aborted // rather than successfully completed. assert.eq([{_id: 1}], testDB.test.getIndexKeys()); + + assert.commandWorked( + testDB.adminCommand({configureFailPoint: 'hangAfterStartingIndexBuild', mode: 'off'})); } testAbortIndexBuild({background: true}); diff --git a/jstests/noPassthrough/indexbg1.js b/jstests/noPassthrough/indexbg1.js index 00670f3f2db..5d8e4a72847 100644 --- a/jstests/noPassthrough/indexbg1.js +++ b/jstests/noPassthrough/indexbg1.js @@ -1,6 +1,7 @@ // Test background index creation load("jstests/libs/slow_weekly_util.js"); +load("jstests/noPassthrough/libs/index_build.js"); var testServer = new SlowWeeklyMongod("indexbg1"); var db = testServer.getDB("test"); @@ -56,7 +57,7 @@ while (1) { // if indexing finishes before we can run checks, try indexing w/ m // wait for indexing to start print("wait for indexing to start"); assert.soon(function() { - return 2 === t.getIndexes().length; + return getIndexBuildOpId(db) != -1; }, "no index created", 30000, 50); print("started."); sleep(1000); // there is a race between when the index build shows up in curop and diff --git a/jstests/noPassthrough/indexbg2.js b/jstests/noPassthrough/indexbg2.js index e9ac45c8b78..815a4e35df3 100644 --- a/jstests/noPassthrough/indexbg2.js +++ b/jstests/noPassthrough/indexbg2.js @@ -1,96 +1,175 @@ // Test background index creation w/ constraints +// @tags: [requires_document_locking] -load("jstests/libs/slow_weekly_util.js"); - -var testServer = new SlowWeeklyMongod("indexbg2"); -var db = testServer.getDB("test"); -var baseName = "jstests_index12"; - -var parallel = function() { - return db[baseName + "_parallelStatus"]; -}; - -var resetParallel = function() { - parallel().drop(); -}; - -// Return the PID to call `waitpid` on for clean shutdown. -var doParallel = function(work) { - resetParallel(); - return startMongoProgramNoConnect( - "mongo", - "--eval", - work + "; db." + baseName + "_parallelStatus.save( {done:1} );", - db.getMongo().host); -}; - -var doneParallel = function() { - return !!parallel().findOne(); -}; - -var waitParallel = function() { - assert.soon(function() { - return doneParallel(); - }, "parallel did not finish in time", 300000, 1000); -}; - -var doTest = function() { +(function() { "use strict"; - var size = 10000; - var bgIndexBuildPid; - while (1) { // if indexing finishes before we can run checks, try indexing w/ more data - print("size: " + size); - var fullName = "db." + baseName; - var t = db[baseName]; - t.drop(); - - for (var i = 0; i < size; ++i) { - db.jstests_index12.save({i: i}); + + load("jstests/libs/check_log.js"); + + const conn = MongoRunner.runMongod({smallfiles: "", nojournal: ""}); + assert.neq(null, conn, "mongod failed to start."); + + let db = conn.getDB("test"); + let baseName = "jstests_index12"; + + let parallel = function() { + return db[baseName + "_parallelStatus"]; + }; + + let resetParallel = function() { + parallel().drop(); + }; + + // Return the PID to call `waitpid` on for clean shutdown. + let doParallel = function(work) { + resetParallel(); + return startMongoProgramNoConnect( + "mongo", + "--eval", + work + "; db." + baseName + "_parallelStatus.save( {done:1} );", + db.getMongo().host); + }; + + let indexBuild = function() { + let fullName = "db." + baseName; + return doParallel(fullName + ".ensureIndex( {i:1}, {background:true, unique:true} )"); + }; + + let doneParallel = function() { + return !!parallel().findOne(); + }; + + let waitParallel = function() { + assert.soon(function() { + return doneParallel(); + }, "parallel did not finish in time", 300000, 1000); + }; + + let turnFailPointOn = function(failPointName, i) { + assert.commandWorked(conn.adminCommand( + {configureFailPoint: failPointName, mode: "alwaysOn", data: {"i": i}})); + }; + + let turnFailPointOff = function(failPointName) { + assert.commandWorked(conn.adminCommand({configureFailPoint: failPointName, mode: "off"})); + }; + + // Unique background index build fails when there exists duplicate indexed values + // for the duration of the build. + let failOnExistingDuplicateValue = function(coll) { + let duplicateKey = 0; + assert.writeOK(coll.save({i: duplicateKey})); + + let bgIndexBuildPid = indexBuild(); + waitProgram(bgIndexBuildPid); + assert.eq(1, coll.getIndexes().length, "Index should fail. There exist duplicate values."); + + // Revert to unique key set + coll.deleteOne({i: duplicateKey}); + }; + + // Unique background index build fails when started with a unique key set, + // but a document with a duplicate key is inserted prior to that key being indexed. + let failOnInsertedDuplicateValue = function(coll) { + let duplicateKey = 7; + + turnFailPointOn("hangBeforeIndexBuildOf", duplicateKey); + + let bgIndexBuildPid; + try { + bgIndexBuildPid = indexBuild(); + jsTestLog("Waiting to hang before index build of i=" + duplicateKey); + checkLog.contains(conn, "Hanging before index build of i=" + duplicateKey); + + assert.writeOK(coll.save({i: duplicateKey})); + } finally { + turnFailPointOff("hangBeforeIndexBuildOf"); } - assert.eq(size, t.count()); - bgIndexBuildPid = - doParallel(fullName + ".ensureIndex( {i:1}, {background:true, unique:true} )"); + waitProgram(bgIndexBuildPid); + assert.eq(1, + coll.getIndexes().length, + "Index should fail. Duplicate key is inserted prior to that key being indexed."); + + // Revert to unique key set + coll.deleteOne({i: duplicateKey}); + }; + + // Unique background index build succeeds: + // 1) when a document is inserted with a key that has already been indexed + // (with the insert failing on duplicate key error). + // 2) when a document with a key not present in the initial set is inserted twice + // (with the initial insert succeeding and the second failing on duplicate key error). + let succeedWithWriteErrors = function(coll, newKey) { + let duplicateKey = 3; + + turnFailPointOn("hangAfterIndexBuildOf", duplicateKey); + + let bgIndexBuildPid; try { - // wait for indexing to start - assert.soon(function() { - return 2 === t.getIndexes().length; - }, "no index created", 30000, 50); - assert.writeError(t.save({i: 0, n: true})); // duplicate key violation - assert.writeOK(t.save({i: size - 1, n: true})); + bgIndexBuildPid = indexBuild(); + + jsTestLog("Waiting to hang after index build of i=" + duplicateKey); + checkLog.contains(conn, "Hanging after index build of i=" + duplicateKey); + + assert.writeError(coll.save({i: duplicateKey, n: true})); + + // First insert on key not present in initial set + assert.writeOK(coll.save({i: newKey, n: true})); } catch (e) { - // only a failure if we're still indexing - // wait for parallel status to update to reflect indexing status - sleep(1000); - if (!doneParallel()) { - waitProgram(bgIndexBuildPid); - throw e; - } + turnFailPointOff("hangAfterIndexBuildOf"); + throw e; } - if (!doneParallel()) { - // Ensure the shell has exited cleanly. Otherwise the test harness may send a SIGTERM - // which can lead to a false test failure. - waitProgram(bgIndexBuildPid); - break; + + try { + // We are currently hanging after indexing document with {i: duplicateKey}. + // To perform next check, we need to hang after indexing document with {i: newKey}. + // Add a hang before indexing document {i: newKey}, then turn off current hang + // so we are always in a known state and don't skip over the indexing of {i: newKey}. + turnFailPointOn("hangBeforeIndexBuildOf", newKey); + turnFailPointOff("hangAfterIndexBuildOf"); + turnFailPointOn("hangAfterIndexBuildOf", newKey); + turnFailPointOff("hangBeforeIndexBuildOf"); + + // Second insert on key not present in intial set fails with duplicate key error + jsTestLog("Waiting to hang after index build of i=" + newKey); + checkLog.contains(conn, "Hanging after index build of i=" + newKey); + + assert.writeError(coll.save({i: newKey, n: true})); + } finally { + turnFailPointOff("hangBeforeIndexBuildOf"); + turnFailPointOff("hangAfterIndexBuildOf"); } - print("indexing finished too soon, retrying..."); - // Although the index build finished, ensure the shell has exited. + waitProgram(bgIndexBuildPid); - size *= 2; - assert(size < 5000000, "unable to run checks in parallel with index creation"); - } + assert.eq(2, coll.getIndexes().length, "Index build should succeed"); + }; + + let doTest = function() { + "use strict"; + const size = 10; + + let coll = db[baseName]; + coll.drop(); + + for (let i = 0; i < size; ++i) { + assert.writeOK(coll.save({i: i})); + } + assert.eq(size, coll.count()); + assert.eq(1, coll.getIndexes().length, "_id index should already exist"); + + failOnExistingDuplicateValue(coll); + assert.eq(size, coll.count()); - waitParallel(); + failOnInsertedDuplicateValue(coll); + assert.eq(size, coll.count()); - /* it could be that there is more than size now but the index failed - to build - which is valid. we check index isn't there. - */ - if (t.count() != size) { - assert.eq(1, t.getIndexes().length, "change in # of elems yet index is there"); - } + succeedWithWriteErrors(coll, size); -}; + waitParallel(); + }; -doTest(); + doTest(); -testServer.stop(); + MongoRunner.stopMongod(conn); +})(); diff --git a/jstests/noPassthrough/libs/index_build.js b/jstests/noPassthrough/libs/index_build.js new file mode 100644 index 00000000000..e3564db5c41 --- /dev/null +++ b/jstests/noPassthrough/libs/index_build.js @@ -0,0 +1,13 @@ +// Returns the op id for the running index build, or -1 if there is no current index build. +function getIndexBuildOpId(db) { + const result = db.currentOp(); + assert.commandWorked(result); + let indexBuildOpId = -1; + + result.inprog.forEach(function(op) { + if (op.op == 'command' && op.query != undefined && 'createIndexes' in op.query) { + indexBuildOpId = op.opid; + } + }); + return indexBuildOpId; +} diff --git a/jstests/noPassthrough/list_indexes_only_ready_indexes.js b/jstests/noPassthrough/list_indexes_only_ready_indexes.js new file mode 100644 index 00000000000..c1748792675 --- /dev/null +++ b/jstests/noPassthrough/list_indexes_only_ready_indexes.js @@ -0,0 +1,51 @@ +// SERVER-25175: Test the listIndexes command only shows ready indexes. +(function() { + "use strict"; + + load("jstests/noPassthrough/libs/index_build.js"); + + const conn = MongoRunner.runMongod({smallfiles: "", nojournal: ""}); + assert.neq(null, conn, "mongod was unable to start up"); + + const testDB = conn.getDB("test"); + assert.commandWorked(testDB.dropDatabase()); + + function assertIndexes(coll, numIndexes, indexes) { + let res = coll.runCommand("listIndexes"); + assert.eq(numIndexes, res.cursor.firstBatch.length); + for (var i = 0; i < numIndexes; i++) { + assert.eq(indexes[i], res.cursor.firstBatch[i].name); + } + } + + let coll = testDB.list_indexes_only_ready_indexes; + coll.drop(); + assert.commandWorked(testDB.createCollection(coll.getName())); + assertIndexes(coll, 1, ["_id_"]); + assert.commandWorked(coll.createIndex({a: 1})); + assertIndexes(coll, 2, ["_id_", "a_1"]); + + assert.commandWorked( + testDB.adminCommand({configureFailPoint: 'hangAfterStartingIndexBuild', mode: 'alwaysOn'})); + const createIdx = startParallelShell( + "let coll = db.getSiblingDB('test').list_indexes_only_ready_indexes;" + + "assert.commandWorked(coll.createIndex({ b: 1 }, { background: true }));", + conn.port); + assert.soon(function() { + return getIndexBuildOpId(testDB) != -1; + }, "Index build operation not found after starting via parallelShell"); + + // Verify there is no third index. + assertIndexes(coll, 2, ["_id_", "a_1"]); + + assert.commandWorked( + testDB.adminCommand({configureFailPoint: 'hangAfterStartingIndexBuild', mode: 'off'})); + // Wait for the index build to stop. + assert.soon(function() { + return getIndexBuildOpId(testDB) == -1; + }); + const exitCode = createIdx(); + assert.eq(0, exitCode, 'expected shell to exit cleanly'); + + assertIndexes(coll, 3, ["_id_", "a_1", "b_1"]); +}()); diff --git a/jstests/noPassthroughWithMongod/indexbg_drop.js b/jstests/noPassthroughWithMongod/indexbg_drop.js index 1f4e1c1be64..244d0cca9c7 100644 --- a/jstests/noPassthroughWithMongod/indexbg_drop.js +++ b/jstests/noPassthroughWithMongod/indexbg_drop.js @@ -56,14 +56,6 @@ jsTest.log("Starting background indexing for test of: " + tojson(dc)); masterDB.getCollection(collection).ensureIndex({b: 1}); masterDB.getCollection(collection).ensureIndex({i: 1}, {background: true}); -assert.eq(3, masterDB.getCollection(collection).getIndexes().length); - -// Wait for the secondary to get the index entry -assert.soon(function() { - return 3 == secondDB.getCollection(collection).getIndexes().length; -}, "index not created on secondary (prior to drop)", 240000); - -jsTest.log("Index created and index entry exists on secondary"); // make sure the index build has started on secondary assert.soon(function() { diff --git a/jstests/replsets/buildindexes_false_with_system_indexes.js b/jstests/replsets/buildindexes_false_with_system_indexes.js new file mode 100644 index 00000000000..7307eeb30a5 --- /dev/null +++ b/jstests/replsets/buildindexes_false_with_system_indexes.js @@ -0,0 +1,92 @@ +/* + * Tests that hidden nodes with buildIndexes: false behave correctly when system tables with + * default indexes are created. + * + * @tags: [requires_persistence] + */ +(function() { + 'use strict'; + + load("jstests/replsets/rslib.js"); + + const testName = "buildindexes_false_with_system_indexes"; + + let rst = new ReplSetTest({ + name: testName, + nodes: [ + {}, + {rsConfig: {priority: 0}}, + {rsConfig: {priority: 0, hidden: true, buildIndexes: false}}, + ], + }); + const nodes = rst.startSet(); + rst.initiate(); + + let primary = rst.getPrimary(); + assert.eq(primary, nodes[0]); + let secondary = nodes[1]; + const hidden = nodes[2]; + + rst.awaitReplication(); + jsTestLog("Creating a role in the admin database"); + let adminDb = primary.getDB("admin"); + adminDb.createRole( + {role: 'test_role', roles: [{role: 'readWrite', db: 'test'}], privileges: []}); + rst.awaitReplication(); + + jsTestLog("Creating a user in the admin database"); + adminDb.createUser({user: 'test_user', pwd: 'test', roles: [{role: 'test_role', db: 'admin'}]}); + rst.awaitReplication(); + + // Make sure the indexes we expect are present on all nodes. The buildIndexes: false node + // should have only the _id_ index. + let secondaryAdminDb = secondary.getDB("admin"); + const hiddenAdminDb = hidden.getDB("admin"); + + assert.eq(["_id_", "user_1_db_1"], adminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_", "role_1_db_1"], adminDb.system.roles.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_", "user_1_db_1"], + secondaryAdminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_", "role_1_db_1"], + secondaryAdminDb.system.roles.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], hiddenAdminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], hiddenAdminDb.system.roles.getIndexes().map(x => x.name).sort()); + + // Drop the indexes and restart the secondary. The indexes should not be re-created. + jsTestLog("Dropping system indexes and restarting secondary."); + adminDb.system.users.dropIndex("user_1_db_1"); + adminDb.system.roles.dropIndex("role_1_db_1"); + rst.awaitReplication(); + assert.eq(["_id_"], adminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], adminDb.system.roles.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], secondaryAdminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], secondaryAdminDb.system.roles.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], hiddenAdminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], hiddenAdminDb.system.roles.getIndexes().map(x => x.name).sort()); + + secondary = rst.restart(secondary, {}, true /* wait for node to become healthy */); + secondaryAdminDb = secondary.getDB("admin"); + assert.eq(["_id_"], secondaryAdminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], secondaryAdminDb.system.roles.getIndexes().map(x => x.name).sort()); + + jsTestLog("Now restarting primary; indexes should be created."); + rst.restart(primary); + primary = rst.getPrimary(); + adminDb = primary.getDB("admin"); + assert.soonNoExcept(() => { + assert.eq(["_id_", "user_1_db_1"], + adminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_", "role_1_db_1"], + adminDb.system.roles.getIndexes().map(x => x.name).sort()); + return true; + }); + rst.awaitReplication(); + assert.eq(["_id_", "user_1_db_1"], + secondaryAdminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_", "role_1_db_1"], + secondaryAdminDb.system.roles.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], hiddenAdminDb.system.users.getIndexes().map(x => x.name).sort()); + assert.eq(["_id_"], hiddenAdminDb.system.roles.getIndexes().map(x => x.name).sort()); + + rst.stopSet(); +}()); diff --git a/jstests/replsets/emptycapped.js b/jstests/replsets/emptycapped.js index 39a5c0eac56..f2d278dfb40 100644 --- a/jstests/replsets/emptycapped.js +++ b/jstests/replsets/emptycapped.js @@ -21,11 +21,13 @@ // Truncate a non-existent collection on a non-existent database. assert.commandWorked(rst.getPrimary().getDB('nonexistent').dropDatabase()); assert.commandFailedWithCode( - rst.getPrimary().getDB('nonexistent').runCommand({emptycapped: 'nonexistent'}), 13429); + rst.getPrimary().getDB('nonexistent').runCommand({emptycapped: 'nonexistent'}), + ErrorCodes.NamespaceNotFound); // Truncate a non-existent collection. primaryTestDB.nonexistent.drop(); - assert.commandFailedWithCode(primaryTestDB.runCommand({emptycapped: 'nonexistent'}), 28584); + assert.commandFailedWithCode(primaryTestDB.runCommand({emptycapped: 'nonexistent'}), + ErrorCodes.NamespaceNotFound); // Truncate a capped collection. assert.commandWorked(primaryTestDB.createCollection("capped", {capped: true, size: 4096})); diff --git a/jstests/replsets/initial_sync4.js b/jstests/replsets/initial_sync4.js index e0ff50af4a2..031acdf35c3 100644 --- a/jstests/replsets/initial_sync4.js +++ b/jstests/replsets/initial_sync4.js @@ -34,7 +34,7 @@ var config = replTest.getReplSetConfig(); config.version = replTest.getReplSetConfigFromNode().version + 1; - config.members.push({_id: 2, host: hostname + ":" + s.port, priority: 0}); + config.members.push({_id: 2, host: hostname + ":" + s.port, priority: 0, votes: 0}); try { m.getDB("admin").runCommand({replSetReconfig: config}); } catch (e) { diff --git a/jstests/replsets/initial_sync_rename_collection_unsafe.js b/jstests/replsets/initial_sync_rename_collection_unsafe.js index a105e1c9287..379851f5fb5 100644 --- a/jstests/replsets/initial_sync_rename_collection_unsafe.js +++ b/jstests/replsets/initial_sync_rename_collection_unsafe.js @@ -30,7 +30,12 @@ const secondary = rst.add({setParameter: {allowUnsafeRenamesDuringInitialSync: true}}); assert.commandWorked(secondary.adminCommand( {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'alwaysOn'})); - rst.reInitiate(); + + jsTestLog('Begin initial sync on secondary'); + let conf = rst.getPrimary().getDB('admin').runCommand({replSetGetConfig: 1}).config; + conf.members.push({_id: 1, host: secondary.host, priority: 0, votes: 0}); + conf.version++; + assert.commandWorked(rst.getPrimary().getDB('admin').runCommand({replSetReconfig: conf})); assert.eq(primary, rst.getPrimary(), 'Primary changed after reconfig'); // Wait for fail point message to be logged. diff --git a/jstests/replsets/libs/tags.js b/jstests/replsets/libs/tags.js index 0b61c990c49..f91484e0296 100644 --- a/jstests/replsets/libs/tags.js +++ b/jstests/replsets/libs/tags.js @@ -200,13 +200,13 @@ var TagsTest = function(options) { // Depending on the order of heartbeats (containing last committed op time) received // by a node, it might hang up on its sync source. This may cause some of the write concern // tests to fail. - var timeout = 20 * 1000; + var failTimeout = 15 * 1000; jsTestLog('test1'); primary = ensurePrimary(2, replTest.nodes.slice(0, 3)); jsTestLog('Non-existent write concern should be rejected.'); - options = {writeConcern: {w: 'blahblah', wtimeout: timeout}}; + options = {writeConcern: {w: 'blahblah', wtimeout: ReplSetTest.kDefaultTimeoutMS}}; assert.writeOK(primary.getDB('foo').bar.insert(doc)); var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); assert.neq(null, result.getWriteConcernError()); @@ -215,7 +215,7 @@ var TagsTest = function(options) { tojson(result.getWriteConcernError())); jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.'); - var options = {writeConcern: {w: '3 or 4', wtimeout: timeout}}; + var options = {writeConcern: {w: '3 or 4', wtimeout: failTimeout}}; assert.writeOK(primary.getDB('foo').bar.insert(doc)); result = primary.getDB('foo').bar.insert(doc, options); assert.neq(null, result.getWriteConcernError()); @@ -228,12 +228,12 @@ var TagsTest = function(options) { jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' + primary.host + ' via node 1 ' + replTest.nodes[1].host); - options = {writeConcern: {w: '3 or 4', wtimeout: timeout}}; + options = {writeConcern: {w: '3 or 4', wtimeout: ReplSetTest.kDefaultTimeoutMS}}; assert.writeOK(primary.getDB('foo').bar.insert(doc)); assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.'); - options = {writeConcern: {w: '3 and 4', wtimeout: timeout}}; + options = {writeConcern: {w: '3 and 4', wtimeout: failTimeout}}; assert.writeOK(primary.getDB('foo').bar.insert(doc)); result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); assert.neq(null, result.getWriteConcernError()); @@ -247,7 +247,7 @@ var TagsTest = function(options) { jsTestLog('Write concern "3 and 4" should work - ' + 'nodes 3 and 4 are connected to primary via node 1.'); - options = {writeConcern: {w: '3 and 4', wtimeout: timeout}}; + options = {writeConcern: {w: '3 and 4', wtimeout: ReplSetTest.kDefaultTimeoutMS}}; assert.writeOK(primary.getDB('foo').bar.insert(doc)); assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); @@ -263,7 +263,7 @@ var TagsTest = function(options) { jsTestLog('Write concern "2 dc and 3 server"'); primary = ensurePrimary(2, replTest.nodes.slice(0, 3), replTest.nodes.length); - options = {writeConcern: {w: '2 dc and 3 server', wtimeout: timeout}}; + options = {writeConcern: {w: '2 dc and 3 server', wtimeout: ReplSetTest.kDefaultTimeoutMS}}; assert.writeOK(primary.getDB('foo').bar.insert(doc)); assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); @@ -284,13 +284,13 @@ var TagsTest = function(options) { jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + primary.host); - options = {writeConcern: {w: '3 and 4', wtimeout: timeout}}; + options = {writeConcern: {w: '3 and 4', wtimeout: ReplSetTest.kDefaultTimeoutMS}}; assert.writeOK(primary.getDB('foo').bar.insert(doc)); assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host + ' is down.'); - options = {writeConcern: {w: '2', wtimeout: timeout}}; + options = {writeConcern: {w: '2', wtimeout: failTimeout}}; assert.writeOK(primary.getDB('foo').bar.insert(doc)); result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); assert.neq(null, result.getWriteConcernError()); diff --git a/jstests/replsets/oplog_note_cmd.js b/jstests/replsets/oplog_note_cmd.js index 0c92609535a..77757b457c2 100644 --- a/jstests/replsets/oplog_note_cmd.js +++ b/jstests/replsets/oplog_note_cmd.js @@ -13,9 +13,10 @@ var statusBefore = db.runCommand({replSetGetStatus: 1}); assert.commandWorked(db.runCommand({appendOplogNote: 1, data: {a: 1}})); var statusAfter = db.runCommand({replSetGetStatus: 1}); if (rs.getReplSetConfigFromNode().protocolVersion != 1) { - assert.lt(statusBefore.members[0].optime, statusAfter.members[0].optime); + assert.lt(bsonWoCompare(statusBefore.members[0].optime, statusAfter.members[0].optime), 0); } else { - assert.lt(statusBefore.members[0].optime.ts, statusAfter.members[0].optime.ts); + assert.lt(bsonWoCompare(statusBefore.members[0].optime.ts, statusAfter.members[0].optime.ts), + 0); } // Make sure note written successfully diff --git a/jstests/sharding/addshard5.js b/jstests/sharding/addshard5.js index cedbe721177..128c52252a1 100644 --- a/jstests/sharding/addshard5.js +++ b/jstests/sharding/addshard5.js @@ -7,20 +7,13 @@ var mongos = st.s; var admin = mongos.getDB('admin'); - var config = mongos.getDB('config'); var coll = mongos.getCollection('foo.bar'); - // Get all the shard info and connections - var shards = []; - config.shards.find().sort({_id: 1}).forEach(function(doc) { - shards.push(Object.merge(doc, {conn: new Mongo(doc.host)})); - }); - // Shard collection assert.commandWorked(mongos.adminCommand({enableSharding: coll.getDB() + ''})); // Just to be sure what primary we start from - st.ensurePrimaryShard(coll.getDB().getName(), shards[0]._id); + st.ensurePrimaryShard(coll.getDB().getName(), st.shard0.shardName); assert.commandWorked(mongos.adminCommand({shardCollection: coll + '', key: {_id: 1}})); // Insert one document @@ -28,23 +21,23 @@ // Migrate the collection to and from shard1 so shard0 loads the shard1 host assert.commandWorked(mongos.adminCommand( - {moveChunk: coll + '', find: {_id: 0}, to: shards[1]._id, _waitForDelete: true})); + {moveChunk: coll + '', find: {_id: 0}, to: st.shard1.shardName, _waitForDelete: true})); assert.commandWorked(mongos.adminCommand( - {moveChunk: coll + '', find: {_id: 0}, to: shards[0]._id, _waitForDelete: true})); + {moveChunk: coll + '', find: {_id: 0}, to: st.shard0.shardName, _waitForDelete: true})); // Drop and re-add shard with the same name but a new host. - assert.commandWorked(mongos.adminCommand({removeShard: shards[1]._id})); - assert.commandWorked(mongos.adminCommand({removeShard: shards[1]._id})); + assert.commandWorked(mongos.adminCommand({removeShard: st.shard1.shardName})); + assert.commandWorked(mongos.adminCommand({removeShard: st.shard1.shardName})); var shard2 = MongoRunner.runMongod({'shardsvr': ''}); - assert.commandWorked(mongos.adminCommand({addShard: shard2.host, name: shards[1]._id})); + assert.commandWorked(mongos.adminCommand({addShard: shard2.host, name: st.shard1.shardName})); jsTest.log('Shard was dropped and re-added with same name...'); st.printShardingStatus(); // Try a migration assert.commandWorked( - mongos.adminCommand({moveChunk: coll + '', find: {_id: 0}, to: shards[1]._id})); + mongos.adminCommand({moveChunk: coll + '', find: {_id: 0}, to: st.shard1.shardName})); assert.eq('world', shard2.getCollection(coll + '').findOne().hello); diff --git a/jstests/sharding/batch_write_command_sharded.js b/jstests/sharding/batch_write_command_sharded.js index 884d5bb85bb..7ede1900455 100644 --- a/jstests/sharding/batch_write_command_sharded.js +++ b/jstests/sharding/batch_write_command_sharded.js @@ -11,7 +11,6 @@ var mongos = st.s0; var admin = mongos.getDB("admin"); var config = mongos.getDB("config"); - var shards = config.shards.find().toArray(); var configConnStr = st._configDB; jsTest.log("Starting sharding batch write tests..."); @@ -106,7 +105,7 @@ // START SETUP var brokenColl = mongos.getCollection("broken.coll"); assert.commandWorked(admin.runCommand({enableSharding: brokenColl.getDB().toString()})); - st.ensurePrimaryShard(brokenColl.getDB().toString(), shards[0]._id); + st.ensurePrimaryShard(brokenColl.getDB().toString(), st.shard0.shardName); assert.commandWorked(admin.runCommand({shardCollection: brokenColl.toString(), key: {_id: 1}})); assert.commandWorked(admin.runCommand({split: brokenColl.toString(), middle: {_id: 0}})); @@ -120,8 +119,8 @@ // Modify the chunks to make shards at a higher version - assert.commandWorked( - admin.runCommand({moveChunk: brokenColl.toString(), find: {_id: 0}, to: shards[1]._id})); + assert.commandWorked(admin.runCommand( + {moveChunk: brokenColl.toString(), find: {_id: 0}, to: st.shard1.shardName})); // Rewrite the old chunks back to the config server diff --git a/jstests/sharding/bouncing_count.js b/jstests/sharding/bouncing_count.js index f00218f5dfb..d73190f4744 100644 --- a/jstests/sharding/bouncing_count.js +++ b/jstests/sharding/bouncing_count.js @@ -15,10 +15,21 @@ var collB = mongosB.getCollection("" + collA); var collC = mongosB.getCollection("" + collA); - var shards = config.shards.find().sort({_id: 1}).toArray(); + var shards = [ + st.shard0, + st.shard1, + st.shard2, + st.shard3, + st.shard4, + st.shard5, + st.shard6, + st.shard7, + st.shard8, + st.shard9 + ]; assert.commandWorked(admin.runCommand({enableSharding: "" + collA.getDB()})); - st.ensurePrimaryShard(collA.getDB().getName(), shards[1]._id); + st.ensurePrimaryShard(collA.getDB().getName(), st.shard1.shardName); assert.commandWorked(admin.runCommand({shardCollection: "" + collA, key: {_id: 1}})); jsTestLog("Splitting up the collection..."); @@ -27,7 +38,7 @@ for (var i = 0; i < shards.length; i++) { assert.commandWorked(admin.runCommand({split: "" + collA, middle: {_id: i}})); assert.commandWorked( - admin.runCommand({moveChunk: "" + collA, find: {_id: i}, to: shards[i]._id})); + admin.runCommand({moveChunk: "" + collA, find: {_id: i}, to: shards[i].shardName})); } mongosB.getDB("admin").runCommand({flushRouterConfig: 1}); @@ -38,8 +49,11 @@ // Change up all the versions... for (var i = 0; i < shards.length; i++) { - assert.commandWorked(admin.runCommand( - {moveChunk: "" + collA, find: {_id: i}, to: shards[(i + 1) % shards.length]._id})); + assert.commandWorked(admin.runCommand({ + moveChunk: "" + collA, + find: {_id: i}, + to: shards[(i + 1) % shards.length].shardName + })); } // Make sure mongos A is up-to-date diff --git a/jstests/sharding/bulk_insert.js b/jstests/sharding/bulk_insert.js index 715660fa67f..9284b9e9d97 100644 --- a/jstests/sharding/bulk_insert.js +++ b/jstests/sharding/bulk_insert.js @@ -6,17 +6,11 @@ var mongos = st.s; var staleMongos = st.s1; - var config = mongos.getDB("config"); var admin = mongos.getDB("admin"); - var shards = config.shards.find().toArray(); - - for (var i = 0; i < shards.length; i++) { - shards[i].conn = new Mongo(shards[i].host); - } var collSh = mongos.getCollection(jsTestName() + ".collSharded"); var collUn = mongos.getCollection(jsTestName() + ".collUnsharded"); - var collDi = shards[0].conn.getCollection(jsTestName() + ".collDirect"); + var collDi = st.shard0.getCollection(jsTestName() + ".collDirect"); jsTest.log('Checking write to config collections...'); assert.writeOK(admin.TestColl.insert({SingleDoc: 1})); @@ -25,9 +19,10 @@ jsTest.log("Setting up collections..."); assert.commandWorked(admin.runCommand({enableSharding: collSh.getDB() + ""})); - st.ensurePrimaryShard(collSh.getDB() + "", shards[0]._id); + st.ensurePrimaryShard(collSh.getDB() + "", st.shard0.shardName); - assert.commandWorked(admin.runCommand({movePrimary: collUn.getDB() + "", to: shards[1]._id})); + assert.commandWorked( + admin.runCommand({movePrimary: collUn.getDB() + "", to: st.shard1.shardName})); printjson(collSh.ensureIndex({ukey: 1}, {unique: true})); printjson(collUn.ensureIndex({ukey: 1}, {unique: true})); @@ -36,7 +31,7 @@ assert.commandWorked(admin.runCommand({shardCollection: collSh + "", key: {ukey: 1}})); assert.commandWorked(admin.runCommand({split: collSh + "", middle: {ukey: 0}})); assert.commandWorked(admin.runCommand( - {moveChunk: collSh + "", find: {ukey: 0}, to: shards[0]._id, _waitForDelete: true})); + {moveChunk: collSh + "", find: {ukey: 0}, to: st.shard0.shardName, _waitForDelete: true})); var resetColls = function() { assert.writeOK(collSh.remove({})); @@ -248,9 +243,9 @@ assert.eq(null, staleCollSh.findOne(), 'Collections should be empty'); assert.commandWorked(admin.runCommand( - {moveChunk: collSh + "", find: {ukey: 0}, to: shards[1]._id, _waitForDelete: true})); + {moveChunk: collSh + "", find: {ukey: 0}, to: st.shard1.shardName, _waitForDelete: true})); assert.commandWorked(admin.runCommand( - {moveChunk: collSh + "", find: {ukey: 0}, to: shards[0]._id, _waitForDelete: true})); + {moveChunk: collSh + "", find: {ukey: 0}, to: st.shard0.shardName, _waitForDelete: true})); assert.writeOK(staleCollSh.insert(inserts)); @@ -274,9 +269,9 @@ assert.eq(null, staleCollSh.findOne(), 'Collections should be empty'); assert.commandWorked(admin.runCommand( - {moveChunk: collSh + "", find: {ukey: 0}, to: shards[1]._id, _waitForDelete: true})); + {moveChunk: collSh + "", find: {ukey: 0}, to: st.shard1.shardName, _waitForDelete: true})); assert.commandWorked(admin.runCommand( - {moveChunk: collSh + "", find: {ukey: 0}, to: shards[0]._id, _waitForDelete: true})); + {moveChunk: collSh + "", find: {ukey: 0}, to: st.shard0.shardName, _waitForDelete: true})); assert.writeOK(staleCollSh.insert(inserts)); diff --git a/jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js b/jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js index e928eaebcf2..499df7c74dc 100644 --- a/jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js +++ b/jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js @@ -14,12 +14,11 @@ load('./jstests/libs/cleanup_orphaned_util.js'); var staticMongod = MongoRunner.runMongod({}); // For startParallelOps. var st = new ShardingTest({shards: 2, other: {separateConfig: true}}); - var mongos = st.s0, admin = mongos.getDB('admin'), - shards = mongos.getCollection('config.shards').find().toArray(), dbName = 'foo', - ns = dbName + '.bar', coll = mongos.getCollection(ns); + var mongos = st.s0, admin = mongos.getDB('admin'), dbName = 'foo', ns = dbName + '.bar', + coll = mongos.getCollection(ns); assert.commandWorked(admin.runCommand({enableSharding: dbName})); - printjson(admin.runCommand({movePrimary: dbName, to: shards[0]._id})); + printjson(admin.runCommand({movePrimary: dbName, to: st.shard0.shardName})); assert.commandWorked(admin.runCommand({shardCollection: ns, key: {key: 'hashed'}})); // Makes four chunks by default, two on each shard. diff --git a/jstests/sharding/cleanup_orphaned_cmd_prereload.js b/jstests/sharding/cleanup_orphaned_cmd_prereload.js index 05fbd8b741a..a5077faa7eb 100644 --- a/jstests/sharding/cleanup_orphaned_cmd_prereload.js +++ b/jstests/sharding/cleanup_orphaned_cmd_prereload.js @@ -6,11 +6,10 @@ var st = new ShardingTest({shards: 2}); var mongos = st.s0; var admin = mongos.getDB("admin"); -var shards = mongos.getCollection("config.shards").find().toArray(); var coll = mongos.getCollection("foo.bar"); assert(admin.runCommand({enableSharding: coll.getDB() + ""}).ok); -printjson(admin.runCommand({movePrimary: coll.getDB() + "", to: shards[0]._id})); +printjson(admin.runCommand({movePrimary: coll.getDB() + "", to: st.shard0.shardName})); assert(admin.runCommand({shardCollection: coll + "", key: {_id: 1}}).ok); jsTest.log("Moving some chunks to shard1..."); @@ -20,11 +19,13 @@ assert(admin.runCommand({split: coll + "", middle: {_id: 1}}).ok); assert( admin - .runCommand({moveChunk: coll + "", find: {_id: 0}, to: shards[1]._id, _waitForDelete: true}) + .runCommand( + {moveChunk: coll + "", find: {_id: 0}, to: st.shard1.shardName, _waitForDelete: true}) .ok); assert( admin - .runCommand({moveChunk: coll + "", find: {_id: 1}, to: shards[1]._id, _waitForDelete: true}) + .runCommand( + {moveChunk: coll + "", find: {_id: 1}, to: st.shard1.shardName, _waitForDelete: true}) .ok); var metadata = @@ -43,7 +44,8 @@ assert(!st.shard1.getDB("admin") jsTest.log("Moving some chunks back to shard0 after empty..."); -admin.runCommand({moveChunk: coll + "", find: {_id: -1}, to: shards[1]._id, _waitForDelete: true}); +admin.runCommand( + {moveChunk: coll + "", find: {_id: -1}, to: st.shard1.shardName, _waitForDelete: true}); var metadata = st.shard0.getDB("admin").runCommand({getShardVersion: coll + "", fullMetadata: true}).metadata; @@ -56,7 +58,8 @@ assert.eq(metadata.pending.length, 0); assert( admin - .runCommand({moveChunk: coll + "", find: {_id: 1}, to: shards[0]._id, _waitForDelete: true}) + .runCommand( + {moveChunk: coll + "", find: {_id: 1}, to: st.shard0.shardName, _waitForDelete: true}) .ok); var metadata = diff --git a/jstests/sharding/coll_epoch_test1.js b/jstests/sharding/coll_epoch_test1.js index 2cc7c26c60b..2203bed6641 100644 --- a/jstests/sharding/coll_epoch_test1.js +++ b/jstests/sharding/coll_epoch_test1.js @@ -14,10 +14,7 @@ var staleMongos = st.s1; var insertMongos = st.s2; - var shards = []; - config.shards.find().forEach(function(doc) { - shards.push(doc._id); - }); + var shards = [st.shard0, st.shard1, st.shard2]; // // Test that inserts and queries go to the correct shard even when the collection has been @@ -29,7 +26,7 @@ assert.commandWorked(admin.runCommand({enableSharding: coll.getDB() + ""})); // TODO(PM-85): Make sure we *always* move the primary after collection lifecyle project is // complete - st.ensurePrimaryShard(coll.getDB().getName(), 'shard0001'); + st.ensurePrimaryShard(coll.getDB().getName(), st.shard1.shardName); assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {_id: 1}})); st.configRS.awaitLastOpCommitted(); // TODO: Remove after collection lifecyle project (PM-85) @@ -49,7 +46,7 @@ jsTest.log("Re-enabling sharding with a different key..."); - st.ensurePrimaryShard(coll.getDB().getName(), 'shard0001'); + st.ensurePrimaryShard(coll.getDB().getName(), st.shard1.shardName); assert.commandWorked(coll.ensureIndex({notId: 1})); assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {notId: 1}})); @@ -88,10 +85,10 @@ jsTest.log("Re-creating sharded collection with different primary..."); - var getOtherShard = function(shard) { - for (var id in shards) { - if (shards[id] != shard) - return shards[id]; + var getOtherShard = function(shardId) { + for (var i = 0; i < shards.length; ++i) { + if (shards[i].shardName != shardId) + return shards[i].shardName; } }; diff --git a/jstests/sharding/coll_epoch_test2.js b/jstests/sharding/coll_epoch_test2.js index dbed610cad6..c2106ebefbe 100644 --- a/jstests/sharding/coll_epoch_test2.js +++ b/jstests/sharding/coll_epoch_test2.js @@ -19,10 +19,7 @@ var coll = st.s.getCollection("foo.bar"); insertMongos.getDB("admin").runCommand({setParameter: 1, traceExceptions: true}); -var shards = {}; -config.shards.find().forEach(function(doc) { - shards[doc._id] = new Mongo(doc.host); -}); +var shards = [st.shard0, st.shard1]; // // Set up a sharded collection @@ -38,10 +35,10 @@ assert.writeOK(coll.insert({hello: "world"})); jsTest.log("Sharding collection across multiple shards..."); -var getOtherShard = function(shard) { - for (id in shards) { - if (id != shard) - return id; +var getOtherShard = function(shardId) { + for (shard in shards) { + if (shard.shardName != shardId) + return shard.shardName; } }; diff --git a/jstests/sharding/covered_shard_key_indexes.js b/jstests/sharding/covered_shard_key_indexes.js index 98168e7dccb..ddef3a4e2d3 100644 --- a/jstests/sharding/covered_shard_key_indexes.js +++ b/jstests/sharding/covered_shard_key_indexes.js @@ -10,7 +10,6 @@ var st = new ShardingTest({shards: 1}); var mongos = st.s0; var admin = mongos.getDB("admin"); -var shards = mongos.getCollection("config.shards").find().toArray(); var coll = mongos.getCollection("foo.bar"); // @@ -18,7 +17,7 @@ var coll = mongos.getCollection("foo.bar"); // Tests with _id : 1 shard key assert(admin.runCommand({enableSharding: coll.getDB() + ""}).ok); -printjson(admin.runCommand({movePrimary: coll.getDB() + "", to: shards[0]._id})); +printjson(admin.runCommand({movePrimary: coll.getDB() + "", to: st.shard0.shardName})); assert(admin.runCommand({shardCollection: coll + "", key: {_id: 1}}).ok); st.printShardingStatus(); diff --git a/jstests/sharding/cursor_cleanup.js b/jstests/sharding/cursor_cleanup.js index 5d0ce46f532..741c7f48b3c 100644 --- a/jstests/sharding/cursor_cleanup.js +++ b/jstests/sharding/cursor_cleanup.js @@ -6,17 +6,15 @@ var st = new ShardingTest({shards: 2, mongos: 1}); var mongos = st.s0; var admin = mongos.getDB("admin"); -var config = mongos.getDB("config"); -var shards = config.shards.find().toArray(); var coll = mongos.getCollection("foo.bar"); var collUnsharded = mongos.getCollection("foo.baz"); // Shard collection printjson(admin.runCommand({enableSharding: coll.getDB() + ""})); -printjson(admin.runCommand({movePrimary: coll.getDB() + "", to: shards[0]._id})); +printjson(admin.runCommand({movePrimary: coll.getDB() + "", to: st.shard0.shardName})); printjson(admin.runCommand({shardCollection: coll + "", key: {_id: 1}})); printjson(admin.runCommand({split: coll + "", middle: {_id: 0}})); -printjson(admin.runCommand({moveChunk: coll + "", find: {_id: 0}, to: shards[1]._id})); +printjson(admin.runCommand({moveChunk: coll + "", find: {_id: 0}, to: st.shard1.shardName})); jsTest.log("Collection set up..."); st.printShardingStatus(true); diff --git a/jstests/sharding/dump_coll_metadata.js b/jstests/sharding/dump_coll_metadata.js index eb60af37cb4..dbce60e1290 100644 --- a/jstests/sharding/dump_coll_metadata.js +++ b/jstests/sharding/dump_coll_metadata.js @@ -9,11 +9,10 @@ var mongos = st.s0; var coll = mongos.getCollection("foo.bar"); var admin = mongos.getDB("admin"); - var shards = mongos.getCollection("config.shards").find().toArray(); var shardAdmin = st.shard0.getDB("admin"); assert.commandWorked(admin.runCommand({enableSharding: coll.getDB() + ""})); - st.ensurePrimaryShard(coll.getDB() + "", shards[0]._id); + st.ensurePrimaryShard(coll.getDB() + "", st.shard0.shardName); assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {_id: 1}})); assert.commandWorked(shardAdmin.runCommand({getShardVersion: coll + ""})); diff --git a/jstests/sharding/empty_doc_results.js b/jstests/sharding/empty_doc_results.js index 8f75d65eb7d..0ee44a76988 100644 --- a/jstests/sharding/empty_doc_results.js +++ b/jstests/sharding/empty_doc_results.js @@ -7,15 +7,14 @@ var mongos = st.s0; var coll = mongos.getCollection("foo.bar"); var admin = mongos.getDB("admin"); - var shards = mongos.getDB("config").shards.find().toArray(); assert.commandWorked(admin.runCommand({enableSharding: coll.getDB().getName()})); - printjson(admin.runCommand({movePrimary: coll.getDB().getName(), to: shards[0]._id})); + printjson(admin.runCommand({movePrimary: coll.getDB().getName(), to: st.shard0.shardName})); assert.commandWorked(admin.runCommand({shardCollection: coll.getFullName(), key: {_id: 1}})); assert.commandWorked(admin.runCommand({split: coll.getFullName(), middle: {_id: 0}})); assert.commandWorked( - admin.runCommand({moveChunk: coll.getFullName(), find: {_id: 0}, to: shards[1]._id})); + admin.runCommand({moveChunk: coll.getFullName(), find: {_id: 0}, to: st.shard1.shardName})); st.printShardingStatus(); diff --git a/jstests/sharding/exact_shard_key_target.js b/jstests/sharding/exact_shard_key_target.js index 885647ec96e..aef428fc8fc 100644 --- a/jstests/sharding/exact_shard_key_target.js +++ b/jstests/sharding/exact_shard_key_target.js @@ -9,14 +9,13 @@ var st = new ShardingTest({shards: 2, verbose: 4}); var mongos = st.s0; var coll = mongos.getCollection("foo.bar"); var admin = mongos.getDB("admin"); -var shards = mongos.getDB("config").shards.find().toArray(); assert.commandWorked(admin.runCommand({enableSharding: coll.getDB().getName()})); -printjson(admin.runCommand({movePrimary: coll.getDB().getName(), to: shards[0]._id})); +printjson(admin.runCommand({movePrimary: coll.getDB().getName(), to: st.shard0.shardName})); assert.commandWorked(admin.runCommand({shardCollection: coll.getFullName(), key: {"a.b": 1}})); assert.commandWorked(admin.runCommand({split: coll.getFullName(), middle: {"a.b": 0}})); assert.commandWorked( - admin.runCommand({moveChunk: coll.getFullName(), find: {"a.b": 0}, to: shards[1]._id})); + admin.runCommand({moveChunk: coll.getFullName(), find: {"a.b": 0}, to: st.shard1.shardName})); st.printShardingStatus(); diff --git a/jstests/sharding/geo_shardedgeonear.js b/jstests/sharding/geo_shardedgeonear.js index 123b4b174cc..9a6a5480f61 100644 --- a/jstests/sharding/geo_shardedgeonear.js +++ b/jstests/sharding/geo_shardedgeonear.js @@ -2,16 +2,13 @@ var coll = 'points'; -function test(db, sharded, indexType) { +function test(st, db, sharded, indexType) { printjson(db); db[coll].drop(); if (sharded) { - var shards = []; + var shards = [st.shard0, st.shard1, st.shard2]; var config = shardedDB.getSiblingDB("config"); - config.shards.find().forEach(function(shard) { - shards.push(shard._id); - }); shardedDB.adminCommand({shardCollection: shardedDB[coll].getFullName(), key: {rand: 1}}); for (var i = 1; i < 10; i++) { @@ -20,7 +17,7 @@ function test(db, sharded, indexType) { shardedDB.adminCommand({ moveChunk: shardedDB[coll].getFullName(), find: {rand: i / 10}, - to: shards[i % shards.length] + to: shards[i % shards.length].shardName }); } @@ -50,5 +47,5 @@ var shardedDB = sharded.getDB('test'); sharded.ensurePrimaryShard('test', 'shard0001'); printjson(shardedDB); -test(shardedDB, true, '2dsphere'); +test(sharded, shardedDB, true, '2dsphere'); sharded.stop(); diff --git a/jstests/sharding/jumbo1.js b/jstests/sharding/jumbo1.js index f3f0e8d2a9d..4aed86feeae 100644 --- a/jstests/sharding/jumbo1.js +++ b/jstests/sharding/jumbo1.js @@ -44,7 +44,7 @@ print("diff: " + d); s.printShardingStatus(true); return d < 5; - }, "balance didn't happen", 1000 * 60 * 5, 5000); + }, "balance didn't happen", 1000 * 60 * 10, 5000); // Check that the jumbo chunk did not move, which shouldn't be possible. var jumboChunk = diff --git a/jstests/sharding/large_skip_one_shard.js b/jstests/sharding/large_skip_one_shard.js index 99c73eb99b3..e1f717a5f5a 100644 --- a/jstests/sharding/large_skip_one_shard.js +++ b/jstests/sharding/large_skip_one_shard.js @@ -4,7 +4,6 @@ var st = new ShardingTest({shards: 2, mongos: 1}); var mongos = st.s0; -var shards = mongos.getDB("config").shards.find().toArray(); var admin = mongos.getDB("admin"); var collSharded = mongos.getCollection("testdb.collSharded"); @@ -12,10 +11,10 @@ var collUnSharded = mongos.getCollection("testdb.collUnSharded"); // Set up a sharded and unsharded collection assert(admin.runCommand({enableSharding: collSharded.getDB() + ""}).ok); -printjson(admin.runCommand({movePrimary: collSharded.getDB() + "", to: shards[0]._id})); +printjson(admin.runCommand({movePrimary: collSharded.getDB() + "", to: st.shard0.shardName})); assert(admin.runCommand({shardCollection: collSharded + "", key: {_id: 1}}).ok); assert(admin.runCommand({split: collSharded + "", middle: {_id: 0}}).ok); -assert(admin.runCommand({moveChunk: collSharded + "", find: {_id: 0}, to: shards[1]._id}).ok); +assert(admin.runCommand({moveChunk: collSharded + "", find: {_id: 0}, to: st.shard1.shardName}).ok); function testSelectWithSkip(coll) { for (var i = -100; i < 100; i++) { diff --git a/jstests/sharding/merge_chunks_test_with_md_ops.js b/jstests/sharding/merge_chunks_test_with_md_ops.js index 591413a109c..63b2504521f 100644 --- a/jstests/sharding/merge_chunks_test_with_md_ops.js +++ b/jstests/sharding/merge_chunks_test_with_md_ops.js @@ -7,11 +7,10 @@ var mongos = st.s0; var admin = mongos.getDB("admin"); - var shards = mongos.getCollection("config.shards").find().toArray(); var coll = mongos.getCollection("foo.bar"); assert.commandWorked(admin.runCommand({enableSharding: coll.getDB() + ""})); - st.ensurePrimaryShard(coll.getDB() + "", shards[0]._id); + st.ensurePrimaryShard(coll.getDB() + "", st.shard0.shardName); assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {_id: 1}})); st.printShardingStatus(); @@ -30,7 +29,7 @@ jsTest.log("Moving to another shard..."); assert.commandWorked( - admin.runCommand({moveChunk: coll + "", find: {_id: 0}, to: shards[1]._id})); + admin.runCommand({moveChunk: coll + "", find: {_id: 0}, to: st.shard1.shardName})); // Split and merge the chunk repeatedly jsTest.log("Splitting and merging repeatedly (again)..."); @@ -46,7 +45,7 @@ jsTest.log("Moving to original shard..."); assert.commandWorked( - admin.runCommand({moveChunk: coll + "", find: {_id: 0}, to: shards[0]._id})); + admin.runCommand({moveChunk: coll + "", find: {_id: 0}, to: st.shard0.shardName})); st.printShardingStatus(); diff --git a/jstests/sharding/migrate_overwrite_id.js b/jstests/sharding/migrate_overwrite_id.js index 1d5bc2f3236..8060a2de8b4 100644 --- a/jstests/sharding/migrate_overwrite_id.js +++ b/jstests/sharding/migrate_overwrite_id.js @@ -6,17 +6,14 @@ var st = new ShardingTest({shards: 2, mongos: 1}); st.stopBalancer(); var mongos = st.s0; -var shards = mongos.getDB("config").shards.find().toArray(); -shards[0].conn = st.shard0; -shards[1].conn = st.shard1; var admin = mongos.getDB("admin"); var coll = mongos.getCollection("foo.bar"); assert(admin.runCommand({enableSharding: coll.getDB() + ""}).ok); -printjson(admin.runCommand({movePrimary: coll.getDB() + "", to: shards[0]._id})); +printjson(admin.runCommand({movePrimary: coll.getDB() + "", to: st.shard0.shardName})); assert(admin.runCommand({shardCollection: coll + "", key: {skey: 1}}).ok); assert(admin.runCommand({split: coll + "", middle: {skey: 0}}).ok); -assert(admin.runCommand({moveChunk: coll + "", find: {skey: 0}, to: shards[1]._id}).ok); +assert(admin.runCommand({moveChunk: coll + "", find: {skey: 0}, to: st.shard1.shardName}).ok); var id = 12345; @@ -25,17 +22,17 @@ jsTest.log("Inserting a document with id : 12345 into both shards with diff shar assert.writeOK(coll.insert({_id: id, skey: -1})); assert.writeOK(coll.insert({_id: id, skey: 1})); -printjson(shards[0].conn.getCollection(coll + "").find({_id: id}).toArray()); -printjson(shards[1].conn.getCollection(coll + "").find({_id: id}).toArray()); +printjson(st.shard0.getCollection(coll + "").find({_id: id}).toArray()); +printjson(st.shard1.getCollection(coll + "").find({_id: id}).toArray()); assert.eq(2, coll.find({_id: id}).itcount()); jsTest.log("Moving both chunks to same shard..."); -var result = admin.runCommand({moveChunk: coll + "", find: {skey: -1}, to: shards[1]._id}); +var result = admin.runCommand({moveChunk: coll + "", find: {skey: -1}, to: st.shard1.shardName}); printjson(result); -printjson(shards[0].conn.getCollection(coll + "").find({_id: id}).toArray()); -printjson(shards[1].conn.getCollection(coll + "").find({_id: id}).toArray()); +printjson(st.shard0.getCollection(coll + "").find({_id: id}).toArray()); +printjson(st.shard1.getCollection(coll + "").find({_id: id}).toArray()); assert.eq(2, coll.find({_id: id}).itcount()); st.stop(); diff --git a/jstests/sharding/migration_sets_fromMigrate_flag.js b/jstests/sharding/migration_sets_fromMigrate_flag.js index 55dbca8b5fa..a61e2efd7e6 100644 --- a/jstests/sharding/migration_sets_fromMigrate_flag.js +++ b/jstests/sharding/migration_sets_fromMigrate_flag.js @@ -28,12 +28,10 @@ load('./jstests/libs/chunk_manipulation_util.js'); var st = new ShardingTest({shards: 2, mongos: 1, rs: {nodes: 3}}); st.stopBalancer(); - var mongos = st.s0, admin = mongos.getDB('admin'), - shards = mongos.getCollection('config.shards').find().toArray(), dbName = "testDB", - ns = dbName + ".foo", coll = mongos.getCollection(ns), donor = st.shard0, - recipient = st.shard1, donorColl = donor.getCollection(ns), - recipientColl = recipient.getCollection(ns), donorLocal = donor.getDB('local'), - recipientLocal = recipient.getDB('local'); + var mongos = st.s0, admin = mongos.getDB('admin'), dbName = "testDB", ns = dbName + ".foo", + coll = mongos.getCollection(ns), donor = st.shard0, recipient = st.shard1, + donorColl = donor.getCollection(ns), recipientColl = recipient.getCollection(ns), + donorLocal = donor.getDB('local'), recipientLocal = recipient.getDB('local'); // Two chunks // Donor: [0, 2) [2, 5) @@ -41,7 +39,7 @@ load('./jstests/libs/chunk_manipulation_util.js'); jsTest.log('Enable sharding of the collection and pre-split into two chunks....'); assert.commandWorked(admin.runCommand({enableSharding: dbName})); - st.ensurePrimaryShard(dbName, shards[0]._id); + st.ensurePrimaryShard(dbName, st.shard0.shardName); assert.commandWorked(donorColl.createIndex({_id: 1})); assert.commandWorked(admin.runCommand({shardCollection: ns, key: {_id: 1}})); assert.commandWorked(admin.runCommand({split: ns, middle: {_id: 2}})); @@ -76,7 +74,7 @@ load('./jstests/libs/chunk_manipulation_util.js'); jsTest.log('Starting chunk migration, pause after cloning...'); var joinMoveChunk = moveChunkParallel( - staticMongod, st.s0.host, {_id: 2}, null, coll.getFullName(), shards[1]._id); + staticMongod, st.s0.host, {_id: 2}, null, coll.getFullName(), st.shard1.shardName); /** * Wait for recipient to finish cloning. diff --git a/jstests/sharding/migration_with_source_ops.js b/jstests/sharding/migration_with_source_ops.js index 31b6fff75e9..b837191c4ee 100644 --- a/jstests/sharding/migration_with_source_ops.js +++ b/jstests/sharding/migration_with_source_ops.js @@ -29,11 +29,9 @@ load('./jstests/libs/chunk_manipulation_util.js'); var st = new ShardingTest({shards: 2, mongos: 1}); st.stopBalancer(); - var mongos = st.s0, admin = mongos.getDB('admin'), - shards = mongos.getCollection('config.shards').find().toArray(), dbName = "testDB", - ns = dbName + ".foo", coll = mongos.getCollection(ns), donor = st.shard0, - recipient = st.shard1, donorColl = donor.getCollection(ns), - recipientColl = recipient.getCollection(ns); + var mongos = st.s0, admin = mongos.getDB('admin'), dbName = "testDB", ns = dbName + ".foo", + coll = mongos.getCollection(ns), donor = st.shard0, recipient = st.shard1, + donorColl = donor.getCollection(ns), recipientColl = recipient.getCollection(ns); /** * Exable sharding, and split collection into two chunks. @@ -44,7 +42,7 @@ load('./jstests/libs/chunk_manipulation_util.js'); // Recipient: jsTest.log('Enabling sharding of the collection and pre-splitting into two chunks....'); assert.commandWorked(admin.runCommand({enableSharding: dbName})); - st.ensurePrimaryShard(dbName, shards[0]._id); + st.ensurePrimaryShard(dbName, st.shard0.shardName); assert.commandWorked(admin.runCommand({shardCollection: ns, key: {a: 1}})); assert.commandWorked(admin.runCommand({split: ns, middle: {a: 20}})); @@ -84,7 +82,7 @@ load('./jstests/libs/chunk_manipulation_util.js'); // Recipient: [20, 40) jsTest.log('Starting migration, pause after cloning...'); var joinMoveChunk = moveChunkParallel( - staticMongod, st.s0.host, {a: 20}, null, coll.getFullName(), shards[1]._id); + staticMongod, st.s0.host, {a: 20}, null, coll.getFullName(), st.shard1.shardName); /** * Wait for recipient to finish cloning step. diff --git a/jstests/sharding/move_primary_basic.js b/jstests/sharding/move_primary_basic.js index 288d4fb03e5..1fd75364f15 100644 --- a/jstests/sharding/move_primary_basic.js +++ b/jstests/sharding/move_primary_basic.js @@ -11,10 +11,8 @@ var kDbName = 'db'; - var shards = mongos.getCollection('config.shards').find().toArray(); - - var shard0 = shards[0]._id; - var shard1 = shards[1]._id; + var shard0 = st.shard0.shardName; + var shard1 = st.shard1.shardName; assert.commandWorked(mongos.adminCommand({enableSharding: kDbName})); st.ensurePrimaryShard(kDbName, shard0); diff --git a/jstests/sharding/moveprimary_ignore_sharded.js b/jstests/sharding/moveprimary_ignore_sharded.js index f73f50939cc..8bad709bd1d 100644 --- a/jstests/sharding/moveprimary_ignore_sharded.js +++ b/jstests/sharding/moveprimary_ignore_sharded.js @@ -37,21 +37,20 @@ printjson(adminA.runCommand({shardCollection: "bar.coll1", key: {_id: 1}})); printjson(adminA.runCommand({shardCollection: "bar.coll2", key: {_id: 1}})); // All collections are now on primary shard -var fooPrimaryShard = configA.databases.findOne({_id: "foo"}).primary; -var barPrimaryShard = configA.databases.findOne({_id: "bar"}).primary; +var fooPrimaryShardId = configA.databases.findOne({_id: "foo"}).primary; +var barPrimaryShardId = configA.databases.findOne({_id: "bar"}).primary; -var shards = configA.shards.find().toArray(); -var fooPrimaryShard = fooPrimaryShard == shards[0]._id ? shards[0] : shards[1]; -var fooOtherShard = fooPrimaryShard._id == shards[0]._id ? shards[1] : shards[0]; -var barPrimaryShard = barPrimaryShard == shards[0]._id ? shards[0] : shards[1]; -var barOtherShard = barPrimaryShard._id == shards[0]._id ? shards[1] : shards[0]; +var fooPrimaryShard = (fooPrimaryShardId == st.shard0.shardName) ? st.shard0 : st.shard1; +var fooOtherShard = (fooPrimaryShard.shardName == st.shard0.shardName) ? st.shard1 : st.shard0; +var barPrimaryShard = (barPrimaryShardId == st.shard0.shardName) ? st.shard0 : st.shard1; +var barOtherShard = (barPrimaryShard.shardName == st.shard0.shardName) ? st.shard1 : st.shard0; st.printShardingStatus(); jsTest.log("Running movePrimary for foo through mongosA ..."); // MongosA should already know about all the collection states -printjson(adminA.runCommand({movePrimary: "foo", to: fooOtherShard._id})); +printjson(adminA.runCommand({movePrimary: "foo", to: fooOtherShard.shardName})); if (st.configRS) { // If we are in CSRS mode need to make sure that mongosB will actually get the most recent @@ -78,11 +77,11 @@ function realCollectionCount(mydb) { } // All collections sane -assert.eq(2, realCollectionCount(new Mongo(fooPrimaryShard.host).getDB("foo"))); -assert.eq(1, realCollectionCount(new Mongo(fooOtherShard.host).getDB("foo"))); +assert.eq(2, realCollectionCount(fooPrimaryShard.getDB("foo"))); +assert.eq(1, realCollectionCount(fooOtherShard.getDB("foo"))); jsTest.log("Running movePrimary for bar through mongosB ..."); -printjson(adminB.runCommand({movePrimary: "bar", to: barOtherShard._id})); +printjson(adminB.runCommand({movePrimary: "bar", to: barOtherShard.shardName})); // We need to flush the cluster config on mongosA, so it can discover that database 'bar' got // moved. Otherwise since the collections are not sharded, we have no way of discovering this. @@ -104,7 +103,7 @@ assert.neq(null, mongosB.getCollection("bar.coll1").findOne()); assert.neq(null, mongosB.getCollection("bar.coll2").findOne()); // All collections sane -assert.eq(2, realCollectionCount(new Mongo(barPrimaryShard.host).getDB("bar"))); -assert.eq(1, realCollectionCount(new Mongo(barOtherShard.host).getDB("bar"))); +assert.eq(2, realCollectionCount(barPrimaryShard.getDB("bar"))); +assert.eq(1, realCollectionCount(barOtherShard.getDB("bar"))); st.stop(); diff --git a/jstests/sharding/pending_chunk.js b/jstests/sharding/pending_chunk.js index 96089b6d491..14e7c3ebf61 100644 --- a/jstests/sharding/pending_chunk.js +++ b/jstests/sharding/pending_chunk.js @@ -9,24 +9,22 @@ var mongos = st.s0; var admin = mongos.getDB('admin'); - var shards = mongos.getCollection('config.shards').find().toArray(); var coll = mongos.getCollection('foo.bar'); var ns = coll.getFullName(); var dbName = coll.getDB().getName(); - var shard0 = st.shard0, shard1 = st.shard1; assert.commandWorked(admin.runCommand({enableSharding: dbName})); - printjson(admin.runCommand({movePrimary: dbName, to: shards[0]._id})); + printjson(admin.runCommand({movePrimary: dbName, to: st.shard0.shardName})); assert.commandWorked(admin.runCommand({shardCollection: ns, key: {_id: 1}})); jsTest.log('Moving some chunks to shard1...'); assert.commandWorked(admin.runCommand({split: ns, middle: {_id: 0}})); assert.commandWorked(admin.runCommand({split: ns, middle: {_id: 1}})); - assert.commandWorked( - admin.runCommand({moveChunk: ns, find: {_id: 0}, to: shards[1]._id, _waitForDelete: true})); - assert.commandWorked( - admin.runCommand({moveChunk: ns, find: {_id: 1}, to: shards[1]._id, _waitForDelete: true})); + assert.commandWorked(admin.runCommand( + {moveChunk: ns, find: {_id: 0}, to: st.shard1.shardName, _waitForDelete: true})); + assert.commandWorked(admin.runCommand( + {moveChunk: ns, find: {_id: 1}, to: st.shard1.shardName, _waitForDelete: true})); function getMetadata(shard) { var admin = shard.getDB('admin'), @@ -36,24 +34,24 @@ return metadata; } - var metadata = getMetadata(shard1); + var metadata = getMetadata(st.shard1); assert.eq(metadata.pending[0][0]._id, 1); assert.eq(metadata.pending[0][1]._id, MaxKey); jsTest.log('Moving some chunks back to shard0 after empty...'); assert.commandWorked(admin.runCommand( - {moveChunk: ns, find: {_id: -1}, to: shards[1]._id, _waitForDelete: true})); + {moveChunk: ns, find: {_id: -1}, to: st.shard1.shardName, _waitForDelete: true})); - metadata = getMetadata(shard0); + metadata = getMetadata(st.shard0); assert.eq(metadata.shardVersion.t, 0); assert.neq(metadata.collVersion.t, 0); assert.eq(metadata.pending.length, 0); - assert.commandWorked( - admin.runCommand({moveChunk: ns, find: {_id: 1}, to: shards[0]._id, _waitForDelete: true})); + assert.commandWorked(admin.runCommand( + {moveChunk: ns, find: {_id: 1}, to: st.shard0.shardName, _waitForDelete: true})); - metadata = getMetadata(shard0); + metadata = getMetadata(st.shard0); assert.eq(metadata.shardVersion.t, 0); assert.neq(metadata.collVersion.t, 0); assert.eq(metadata.pending[0][0]._id, 1); @@ -65,7 +63,7 @@ assert.eq(null, coll.findOne({_id: 1})); - metadata = getMetadata(shard0); + metadata = getMetadata(st.shard0); assert.neq(metadata.shardVersion.t, 0); assert.neq(metadata.collVersion.t, 0); assert.eq(metadata.chunks[0][0]._id, 1); diff --git a/jstests/sharding/prefix_shard_key.js b/jstests/sharding/prefix_shard_key.js index a13b133e3ef..71a7ef03090 100644 --- a/jstests/sharding/prefix_shard_key.js +++ b/jstests/sharding/prefix_shard_key.js @@ -14,9 +14,6 @@ var db = s.getDB("test"); var admin = s.getDB("admin"); var config = s.getDB("config"); - var shards = config.shards.find().toArray(); - var shard0 = new Mongo(shards[0].host); - var shard1 = new Mongo(shards[1].host); assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); @@ -127,13 +124,13 @@ } }); - assert.eq(expectedShardCount['shard0000'], shard0.getDB('test').user.find().count()); - assert.eq(expectedShardCount['shard0001'], shard1.getDB('test').user.find().count()); + assert.eq(expectedShardCount['shard0000'], s.shard0.getDB('test').user.find().count()); + assert.eq(expectedShardCount['shard0001'], s.shard1.getDB('test').user.find().count()); assert.commandWorked(admin.runCommand({split: 'test.user', middle: {num: 70}})); - assert.eq(expectedShardCount['shard0000'], shard0.getDB('test').user.find().count()); - assert.eq(expectedShardCount['shard0001'], shard1.getDB('test').user.find().count()); + assert.eq(expectedShardCount['shard0000'], s.shard0.getDB('test').user.find().count()); + assert.eq(expectedShardCount['shard0001'], s.shard1.getDB('test').user.find().count()); //******************Part 3******************** @@ -144,8 +141,9 @@ // setup new collection on shard0 var coll2 = db.foo2; coll2.drop(); - if (s.getPrimaryShardIdForDatabase(coll2.getDB()) != shards[0]._id) { - var moveRes = admin.runCommand({movePrimary: coll2.getDB() + "", to: shards[0]._id}); + if (s.getPrimaryShardIdForDatabase(coll2.getDB()) != s.shard0.shardName) { + var moveRes = + admin.runCommand({movePrimary: coll2.getDB() + "", to: s.shard0.shardName}); assert.eq(moveRes.ok, 1, "primary not moved correctly"); } @@ -178,7 +176,7 @@ // movechunk should move ALL docs since they have same value for skey moveRes = admin.runCommand( - {moveChunk: coll2 + "", find: {skey: 0}, to: shards[1]._id, _waitForDelete: true}); + {moveChunk: coll2 + "", find: {skey: 0}, to: s.shard1.shardName, _waitForDelete: true}); assert.eq(moveRes.ok, 1, "movechunk didn't work"); // Make sure our migration eventually goes through before testing individual shards @@ -188,8 +186,8 @@ }); // check no orphaned docs on the shards - assert.eq(0, shard0.getCollection(coll2 + "").find().itcount()); - assert.eq(25, shard1.getCollection(coll2 + "").find().itcount()); + assert.eq(0, s.shard0.getCollection(coll2 + "").find().itcount()); + assert.eq(25, s.shard1.getCollection(coll2 + "").find().itcount()); // and check total assert.eq(25, coll2.find().itcount(), "bad total number of docs after move"); diff --git a/jstests/sharding/regex_targeting.js b/jstests/sharding/regex_targeting.js index 2a8ca1ad7d5..e55e0f6cab9 100644 --- a/jstests/sharding/regex_targeting.js +++ b/jstests/sharding/regex_targeting.js @@ -6,7 +6,6 @@ var mongos = st.s0; var admin = mongos.getDB("admin"); - var shards = mongos.getDB("config").shards.find().toArray(); // // Set up multiple collections to target with regex shard keys on two shards @@ -19,7 +18,7 @@ var collHashed = mongos.getCollection("foo.barHashed"); assert.commandWorked(admin.runCommand({enableSharding: coll.getDB().toString()})); - st.ensurePrimaryShard(coll.getDB().toString(), shards[0]._id); + st.ensurePrimaryShard(coll.getDB().toString(), st.shard0.shardName); // // Split the collection so that "abcde-0" and "abcde-1" go on different shards when possible @@ -30,7 +29,7 @@ assert.commandWorked(admin.runCommand({ moveChunk: collSharded.toString(), find: {a: 0}, - to: shards[1]._id, + to: st.shard1.shardName, _waitForDelete: true })); @@ -41,7 +40,7 @@ assert.commandWorked(admin.runCommand({ moveChunk: collCompound.toString(), find: {a: 0, b: 0}, - to: shards[1]._id, + to: st.shard1.shardName, _waitForDelete: true })); @@ -52,7 +51,7 @@ assert.commandWorked(admin.runCommand({ moveChunk: collNested.toString(), find: {a: {b: 0}}, - to: shards[1]._id, + to: st.shard1.shardName, _waitForDelete: true })); diff --git a/jstests/sharding/return_partial_shards_down.js b/jstests/sharding/return_partial_shards_down.js index a8eca975283..6d1c215127f 100644 --- a/jstests/sharding/return_partial_shards_down.js +++ b/jstests/sharding/return_partial_shards_down.js @@ -8,19 +8,13 @@ var st = new ShardingTest({shards: 3, mongos: 1, other: {mongosOptions: {verbose st.stopBalancer(); var mongos = st.s; -var config = mongos.getDB("config"); var admin = mongos.getDB("admin"); -var shards = config.shards.find().toArray(); - -for (var i = 0; i < shards.length; i++) { - shards[i].conn = new Mongo(shards[i].host); -} var collOneShard = mongos.getCollection("foo.collOneShard"); var collAllShards = mongos.getCollection("foo.collAllShards"); printjson(admin.runCommand({enableSharding: collOneShard.getDB() + ""})); -printjson(admin.runCommand({movePrimary: collOneShard.getDB() + "", to: shards[0]._id})); +printjson(admin.runCommand({movePrimary: collOneShard.getDB() + "", to: st.shard0.shardName})); printjson(admin.runCommand({shardCollection: collOneShard + "", key: {_id: 1}})); printjson(admin.runCommand({shardCollection: collAllShards + "", key: {_id: 1}})); @@ -29,8 +23,10 @@ printjson(admin.runCommand({shardCollection: collAllShards + "", key: {_id: 1}}) printjson(admin.runCommand({split: collAllShards + "", middle: {_id: 0}})); printjson(admin.runCommand({split: collAllShards + "", middle: {_id: 1000}})); -printjson(admin.runCommand({moveChunk: collAllShards + "", find: {_id: 0}, to: shards[1]._id})); -printjson(admin.runCommand({moveChunk: collAllShards + "", find: {_id: 1000}, to: shards[2]._id})); +printjson( + admin.runCommand({moveChunk: collAllShards + "", find: {_id: 0}, to: st.shard1.shardName})); +printjson( + admin.runCommand({moveChunk: collAllShards + "", find: {_id: 1000}, to: st.shard2.shardName})); // Collections are now distributed correctly jsTest.log("Collections now distributed correctly."); diff --git a/jstests/sharding/shard_insert_getlasterror_w2.js b/jstests/sharding/shard_insert_getlasterror_w2.js index 09ea5b5ec46..ac80ebade7f 100644 --- a/jstests/sharding/shard_insert_getlasterror_w2.js +++ b/jstests/sharding/shard_insert_getlasterror_w2.js @@ -42,7 +42,7 @@ var testDB = mongosConn.getDB(testDBName); // Add replSet1 as only shard - mongosConn.adminCommand({addshard: replSet1.getURL()}); + assert.commandWorked(mongosConn.adminCommand({addshard: replSet1.getURL()})); // Enable sharding on test db and its collection foo assert.commandWorked(mongosConn.getDB('admin').runCommand({enablesharding: testDBName})); @@ -51,7 +51,7 @@ {shardcollection: testDBName + '.' + testCollName, key: {x: 1}})); // Test case where GLE should return an error - testDB.foo.insert({_id: 'a', x: 1}); + assert.writeOK(testDB.foo.insert({_id: 'a', x: 1})); assert.writeError(testDB.foo.insert({_id: 'a', x: 1}, {writeConcern: {w: 2, wtimeout: 30000}})); // Add more data diff --git a/jstests/sharding/test_stacked_migration_cleanup.js b/jstests/sharding/test_stacked_migration_cleanup.js index 523f5de1a0c..b8baba5f5b2 100644 --- a/jstests/sharding/test_stacked_migration_cleanup.js +++ b/jstests/sharding/test_stacked_migration_cleanup.js @@ -8,12 +8,11 @@ var mongos = st.s; var admin = mongos.getDB("admin"); - var shards = mongos.getDB("config").shards.find().toArray(); var coll = mongos.getCollection("foo.bar"); // Enable sharding of the collection assert.commandWorked(mongos.adminCommand({enablesharding: coll.getDB() + ""})); - st.ensurePrimaryShard(coll.getDB() + "", shards[0]._id); + st.ensurePrimaryShard(coll.getDB() + "", st.shard0.shardName); assert.commandWorked(mongos.adminCommand({shardcollection: coll + "", key: {_id: 1}})); var numChunks = 30; @@ -43,7 +42,7 @@ // Move a bunch of chunks, but don't close the cursor so they stack. for (var i = 0; i < numChunks; i++) { assert.commandWorked( - mongos.adminCommand({moveChunk: coll + "", find: {_id: i}, to: shards[1]._id})); + mongos.adminCommand({moveChunk: coll + "", find: {_id: i}, to: st.shard1.shardName})); } jsTest.log("Dropping and re-creating collection..."); diff --git a/jstests/sharding/trace_missing_docs_test.js b/jstests/sharding/trace_missing_docs_test.js index 7e5eaf83cc6..b09003617d4 100644 --- a/jstests/sharding/trace_missing_docs_test.js +++ b/jstests/sharding/trace_missing_docs_test.js @@ -16,10 +16,9 @@ load('jstests/libs/trace_missing_docs.js'); var mongos = st.s0; var coll = mongos.getCollection("foo.bar"); var admin = mongos.getDB("admin"); - var shards = mongos.getCollection("config.shards").find().toArray(); assert.commandWorked(admin.runCommand({enableSharding: coll.getDB() + ""})); - st.ensurePrimaryShard(coll.getDB() + "", shards[0]._id); + st.ensurePrimaryShard(coll.getDB() + "", st.shard0.shardName); coll.ensureIndex({sk: 1}); assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {sk: 1}})); @@ -29,7 +28,7 @@ load('jstests/libs/trace_missing_docs.js'); assert.writeOK(coll.update({sk: 67890}, {$set: {baz: 'boz'}})); assert.commandWorked(admin.runCommand( - {moveChunk: coll + "", find: {sk: 0}, to: shards[1]._id, _waitForDelete: true})); + {moveChunk: coll + "", find: {sk: 0}, to: st.shard1.shardName, _waitForDelete: true})); st.printShardingStatus(); diff --git a/jstests/sharding/upsert_sharded.js b/jstests/sharding/upsert_sharded.js index c8398142768..9ee8f72d1bc 100644 --- a/jstests/sharding/upsert_sharded.js +++ b/jstests/sharding/upsert_sharded.js @@ -9,7 +9,6 @@ var mongos = st.s0; var admin = mongos.getDB("admin"); - var shards = mongos.getCollection("config.shards").find().toArray(); var coll = mongos.getCollection("foo.bar"); assert(admin.runCommand({enableSharding: coll.getDB() + ""}).ok); @@ -33,11 +32,11 @@ return upsertedField(query, expr, "x"); }; - st.ensurePrimaryShard(coll.getDB() + "", shards[0]._id); + st.ensurePrimaryShard(coll.getDB() + "", st.shard0.shardName); assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {x: 1}})); assert.commandWorked(admin.runCommand({split: coll + "", middle: {x: 0}})); assert.commandWorked(admin.runCommand( - {moveChunk: coll + "", find: {x: 0}, to: shards[1]._id, _waitForDelete: true})); + {moveChunk: coll + "", find: {x: 0}, to: st.shard1.shardName, _waitForDelete: true})); st.printShardingStatus(); @@ -70,11 +69,11 @@ coll.drop(); - st.ensurePrimaryShard(coll.getDB() + "", shards[0]._id); + st.ensurePrimaryShard(coll.getDB() + "", st.shard0.shardName); assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {'x.x': 1}})); assert.commandWorked(admin.runCommand({split: coll + "", middle: {'x.x': 0}})); assert.commandWorked(admin.runCommand( - {moveChunk: coll + "", find: {'x.x': 0}, to: shards[1]._id, _waitForDelete: true})); + {moveChunk: coll + "", find: {'x.x': 0}, to: st.shard1.shardName, _waitForDelete: true})); st.printShardingStatus(); diff --git a/jstests/ssl/ssl_client_certificate_warning_suppression.js b/jstests/ssl/ssl_client_certificate_warning_suppression.js new file mode 100644 index 00000000000..f2bbf93e110 --- /dev/null +++ b/jstests/ssl/ssl_client_certificate_warning_suppression.js @@ -0,0 +1,65 @@ +/** + * Tests the startup-only setParameter value suppressNoTLSPeerCertificateWarning which suppresses + * the log message "no SSL certificate provided by peer" when a client certificate is not provided. + * This only works if weak validation is enabled. + * + * This test confirms that the log message is output when the setParameter is set to true, + * and is not output when the setParameter is set to false. + */ + +load('jstests/ssl/libs/ssl_helpers.js'); + +(function() { + 'use strict'; + + function test(suppress) { + const opts = { + sslMode: 'requireSSL', + sslPEMKeyFile: "jstests/libs/server.pem", + sslCAFile: "jstests/libs/ca.pem", + waitForConnect: false, + sslAllowConnectionsWithoutCertificates: "", + setParameter: {suppressNoTLSPeerCertificateWarning: suppress} + }; + clearRawMongoProgramOutput(); + const mongod = MongoRunner.runMongod(opts); + + assert.soon(function() { + return runMongoProgram('mongo', + '--ssl', + '--sslAllowInvalidHostnames', + '--sslCAFile', + CA_CERT, + '--port', + mongod.port, + '--eval', + 'quit()') === 0; + }, "mongo did not initialize properly"); + + // Keep checking the log file until client metadata is logged since the SSL warning is + // logged before it. + assert.soon( + () => { + const log = rawMongoProgramOutput(); + return log.search('client metadata') !== -1; + }, + "logfile should contain 'client metadata'.\n" + + "Log File Contents\n==============================\n" + rawMongoProgramOutput() + + "\n==============================\n"); + + // Now check for the message + const log = rawMongoProgramOutput(); + assert.eq(suppress, log.search('no SSL certificate provided by peer') === -1); + + try { + MongoRunner.stopMongod(mongod); + } catch (e) { + // Depending on timing, exitCode might be 0, 1, or -9. + // All that matters is that it dies, resmoke will tell us if that failed. + // So just let it go, the exit code never bothered us anyway. + } + } + + test(true); + test(false); +})(); diff --git a/jstests/ssl/ssl_cluster_ca.js b/jstests/ssl/ssl_cluster_ca.js new file mode 100644 index 00000000000..2ee05406ea1 --- /dev/null +++ b/jstests/ssl/ssl_cluster_ca.js @@ -0,0 +1,82 @@ +// Verify certificates and CAs between intra-cluster +// and client->server communication using different CAs. + +(function() { + "use strict"; + + function testRS(opts, succeed) { + const origSkipCheck = TestData.skipCheckDBHashes; + const rsOpts = { + // Use localhost so that SAN matches. + useHostName: false, + nodes: {node0: opts, node1: opts}, + }; + const rs = new ReplSetTest(rsOpts); + rs.startSet(); + if (succeed) { + rs.initiate(); + assert.commandWorked(rs.getPrimary().getDB('admin').runCommand({isMaster: 1})); + } else { + assert.throws(function() { + rs.initiate(); + }); + TestData.skipCheckDBHashes = true; + } + rs.stopSet(); + TestData.skipCheckDBHashes = origSkipCheck; + } + + // The name "trusted" in these certificates is misleading. + // They're just a separate trust chain from the ones without the name. + // ca.pem signed client.pem and server.pem + // trusted-ca.pem signed trusted-client.pem and trusted-server.pem + const valid_options = { + sslMode: 'requireSSL', + // Servers present trusted-server.pem to clients and each other for inbound connections. + // Peers validate trusted-server.pem using trusted-ca.pem when making those connections. + sslPEMKeyFile: 'jstests/libs/trusted-server.pem', + sslCAFile: 'jstests/libs/trusted-ca.pem', + // Servers making outbound connections to other servers present server.pem to their peers + // which their peers validate using ca.pem. + sslClusterFile: 'jstests/libs/server.pem', + sslClusterCAFile: 'jstests/libs/ca.pem', + // SERVER-36895: IP based hostname validation with SubjectAlternateName + sslAllowInvalidHostnames: '', + }; + + testRS(valid_options, true); + + const wrong_cluster_file = + Object.assign({}, valid_options, {sslClusterFile: valid_options.sslPEMKeyFile}); + testRS(wrong_cluster_file, false); + + const wrong_key_file = + Object.assign({}, valid_options, {sslPEMKeyFile: valid_options.sslClusterFile}); + testRS(wrong_key_file, false); + + const mongod = MongoRunner.runMongod(valid_options); + assert(mongod, "Failed starting standalone mongod with alternate CA"); + + function testConnect(cert, succeed) { + const mongo = runMongoProgram("mongo", + "--host", + "localhost", + "--port", + mongod.port, + "--ssl", + "--sslCAFile", + valid_options.sslCAFile, + "--sslPEMKeyFile", + cert, + "--eval", + ";"); + + // runMongoProgram returns 0 on success + assert.eq(mongo === 0, succeed); + } + + testConnect('jstests/libs/client.pem', true); + testConnect('jstests/libs/trusted-client.pem', false); + + MongoRunner.stopMongod(mongod); +}()); diff --git a/jstests/ssl/ssl_count_protocols.js b/jstests/ssl/ssl_count_protocols.js new file mode 100644 index 00000000000..b19268a01ba --- /dev/null +++ b/jstests/ssl/ssl_count_protocols.js @@ -0,0 +1,63 @@ +// Ensure the server counts the server TLS versions used +(function() { + 'use strict'; + + var SERVER_CERT = "jstests/libs/server.pem"; + var CLIENT_CERT = "jstests/libs/client.pem"; + var CA_CERT = "jstests/libs/ca.pem"; + + function runTestWithoutSubset(client) { + let disabledProtocols = ["TLS1_0", "TLS1_1", "TLS1_2"]; + let expectedCounts = [0, 0, 0]; + let clientIndex = 2; + if (getBuildInfo().buildEnvironment.target_os === "osx") { + clientIndex = 0; + } + expectedCounts[clientIndex] = 1; + var index = disabledProtocols.indexOf(client); + disabledProtocols.splice(index, 1); + expectedCounts[index] += 1; + + const conn = MongoRunner.runMongod({ + sslMode: 'allowSSL', + sslPEMKeyFile: SERVER_CERT, + sslDisabledProtocols: 'none', + }); + + print(disabledProtocols); + const version_number = client.replace(/TLS/, "").replace(/_/, "."); + + const exitStatus = + runMongoProgram('mongo', + '--ssl', + '--sslAllowInvalidHostnames', + '--sslPEMKeyFile', + CLIENT_CERT, + '--sslCAFile', + CA_CERT, + '--port', + conn.port, + '--sslDisabledProtocols', + disabledProtocols.join(","), + '--eval', + // The Javascript string "1.0" is implicitly converted to the Number(1) + // Workaround this with parseFloat + 'one = Number.parseFloat(1).toPrecision(2); a = {};' + + 'a[one] = NumberLong(' + expectedCounts[0] + ');' + + 'a["1.1"] = NumberLong(' + expectedCounts[1] + ');' + + 'a["1.2"] = NumberLong(' + expectedCounts[2] + ');' + + 'assert.eq(db.serverStatus().transportSecurity, a);'); + + assert.eq(0, exitStatus, ""); + + MongoRunner.stopMongod(conn); + } + + runTestWithoutSubset("TLS1_0"); + + // OpenSSL 0.9.8 on macOS only supports TLS 1.0 + if (getBuildInfo().buildEnvironment.target_os !== "osx") { + runTestWithoutSubset("TLS1_1"); + runTestWithoutSubset("TLS1_2"); + } +})(); diff --git a/jstests/ssl/ssl_withhold_client_cert.js b/jstests/ssl/ssl_withhold_client_cert.js new file mode 100644 index 00000000000..f8d7d287519 --- /dev/null +++ b/jstests/ssl/ssl_withhold_client_cert.js @@ -0,0 +1,45 @@ +// Test setParameter sslWithholdClientCertificate + +(function() { + "use strict"; + + function testRS(opts, expectWarning) { + const rsOpts = { + nodes: {node0: opts, node1: opts}, + }; + const rs = new ReplSetTest(rsOpts); + rs.startSet(); + rs.initiate(); + rs.awaitReplication(); + + const test = rs.getPrimary().getDB('test'); + test.foo.insert({bar: "baz"}); + rs.awaitReplication(); + + function checkWarning(member) { + const observed = + /no SSL certificate provided by peer/.test(cat(member.fullOptions.logFile)); + assert.eq(observed, expectWarning); + } + checkWarning(rs.getPrimary()); + checkWarning(rs.getSecondary()); + rs.stopSet(); + } + + const base_options = { + sslMode: 'requireSSL', + sslPEMKeyFile: 'jstests/libs/server.pem', + sslCAFile: 'jstests/libs/ca.pem', + sslAllowInvalidHostnames: '', + useLogFiles: true, + }; + testRS(base_options, false); + + const test_options = Object.extend({ + sslAllowConnectionsWithoutCertificates: '', + setParameter: 'sslWithholdClientCertificate=true', + }, + base_options); + + testRS(test_options, true); +}()); diff --git a/jstests/ssl/x509_custom.js b/jstests/ssl/x509_custom.js new file mode 100644 index 00000000000..a52844a0efa --- /dev/null +++ b/jstests/ssl/x509_custom.js @@ -0,0 +1,56 @@ +// Test X509 auth with custom OIDs. + +(function() { + 'use strict'; + + const SERVER_CERT = 'jstests/libs/server.pem'; + const CA_CERT = 'jstests/libs/ca.pem'; + + function testClient(conn, name) { + let auth = {mechanism: 'MONGODB-X509'}; + if (name !== null) { + auth.user = name; + } + const script = 'assert(db.getSiblingDB(\'$external\').auth(' + tojson(auth) + '));'; + clearRawMongoProgramOutput(); + const exitCode = runMongoProgram('mongo', + '--ssl', + '--sslAllowInvalidHostnames', + '--sslPEMKeyFile', + 'jstests/libs/client-custom-oids.pem', + '--sslCAFile', + CA_CERT, + '--port', + conn.port, + '--eval', + script); + + assert.eq(exitCode, 0); + } + + function runTest(conn) { + const NAME = + 'C=US,ST=New York,L=New York City,O=MongoDB,OU=KernelUser,CN=client,1.2.3.56=RandoValue,1.2.3.45=Value\\,Rando'; + + const admin = conn.getDB('admin'); + admin.createUser({user: "admin", pwd: "admin", roles: ["root"]}); + admin.auth('admin', 'admin'); + + const external = conn.getDB('$external'); + external.createUser({user: NAME, roles: [{'role': 'readWrite', 'db': 'test'}]}); + + testClient(conn, NAME); + testClient(conn, null); + } + + // Standalone. + const mongod = MongoRunner.runMongod({ + auth: '', + sslMode: 'requireSSL', + sslPEMKeyFile: SERVER_CERT, + sslCAFile: CA_CERT, + sslAllowInvalidCertificates: '', + }); + runTest(mongod); + MongoRunner.stopMongod(mongod); +})(); diff --git a/jstests/ssl/x509_multivalue.js b/jstests/ssl/x509_multivalue.js new file mode 100644 index 00000000000..fc7ef3b0c76 --- /dev/null +++ b/jstests/ssl/x509_multivalue.js @@ -0,0 +1,55 @@ +// Test X509 auth with custom OIDs. + +(function() { + 'use strict'; + + const SERVER_CERT = 'jstests/libs/server.pem'; + const CA_CERT = 'jstests/libs/ca.pem'; + + function testClient(conn, name) { + let auth = {mechanism: 'MONGODB-X509'}; + if (name !== null) { + auth.name = name; + } + const script = 'assert(db.getSiblingDB(\'$external\').auth(' + tojson(auth) + '));'; + clearRawMongoProgramOutput(); + const exitCode = runMongoProgram('mongo', + '--ssl', + '--sslAllowInvalidHostnames', + '--sslPEMKeyFile', + 'jstests/libs/client-multivalue-rdn.pem', + '--sslCAFile', + CA_CERT, + '--port', + conn.port, + '--eval', + script); + + assert.eq(exitCode, 0); + } + + function runTest(conn) { + const NAME = 'L=New York City+ST=New York+C=US,OU=KernelUser+O=MongoDB+CN=client'; + + const admin = conn.getDB('admin'); + admin.createUser({user: "admin", pwd: "admin", roles: ["root"]}); + admin.auth('admin', 'admin'); + + const external = conn.getDB('$external'); + external.createUser({user: NAME, roles: [{'role': 'readWrite', 'db': 'test'}]}); + + testClient(conn, NAME); + testClient(conn, null); + } + + // Standalone. + const mongod = MongoRunner.runMongod({ + auth: '', + sslMode: 'requireSSL', + sslPEMKeyFile: SERVER_CERT, + sslCAFile: CA_CERT, + sslAllowInvalidCertificates: '', + }); + runTest(mongod); + MongoRunner.stopMongod(mongod); +})(); diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 66ba4054a11..de51c167ce8 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -355,6 +355,7 @@ env.Install( 'util/clock_sources', 'util/fail_point', 'util/ntservice', + 'util/net/ssl_manager_status', 'util/options_parser/options_parser_init', 'util/version_impl', ])) diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index 98d41461fce..0604e9b4cf7 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -201,6 +201,7 @@ error_code("ChunkRangeCleanupPending", 200) error_code("CannotBuildIndexKeys", 201) error_code("NetworkInterfaceExceededTimeLimit", 202) error_code("TooManyLocks", 208) +error_code("KeyNotFound", 211) error_code("UpdateOperationFailed", 218) error_code("FTDCPathNotSet", 219) error_code("FTDCPathAlreadySet", 220) diff --git a/src/mongo/client/dbclient.cpp b/src/mongo/client/dbclient.cpp index 6f75a704d34..baa156c5b9c 100644 --- a/src/mongo/client/dbclient.cpp +++ b/src/mongo/client/dbclient.cpp @@ -441,7 +441,7 @@ void DBClientWithCommands::_auth(const BSONObj& params) { std::string clientName = ""; #ifdef MONGO_CONFIG_SSL if (sslManager() != nullptr) { - clientName = sslManager()->getSSLConfiguration().clientSubjectName; + clientName = sslManager()->getSSLConfiguration().clientSubjectName.toString(); } #endif diff --git a/src/mongo/client/replica_set_monitor_manager.h b/src/mongo/client/replica_set_monitor_manager.h index 2730167668f..731b5122984 100644 --- a/src/mongo/client/replica_set_monitor_manager.h +++ b/src/mongo/client/replica_set_monitor_manager.h @@ -98,11 +98,13 @@ private: // Protects access to the replica set monitors stdx::mutex _mutex; - ReplicaSetMonitorsMap _monitors; // Executor for monitoring replica sets. std::unique_ptr<executor::TaskExecutor> _taskExecutor; + // Needs to be after `_taskExecutor`, so that it will be destroyed before the `_taskExecutor`. + ReplicaSetMonitorsMap _monitors; + void _setupTaskExecutorInLock(const std::string& name); // set to true when shutdown has been called. diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index ded6480875c..6c3638a9a10 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -663,6 +663,7 @@ serveronlyLibdeps = [ "$BUILD_DIR/mongo/util/clock_sources", "$BUILD_DIR/mongo/util/elapsed_tracker", "$BUILD_DIR/mongo/util/net/network", + "$BUILD_DIR/mongo/util/net/ssl_manager_status", "$BUILD_DIR/mongo/db/storage/mmap_v1/file_allocator", "$BUILD_DIR/third_party/shim_snappy", '$BUILD_DIR/mongo/db/ttl_collection_cache', diff --git a/src/mongo/db/auth/auth_index_d.cpp b/src/mongo/db/auth/auth_index_d.cpp index 47987ac40c4..9944a582f5c 100644 --- a/src/mongo/db/auth/auth_index_d.cpp +++ b/src/mongo/db/auth/auth_index_d.cpp @@ -47,6 +47,7 @@ #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/jsobj.h" +#include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/storage/storage_options.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" @@ -105,6 +106,15 @@ void generateSystemIndexForExistingCollection(OperationContext* opCtx, return; } + // Do not try to generate any system indexes on a secondary. + auto replCoord = repl::ReplicationCoordinator::get(opCtx); + uassert(ErrorCodes::NotMaster, + "Not primary while creating authorization index", + replCoord->getReplicationMode() != repl::ReplicationCoordinator::modeReplSet || + replCoord->canAcceptWritesForDatabase(ns.db())); + + invariant(!opCtx->lockState()->inAWriteUnitOfWork()); + try { auto indexSpecStatus = index_key_validate::validateIndexSpec( spec.toBSON(), ns, serverGlobalParams.featureCompatibility); @@ -115,8 +125,10 @@ void generateSystemIndexForExistingCollection(OperationContext* opCtx, MultiIndexBlock indexer(opCtx, collection); + std::vector<BSONObj> indexInfoObjs; MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { - fassertStatusOK(40453, indexer.init(indexSpec)); + indexInfoObjs = fassertStatusOK(40453, indexer.init(indexSpec)); + invariant(indexInfoObjs.size() == 1); } MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "authorization index regeneration", ns.ns()); @@ -126,6 +138,8 @@ void generateSystemIndexForExistingCollection(OperationContext* opCtx, WriteUnitOfWork wunit(opCtx); indexer.commit(); + opCtx->getServiceContext()->getOpObserver()->onCreateIndex( + opCtx, ns.getSystemIndexesCollection(), indexInfoObjs[0], false /* fromMigrate */); wunit.commit(); } @@ -205,22 +219,23 @@ Status verifySystemIndexes(OperationContext* txn) { void createSystemIndexes(OperationContext* txn, Collection* collection) { invariant(collection); const NamespaceString& ns = collection->ns(); + BSONObj indexSpec; if (ns == AuthorizationManager::usersCollectionNamespace) { - auto indexSpec = fassertStatusOK( + indexSpec = fassertStatusOK( 40455, index_key_validate::validateIndexSpec( v3SystemUsersIndexSpec.toBSON(), ns, serverGlobalParams.featureCompatibility)); - - fassertStatusOK( - 40456, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec)); } else if (ns == AuthorizationManager::rolesCollectionNamespace) { - auto indexSpec = fassertStatusOK( + indexSpec = fassertStatusOK( 40457, index_key_validate::validateIndexSpec( v3SystemRolesIndexSpec.toBSON(), ns, serverGlobalParams.featureCompatibility)); - + } + if (!indexSpec.isEmpty()) { + txn->getServiceContext()->getOpObserver()->onCreateIndex( + txn, ns.getSystemIndexesCollection(), indexSpec, false /* fromMigrate */); fassertStatusOK( - 40458, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec)); + 40456, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec)); } } diff --git a/src/mongo/db/auth/authorization_manager_test.cpp b/src/mongo/db/auth/authorization_manager_test.cpp index 50f823f014a..ea39d0834a7 100644 --- a/src/mongo/db/auth/authorization_manager_test.cpp +++ b/src/mongo/db/auth/authorization_manager_test.cpp @@ -32,6 +32,7 @@ */ #include "mongo/base/status.h" #include "mongo/bson/mutable/document.h" +#include "mongo/config.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" @@ -55,6 +56,12 @@ namespace mongo { namespace { +// Construct a simple, structured X509 name equivalent to "CN=mongodb.com" +SSLX509Name buildX509Name() { + return SSLX509Name(std::vector<std::vector<SSLX509Name::Entry>>( + {{{kOID_CommonName.toString(), 19 /* Printable String */, "mongodb.com"}}})); +} + using std::vector; TEST(RoleParsingTest, BuildRoleBSON) { @@ -241,13 +248,14 @@ TEST_F(AuthorizationManagerTest, testAcquireV2User) { authzManager->releaseUser(v2cluster); } +#ifdef MONGO_CONFIG_SSL TEST_F(AuthorizationManagerTest, testLocalX509Authorization) { ServiceContextNoop serviceContext; transport::TransportLayerMock transportLayer{}; transport::SessionHandle session = transportLayer.createSession(); transportLayer.setX509PeerInfo( session, - SSLPeerInfo("CN=mongodb.com", {RoleName("read", "test"), RoleName("readWrite", "test")})); + SSLPeerInfo(buildX509Name(), {RoleName("read", "test"), RoleName("readWrite", "test")})); ServiceContext::UniqueClient client = serviceContext.makeClient("testClient", session); ServiceContext::UniqueOperationContext txn = client->makeOperationContext(); @@ -274,6 +282,7 @@ TEST_F(AuthorizationManagerTest, testLocalX509Authorization) { authzManager->releaseUser(x509User); } +#endif TEST_F(AuthorizationManagerTest, testLocalX509AuthorizationInvalidUser) { ServiceContextNoop serviceContext; @@ -281,7 +290,7 @@ TEST_F(AuthorizationManagerTest, testLocalX509AuthorizationInvalidUser) { transport::SessionHandle session = transportLayer.createSession(); transportLayer.setX509PeerInfo( session, - SSLPeerInfo("CN=mongodb.com", {RoleName("read", "test"), RoleName("write", "test")})); + SSLPeerInfo(buildX509Name(), {RoleName("read", "test"), RoleName("write", "test")})); ServiceContext::UniqueClient client = serviceContext.makeClient("testClient", session); ServiceContext::UniqueOperationContext txn = client->makeOperationContext(); diff --git a/src/mongo/db/auth/authorization_session.cpp b/src/mongo/db/auth/authorization_session.cpp index a91f350e24d..c4f219c15de 100644 --- a/src/mongo/db/auth/authorization_session.cpp +++ b/src/mongo/db/auth/authorization_session.cpp @@ -679,7 +679,12 @@ static int buildResourceSearchList(const ResourcePattern& target, // Some databases should not be matchable with ResourcePattern::forAnyNormalResource. // 'local' and 'config' are used to store special system collections, which user level // administrators should not be able to manipulate. - if (target.ns().db() != "local" && target.ns().db() != "config") { + // '$setFeatureCompatibilityVersion' is a virtual database that + // setFeatureCompatibilityVersion performs auth checks against. When this command was + // first written, there was a moratorium on creating new ActionTypes. SERVER-31983 + // introduced the ActionType after the moratorium expired. + if (target.ns().db() != "local" && target.ns().db() != "config" && + target.ns().db() != "$setFeatureCompatibilityVersion") { resourceSearchList[size++] = ResourcePattern::forAnyNormalResource(); } resourceSearchList[size++] = ResourcePattern::forDatabaseName(target.ns().db()); diff --git a/src/mongo/db/auth/authz_manager_external_state.cpp b/src/mongo/db/auth/authz_manager_external_state.cpp index ed5f0fe6bfd..0403af8e256 100644 --- a/src/mongo/db/auth/authz_manager_external_state.cpp +++ b/src/mongo/db/auth/authz_manager_external_state.cpp @@ -28,6 +28,7 @@ #include "mongo/platform/basic.h" +#include "mongo/config.h" #include "mongo/db/auth/authz_manager_external_state.h" #include "mongo/db/auth/user_name.h" #include "mongo/db/operation_context.h" @@ -42,10 +43,17 @@ AuthzManagerExternalState::~AuthzManagerExternalState() = default; bool AuthzManagerExternalState::shouldUseRolesFromConnection(OperationContext* txn, const UserName& userName) { - return txn && txn->getClient() && txn->getClient()->session() && - txn->getClient()->session()->getX509PeerInfo().subjectName == userName.getUser() && - userName.getDB() == "$external" && - !txn->getClient()->session()->getX509PeerInfo().roles.empty(); +#ifdef MONGO_CONFIG_SSL + if (!txn || !txn->getClient() || !txn->getClient()->session()) { + return false; + } + + auto sslPeerInfo = txn->getClient()->session()->getX509PeerInfo(); + return sslPeerInfo.subjectName.toString() == userName.getUser() && + userName.getDB() == "$external" && !sslPeerInfo.roles.empty(); +#else + return false; +#endif } diff --git a/src/mongo/db/catalog/apply_ops.cpp b/src/mongo/db/catalog/apply_ops.cpp index a7306940d68..552376301cb 100644 --- a/src/mongo/db/catalog/apply_ops.cpp +++ b/src/mongo/db/catalog/apply_ops.cpp @@ -180,39 +180,32 @@ Status _applyOps(OperationContext* opCtx, NamespaceString requestNss{ns}; if (nss.isSystemDotIndexes()) { - BSONObj indexSpec; - NamespaceString indexNss; - std::tie(indexSpec, indexNss) = - repl::prepForApplyOpsIndexInsert(fieldO, opObj, requestNss); - if (!indexSpec["collation"]) { - // If the index spec does not include a collation, explicitly - // specify the simple collation, so the index does not inherit the - // collection default collation. - auto indexVersion = indexSpec["v"]; - // The index version is populated by prepForApplyOpsIndexInsert(). - invariant(indexVersion); - if (indexVersion.isNumber() && - (indexVersion.numberInt() >= - static_cast<int>(IndexDescriptor::IndexVersion::kV2))) { - BSONObjBuilder bob; - bob.append("collation", CollationSpec::kSimpleSpec); - bob.appendElements(indexSpec); - indexSpec = bob.obj(); - } - } - BSONObjBuilder command; - command.append("createIndexes", indexNss.coll()); - { - BSONArrayBuilder indexes(command.subarrayStart("indexes")); - indexes.append(indexSpec); - indexes.doneFast(); + invariant(opCtx->lockState()->isW()); + + // Disable background index builds when inserting into system.indexes. + // This causes the TempRelease to fail within applyOperation_inlock(), + // leading to the background index being built in the foreground. + // We do not want a background index build because we need to validate + // the index spec and also to avoid issues resulting from any metadata + // changes before the background thread starts. + Lock::GlobalWrite nestedGlobalWriteLock(opCtx->lockState()); + + OldClientContext ctx(opCtx, nss.ns()); + status = + repl::applyOperation_inlock(opCtx, ctx.db(), opObj, alwaysUpsert); + + // applyOperation_inlock() builds the index but does not notify the + // OpObserver. Previously, applyOps relied on the createIndexes command + // to perform this function. The value used for the 'forMigrate' + // argument is consistent with create_indexes.cpp. + if (status.isOK()) { + WriteUnitOfWork wuow(opCtx); + auto opObserver = getGlobalServiceContext()->getOpObserver(); + invariant(opObserver); + auto indexSpec = fieldO.embeddedObject(); + opObserver->onCreateIndex(opCtx, nss.ns(), indexSpec, false); + wuow.commit(); } - const BSONObj commandObj = command.done(); - - DBDirectClient client(opCtx); - BSONObj infoObj; - client.runCommand(nsToDatabase(ns), commandObj, infoObj); - status = getStatusFromCommandResult(infoObj); } else { AutoGetCollection autoColl(opCtx, nss, MODE_IX); if (!autoColl.getCollection() && !nss.isSystemDotIndexes()) { diff --git a/src/mongo/db/catalog/capped_utils.cpp b/src/mongo/db/catalog/capped_utils.cpp index 71f74628f01..63b46d3c3a8 100644 --- a/src/mongo/db/catalog/capped_utils.cpp +++ b/src/mongo/db/catalog/capped_utils.cpp @@ -66,13 +66,13 @@ Status emptyCapped(OperationContext* txn, const NamespaceString& collectionName) } Database* db = autoDb.getDb(); - massert(13429, "no such database", db); + uassert(ErrorCodes::NamespaceNotFound, "no such database", db); Collection* collection = db->getCollection(collectionName); uassert(ErrorCodes::CommandNotSupportedOnView, str::stream() << "emptycapped not supported on view: " << collectionName.ns(), collection || !db->getViewCatalog()->lookup(txn, collectionName.ns())); - massert(28584, "no such collection", collection); + uassert(ErrorCodes::NamespaceNotFound, "no such collection", collection); if (collectionName.isSystem() && !collectionName.isSystemDotProfile()) { return Status(ErrorCodes::IllegalOperation, @@ -268,6 +268,7 @@ Status convertToCapped(OperationContext* txn, const NamespaceString& collectionN Status status = db->dropCollection(txn, longTmpName); if (!status.isOK()) return status; + wunit.commit(); } diff --git a/src/mongo/db/catalog/collection_catalog_entry.h b/src/mongo/db/catalog/collection_catalog_entry.h index dd6a1f506f1..38e5b12d234 100644 --- a/src/mongo/db/catalog/collection_catalog_entry.h +++ b/src/mongo/db/catalog/collection_catalog_entry.h @@ -64,6 +64,8 @@ public: virtual BSONObj getIndexSpec(OperationContext* txn, StringData idxName) const = 0; + virtual void getReadyIndexes(OperationContext* txn, std::vector<std::string>* names) const = 0; + /** * Returns true if the index identified by 'indexName' is multikey, and returns false otherwise. * diff --git a/src/mongo/db/catalog/database.cpp b/src/mongo/db/catalog/database.cpp index 23fdfb84e95..4b27ff0e48f 100644 --- a/src/mongo/db/catalog/database.cpp +++ b/src/mongo/db/catalog/database.cpp @@ -564,15 +564,26 @@ Collection* Database::createCollection(OperationContext* txn, : ic->getDefaultIdIndexSpec(featureCompatibilityVersion))); } } - - if (nss.isSystem()) { - authindex::createSystemIndexes(txn, collection); - } } getGlobalServiceContext()->getOpObserver()->onCreateCollection( txn, nss, options, fullIdIndexSpec); + // It is necessary to create the system index *after* running the onCreateCollection so that + // the oplog timestamp for the index creation is after the oplog timestamp for the + // collection creation. This way both primary and any secondaries will see the index created + // after the collection is created. + if (createIdIndex && nss.isSystem()) { + // We only want to create the indexes here on the primary. On secondaries, they will + // be created by the normal oplog application process. + auto coordinator = repl::ReplicationCoordinator::get(txn); + const bool canAcceptWrites = + (coordinator->getReplicationMode() != repl::ReplicationCoordinator::modeReplSet) || + coordinator->canAcceptWritesForDatabase(nss.db()) || nss.isSystemDotProfile(); + if (canAcceptWrites) { + authindex::createSystemIndexes(txn, collection); + } + } return collection; } diff --git a/src/mongo/db/catalog/index_create.cpp b/src/mongo/db/catalog/index_create.cpp index 17a3d981d5d..6a323605521 100644 --- a/src/mongo/db/catalog/index_create.cpp +++ b/src/mongo/db/catalog/index_create.cpp @@ -65,6 +65,8 @@ using std::endl; MONGO_FP_DECLARE(crashAfterStartingIndexBuild); MONGO_FP_DECLARE(hangAfterStartingIndexBuild); MONGO_FP_DECLARE(hangAfterStartingIndexBuildUnlocked); +MONGO_FP_DECLARE(hangBeforeIndexBuildOf); +MONGO_FP_DECLARE(hangAfterIndexBuildOf); std::atomic<std::int32_t> maxIndexBuildMemoryUsageMegabytes(500); // NOLINT @@ -282,6 +284,16 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(const std::vector<BSONObj return indexInfoObjs; } +void failPointHangDuringBuild(FailPoint* fp, StringData where, const BSONObj& doc) { + MONGO_FAIL_POINT_BLOCK(*fp, data) { + int i = doc.getIntField("i"); + if (data.getData()["i"].numberInt() == i) { + log() << "Hanging " << where << " index build of i=" << i; + MONGO_FAIL_POINT_PAUSE_WHILE_SET((*fp)); + } + } +} + Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsOut) { const char* curopMessage = _buildInBackground ? "Index Build (background)" : "Index Build"; const auto numRecords = _collection->numRecords(_txn); @@ -307,11 +319,20 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsO PlanExecutor::ExecState state; int retries = 0; // non-zero when retrying our last document. while (retries || - (PlanExecutor::ADVANCED == (state = exec->getNextSnapshotted(&objToIndex, &loc)))) { + (PlanExecutor::ADVANCED == (state = exec->getNextSnapshotted(&objToIndex, &loc))) || + MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { try { if (_allowInterruption) _txn->checkForInterrupt(); + if (!(retries || (PlanExecutor::ADVANCED == state))) { + // The only reason we are still in the loop is hangAfterStartingIndexBuild. + log() << "Hanging index build due to 'hangAfterStartingIndexBuild' failpoint"; + invariant(_allowInterruption); + sleepmillis(1000); + continue; + } + // Make sure we are working with the latest version of the document. if (objToIndex.snapshotId() != _txn->recoveryUnit()->getSnapshotId() && !_collection->findDoc(_txn, loc, &objToIndex)) { @@ -323,6 +344,8 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsO // Done before insert so we can retry document if it WCEs. progress->setTotalWhileRunning(_collection->numRecords(_txn)); + failPointHangDuringBuild(&hangBeforeIndexBuildOf, "before", objToIndex.value()); + WriteUnitOfWork wunit(_txn); Status ret = insert(objToIndex.value(), loc); if (_buildInBackground) @@ -340,6 +363,8 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsO if (_buildInBackground) exec->restoreState(); // Handles any WCEs internally. + failPointHangDuringBuild(&hangAfterIndexBuildOf, "after", objToIndex.value()); + // Go to the next document progress->hit(); n++; @@ -362,18 +387,6 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsO WorkingSetCommon::toStatusString(objToIndex.value()), state == PlanExecutor::IS_EOF); - if (MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { - // Need the index build to hang before the progress meter is marked as finished so we can - // reliably check that the index build has actually started in js tests. - while (MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { - log() << "Hanging index build due to 'hangAfterStartingIndexBuild' failpoint"; - sleepmillis(1000); - } - - // Check for interrupt to allow for killop prior to index build completion. - _txn->checkForInterrupt(); - } - if (MONGO_FAIL_POINT(hangAfterStartingIndexBuildUnlocked)) { // Unlock before hanging so replication recognizes we've completed. Locker::LockSnapshot lockInfo; diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp index 42bad714800..8f9baa02321 100644 --- a/src/mongo/db/commands.cpp +++ b/src/mongo/db/commands.cpp @@ -325,9 +325,9 @@ void Command::generateErrorResponse(OperationContext* txn, const BSONObj& metadata) { LOG(1) << "assertion while executing command '" << request.getCommandName() << "' " << "on database '" << request.getDatabase() << "' " - << "with arguments '" << command->getRedactedCopyForLogging(request.getCommandArgs()) - << "' " - << "and metadata '" << request.getMetadata() << "': " << exception.toString(); + << "with arguments '" + << redact(command->getRedactedCopyForLogging(request.getCommandArgs())) << "' " + << "and metadata '" << request.getMetadata() << "': " << redact(exception.toString()); _generateErrorResponse(txn, replyBuilder, exception, metadata); } @@ -337,7 +337,7 @@ void Command::generateErrorResponse(OperationContext* txn, const DBException& exception, const rpc::RequestInterface& request) { LOG(1) << "assertion while executing command '" << request.getCommandName() << "' " - << "on database '" << request.getDatabase() << "': " << exception.toString(); + << "on database '" << request.getDatabase() << "': " << redact(exception.toString()); _generateErrorResponse(txn, replyBuilder, exception, rpc::makeEmptyMetadata()); } @@ -345,7 +345,7 @@ void Command::generateErrorResponse(OperationContext* txn, void Command::generateErrorResponse(OperationContext* txn, rpc::ReplyBuilderInterface* replyBuilder, const DBException& exception) { - LOG(1) << "assertion while executing command: " << exception.toString(); + LOG(1) << "assertion while executing command: " << redact(exception.toString()); _generateErrorResponse(txn, replyBuilder, exception, rpc::makeEmptyMetadata()); } diff --git a/src/mongo/db/commands/authentication_commands.cpp b/src/mongo/db/commands/authentication_commands.cpp index 1eb8569b2be..68a483b24f9 100644 --- a/src/mongo/db/commands/authentication_commands.cpp +++ b/src/mongo/db/commands/authentication_commands.cpp @@ -166,14 +166,14 @@ bool CmdAuthenticate::run(OperationContext* txn, if (mechanism.empty()) { mechanism = "MONGODB-CR"; } - UserName user; - if (mechanism == "MONGODB-X509" && !cmdObj.hasField("user")) { - Client* client = txn->getClient(); - auto clientName = client->session()->getX509PeerInfo().subjectName; - user = UserName(clientName, dbname); - } else { - user = UserName(cmdObj.getStringField("user"), dbname); + + UserName user(cmdObj.getStringField("user"), dbname); +#ifdef MONGO_CONFIG_SSL + if (mechanism == "MONGODB-X509" && user.getUser().empty()) { + auto sslPeerInfo = txn->getClient()->session()->getX509PeerInfo(); + user = UserName(sslPeerInfo.subjectName.toString(), dbname); } +#endif uassert(ErrorCodes::AuthenticationFailed, "No user name provided", !user.getUser().empty()); if (Command::testCommandsEnabled && user.getDB() == "admin" && @@ -331,7 +331,7 @@ Status CmdAuthenticate::_authenticateX509(OperationContext* txn, if (!getSSLManager()->getSSLConfiguration().hasCA) { return Status(ErrorCodes::AuthenticationFailed, "Unable to verify x.509 certificate, as no CA has been provided."); - } else if (user.getUser() != clientName) { + } else if (user.getUser() != clientName.toString()) { return Status(ErrorCodes::AuthenticationFailed, "There is no x.509 client certificate matching the user."); } else { diff --git a/src/mongo/db/commands/list_indexes.cpp b/src/mongo/db/commands/list_indexes.cpp index 1a581074d55..5e8e47a7dfd 100644 --- a/src/mongo/db/commands/list_indexes.cpp +++ b/src/mongo/db/commands/list_indexes.cpp @@ -152,7 +152,7 @@ public: vector<string> indexNames; MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { indexNames.clear(); - cce->getAllIndexes(txn, &indexNames); + cce->getReadyIndexes(txn, &indexNames); } MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "listIndexes", ns.ns()); diff --git a/src/mongo/db/commands/parameters.cpp b/src/mongo/db/commands/parameters.cpp index a3c252d050a..fe8f157974b 100644 --- a/src/mongo/db/commands/parameters.cpp +++ b/src/mongo/db/commands/parameters.cpp @@ -577,7 +577,7 @@ public: << saslCommandUserDBFieldName << "$external" << saslCommandUserFieldName - << getSSLManager()->getSSLConfiguration().clientSubjectName)); + << getSSLManager()->getSSLConfiguration().clientSubjectName.toString())); #endif } else if (str == "x509" && oldMode == ServerGlobalParams::ClusterAuthMode_sendX509) { serverGlobalParams.clusterAuthMode.store(ServerGlobalParams::ClusterAuthMode_x509); diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp index 3b5f71f23ce..5e1a14663dc 100644 --- a/src/mongo/db/curop.cpp +++ b/src/mongo/db/curop.cpp @@ -459,7 +459,7 @@ string OpDebug::report(Client* client, } if (!curop.getPlanSummary().empty()) { - s << " planSummary: " << redact(curop.getPlanSummary().toString()); + s << " planSummary: " << curop.getPlanSummary().toString(); } if (!updateobj.isEmpty()) { diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index a99ce97d57c..05fc368e3b6 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -647,7 +647,7 @@ ExitCode _initAndListen(int listenPort) { logMongodStartupWarnings(storageGlobalParams, serverGlobalParams); -#if MONGO_CONFIG_SSL +#ifdef MONGO_CONFIG_SSL if (sslGlobalParams.sslAllowInvalidCertificates && ((serverGlobalParams.clusterAuthMode.load() == ServerGlobalParams::ClusterAuthMode_x509) || sequenceContains(saslGlobalParams.authenticationMechanisms, "MONGODB-X509"))) { @@ -724,6 +724,9 @@ ExitCode _initAndListen(int listenPort) { log() << redact(status); if (status.code() == ErrorCodes::AuthSchemaIncompatible) { exitCleanly(EXIT_NEED_UPGRADE); + } else if (status == ErrorCodes::NotMaster) { + // Try creating the indexes if we become master. If we do not become master, + // the master will create the indexes and we will replicate them. } else { quickExit(EXIT_FAILURE); } diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index 96eb011018d..a372d42824b 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -60,6 +60,7 @@ #include "mongo/db/s/collection_metadata.h" #include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/s/sharding_state.h" +#include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/data_protector.h" #include "mongo/db/storage/encryption_hooks.h" @@ -266,6 +267,10 @@ BSONObj Helpers::inferKeyPattern(const BSONObj& o) { return kpBuilder.obj(); } +// After completing internalQueryExecYieldIterations document deletions, the time in millis to wait +// before continuing deletions. +MONGO_EXPORT_SERVER_PARAMETER(rangeDeleterBatchDelayMS, int, 20); + long long Helpers::removeRange(OperationContext* txn, const KeyRange& range, BoundInclusion boundInclusion, @@ -323,7 +328,7 @@ long long Helpers::removeRange(OperationContext* txn, MONGO_LOG_COMPONENT(1, LogComponent::kSharding) - << "begin removal of " << min << " to " << max << " in " << ns + << "begin removal of " << redact(min) << " to " << redact(max) << " in " << ns << " with write concern: " << writeConcern.toBSON() << endl; long long numDeleted = 0; @@ -331,20 +336,25 @@ long long Helpers::removeRange(OperationContext* txn, replWaitDuration = Milliseconds::zero(); while (1) { + long long numDeletedPreviously = numDeleted; + long long iterationsBetweenSleeps = internalQueryExecYieldIterations.load(); + long long batchSize = writeConcern.shouldWaitForOtherNodes() ? 1 : iterationsBetweenSleeps; + // Scoping for write lock. { ScopedTransaction scopedXact(txn, MODE_IX); AutoGetCollection ctx(txn, NamespaceString(ns), MODE_IX, MODE_IX); Collection* collection = ctx.getCollection(); - if (!collection) + if (!collection) { break; + } IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName(txn, indexName); if (!desc) { warning(LogComponent::kSharding) << "shard key index '" << indexName << "' on '" << ns << "' was dropped"; - return -1; + break; } unique_ptr<PlanExecutor> exec( @@ -357,99 +367,153 @@ long long Helpers::removeRange(OperationContext* txn, PlanExecutor::YIELD_MANUAL, InternalPlanner::FORWARD, InternalPlanner::IXSCAN_FETCH)); - exec->setYieldPolicy(PlanExecutor::YIELD_AUTO, collection); - - RecordId rloc; - BSONObj obj; - PlanExecutor::ExecState state; - // This may yield so we cannot touch nsd after this. - state = exec->getNext(&obj, &rloc); - if (PlanExecutor::IS_EOF == state) { - break; - } - if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { - warning(LogComponent::kSharding) - << PlanExecutor::statestr(state) << " - cursor error while trying to delete " - << min << " to " << max << " in " << ns << ": " - << WorkingSetCommon::toStatusString(obj) - << ", stats: " << Explain::getWinningPlanStats(exec.get()) << endl; - break; - } - - verify(PlanExecutor::ADVANCED == state); - - if (onlyRemoveOrphanedDocs) { - // Do a final check in the write lock to make absolutely sure that our - // collection hasn't been modified in a way that invalidates our migration - // cleanup. + bool errorOccurred = false; - // We should never be able to turn off the sharding state once enabled, but - // in the future we might want to. - verify(ShardingState::get(txn)->enabled()); + while (numDeleted - numDeletedPreviously < batchSize) { - bool docIsOrphan; - - // In write lock, so will be the most up-to-date version - auto metadataNow = CollectionShardingState::get(txn, ns)->getMetadata(); - if (metadataNow) { - ShardKeyPattern kp(metadataNow->getKeyPattern()); - BSONObj key = kp.extractShardKeyFromDoc(obj); - docIsOrphan = - !metadataNow->keyBelongsToMe(key) && !metadataNow->keyIsPending(key); - } else { - docIsOrphan = false; + RecordId rloc; + BSONObj obj; + PlanExecutor::ExecState state; + // This may yield so we cannot touch nsd after this. + state = exec->getNext(&obj, &rloc); + if (PlanExecutor::IS_EOF == state) { + errorOccurred = true; + break; } - if (!docIsOrphan) { + if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { warning(LogComponent::kSharding) - << "aborting migration cleanup for chunk " << min << " to " << max - << (metadataNow ? (string) " at document " + obj.toString() : "") - << ", collection " << ns << " has changed " << endl; + << PlanExecutor::statestr(state) + << " - cursor error while trying to delete " << redact(min) << " to " + << redact(max) << " in " << ns << ": " + << redact(WorkingSetCommon::toStatusString(obj)) + << ", stats: " << Explain::getWinningPlanStats(exec.get()) << endl; + errorOccurred = true; break; } - } - MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { - WriteUnitOfWork wuow(txn); - NamespaceString nss(ns); - if (!repl::getGlobalReplicationCoordinator()->canAcceptWritesFor(nss)) { - warning() << "stepped down from primary while deleting chunk; " - << "orphaning data in " << ns << " in range [" << redact(min) << ", " - << redact(max) << ")"; - return numDeleted; + verify(PlanExecutor::ADVANCED == state); + + if (onlyRemoveOrphanedDocs) { + // Do a final check in the write lock to make absolutely sure that our + // collection hasn't been modified in a way that invalidates our migration + // cleanup. + + // We should never be able to turn off the sharding state once enabled, but + // in the future we might want to. + verify(ShardingState::get(txn)->enabled()); + + bool docIsOrphan; + + // In write lock, so will be the most up-to-date version + auto metadataNow = CollectionShardingState::get(txn, ns)->getMetadata(); + if (metadataNow) { + ShardKeyPattern kp(metadataNow->getKeyPattern()); + BSONObj key = kp.extractShardKeyFromDoc(obj); + docIsOrphan = + !metadataNow->keyBelongsToMe(key) && !metadataNow->keyIsPending(key); + } else { + docIsOrphan = false; + } + + if (!docIsOrphan) { + warning(LogComponent::kSharding) + << "aborting migration cleanup for chunk " << redact(min) << " to " + << redact(max) + << (metadataNow ? (string) " at document " + redact(obj.toString()) + : "") + << ", collection " << ns << " has changed " << endl; + // No chance of success with a new plan, so fully abort. + errorOccurred = true; + break; + } } - if (callback) - callback->goingToDelete(obj); + exec->saveState(); + + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { + WriteUnitOfWork wuow(txn); + NamespaceString nss(ns); + if (!repl::getGlobalReplicationCoordinator()->canAcceptWritesFor(nss)) { + warning() << "stepped down from primary while deleting chunk; " + << "orphaning data in " << ns << " in range [" << redact(min) + << ", " << redact(max) << ")"; + // No chance of success with a new plan, so fully abort. + errorOccurred = true; + break; + } + + if (callback) + callback->goingToDelete(obj); + + OpDebug* const nullOpDebug = nullptr; + collection->deleteDocument(txn, rloc, nullOpDebug, fromMigrate); + wuow.commit(); + } + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "delete range", ns); + + if (!exec->restoreState()) { + MONGO_LOG_COMPONENT(1, LogComponent::kSharding) + << "unable to restore cursor state while trying to delete " << redact(min) + << " to " << redact(max) << " in " << ns + << ", stats: " << Explain::getWinningPlanStats(exec.get()) + << ", replanning"; + // Try again with a new plan. + break; + } - OpDebug* const nullOpDebug = nullptr; - collection->deleteDocument(txn, rloc, nullOpDebug, fromMigrate); - wuow.commit(); + numDeleted++; } - MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "delete range", ns); - numDeleted++; - } + if (errorOccurred) { + break; + } - // TODO remove once the yielding below that references this timer has been removed - Timer secondaryThrottleTime; - - if (writeConcern.shouldWaitForOtherNodes() && numDeleted > 0) { - repl::ReplicationCoordinator::StatusAndDuration replStatus = - repl::getGlobalReplicationCoordinator()->awaitReplication( - txn, - repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(), - writeConcern); - if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit || - replStatus.status.code() == ErrorCodes::WriteConcernFailed) { - warning(LogComponent::kSharding) << "replication to secondaries for removeRange at " - "least 60 seconds behind"; - } else { - uassertStatusOK(replStatus.status); + } // End scope for write lock. + + if (numDeleted > 0) { + if (writeConcern.shouldWaitForOtherNodes()) { + repl::ReplicationCoordinator::StatusAndDuration replStatus = + repl::getGlobalReplicationCoordinator()->awaitReplication( + txn, + repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(), + writeConcern); + if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit || + replStatus.status.code() == ErrorCodes::WriteConcernFailed) { + warning(LogComponent::kSharding) + << "replication to secondaries for removeRange at " + "least 60 seconds behind"; + } else { + uassertStatusOK(replStatus.status); + } + replWaitDuration += replStatus.duration; + } + + // The `rangeDeleterBatchDelayMS` parameter is defined as a delay every + // `internalQueryExecYieldIterations` (aka `batchSize`) document deletions. + // + // In v3.6+, this applies regardless of waiting for replication (aka + // _secondaryThrottle), because in those versions the replication waits and query + // replanning also happen after each batch of `batchSize` deletions. + // + // However, here in v3.4, when _secondaryThrottle is on it's necessary to preserve the + // semantics of waiting for replication after each document deletion. But it's also + // necessary for `rangeDeleterBatchDelayMS` - which is the only wait when + // _secondaryThrottle is off - to behave as it does in other versions (despite query + // planning still occuring every document deletion in v3.4 when _secondaryThrottle is + // on). + // + // Therefore, we sleep for `rangeDeleterBatchDelayMS` here (every `batchSize` + // iterations), even if we have also waited for replication above. This approach also + // makes the RangeDeleter behavior more consistent when enabling/disabling + // _secondaryThrottle. + if (batchSize != 1 || numDeleted % iterationsBetweenSleeps == 0) { + sleepmillis(rangeDeleterBatchDelayMS.load()); } - replWaitDuration += replStatus.duration; } + + // Loop back to get a new plan and go again. } if (writeConcern.shouldWaitForOtherNodes()) @@ -457,8 +521,8 @@ long long Helpers::removeRange(OperationContext* txn, << "Helpers::removeRangeUnlocked time spent waiting for replication: " << durationCount<Milliseconds>(replWaitDuration) << "ms" << endl; - MONGO_LOG_COMPONENT(1, LogComponent::kSharding) << "end removal of " << min << " to " << max - << " in " << ns << " (took " + MONGO_LOG_COMPONENT(1, LogComponent::kSharding) << "end removal of " << redact(min) << " to " + << redact(max) << " in " << ns << " (took " << rangeRemoveTimer.millis() << "ms)" << endl; return numDeleted; diff --git a/src/mongo/db/exec/cached_plan.cpp b/src/mongo/db/exec/cached_plan.cpp index f1867423b78..8c729927e71 100644 --- a/src/mongo/db/exec/cached_plan.cpp +++ b/src/mongo/db/exec/cached_plan.cpp @@ -140,7 +140,7 @@ Status CachedPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { LOG(1) << "Execution of cached plan failed, falling back to replan." << " query: " << redact(_canonicalQuery->toStringShort()) - << " planSummary: " << redact(Explain::getPlanSummary(child().get())) + << " planSummary: " << Explain::getPlanSummary(child().get()) << " status: " << redact(statusObj); const bool shouldCache = false; @@ -151,7 +151,7 @@ Status CachedPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { LOG(1) << "Execution of cached plan failed: PlanStage died" << ", query: " << redact(_canonicalQuery->toStringShort()) - << " planSummary: " << redact(Explain::getPlanSummary(child().get())) + << " planSummary: " << Explain::getPlanSummary(child().get()) << " status: " << redact(statusObj); return WorkingSetCommon::getMemberObjectStatus(statusObj); @@ -166,7 +166,7 @@ Status CachedPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { << " works, but was originally cached with only " << _decisionWorks << " works. Evicting cache entry and replanning query: " << redact(_canonicalQuery->toStringShort()) - << " plan summary before replan: " << redact(Explain::getPlanSummary(child().get())); + << " plan summary before replan: " << Explain::getPlanSummary(child().get()); const bool shouldCache = true; return replan(yieldPolicy, shouldCache); @@ -241,7 +241,7 @@ Status CachedPlanStage::replan(PlanYieldPolicy* yieldPolicy, bool shouldCache) { LOG(1) << "Replanning of query resulted in single query solution, which will not be cached. " << redact(_canonicalQuery->toStringShort()) - << " plan summary after replan: " << redact(Explain::getPlanSummary(child().get())) + << " plan summary after replan: " << Explain::getPlanSummary(child().get()) << " previous cache entry evicted: " << (shouldCache ? "yes" : "no"); return Status::OK(); } @@ -274,7 +274,7 @@ Status CachedPlanStage::replan(PlanYieldPolicy* yieldPolicy, bool shouldCache) { } LOG(1) << "Replanning " << redact(_canonicalQuery->toStringShort()) - << " resulted in plan with summary: " << redact(Explain::getPlanSummary(child().get())) + << " resulted in plan with summary: " << Explain::getPlanSummary(child().get()) << ", which " << (shouldCache ? "has" : "has not") << " been written to the cache"; return Status::OK(); } diff --git a/src/mongo/db/exec/multi_plan.cpp b/src/mongo/db/exec/multi_plan.cpp index 28c14dbcba3..10b36d49b8d 100644 --- a/src/mongo/db/exec/multi_plan.cpp +++ b/src/mongo/db/exec/multi_plan.cpp @@ -239,7 +239,7 @@ Status MultiPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { const auto& bestSolution = bestCandidate.solution; LOG(5) << "Winning solution:\n" << redact(bestSolution->toString()); - LOG(2) << "Winning plan: " << redact(Explain::getPlanSummary(bestCandidate.root)); + LOG(2) << "Winning plan: " << Explain::getPlanSummary(bestCandidate.root); _backupPlanIdx = kNoSuchPlan; if (bestSolution->hasBlockingStage && (0 == alreadyProduced.size())) { @@ -276,10 +276,10 @@ Status MultiPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { LOG(1) << "Winning plan tied with runner-up. Not caching." << " ns: " << _collection->ns() << " " << redact(_query->toStringShort()) - << " winner score: " << ranking->scores[0] << " winner summary: " - << redact(Explain::getPlanSummary(_candidates[winnerIdx].root)) + << " winner score: " << ranking->scores[0] + << " winner summary: " << Explain::getPlanSummary(_candidates[winnerIdx].root) << " runner-up score: " << ranking->scores[1] << " runner-up summary: " - << redact(Explain::getPlanSummary(_candidates[runnerUpIdx].root)); + << Explain::getPlanSummary(_candidates[runnerUpIdx].root); } if (alreadyProduced.empty()) { @@ -290,8 +290,8 @@ Status MultiPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { size_t winnerIdx = ranking->candidateOrder[0]; LOG(1) << "Winning plan had zero results. Not caching." << " ns: " << _collection->ns() << " " << redact(_query->toStringShort()) - << " winner score: " << ranking->scores[0] << " winner summary: " - << redact(Explain::getPlanSummary(_candidates[winnerIdx].root)); + << " winner score: " << ranking->scores[0] + << " winner summary: " << Explain::getPlanSummary(_candidates[winnerIdx].root); } } diff --git a/src/mongo/db/ftdc/compressor.cpp b/src/mongo/db/ftdc/compressor.cpp index ecf9c7ece6c..5be24ceb1f7 100644 --- a/src/mongo/db/ftdc/compressor.cpp +++ b/src/mongo/db/ftdc/compressor.cpp @@ -45,7 +45,12 @@ using std::swap; StatusWith<boost::optional<std::tuple<ConstDataRange, FTDCCompressor::CompressorState, Date_t>>> FTDCCompressor::addSample(const BSONObj& sample, Date_t date) { if (_referenceDoc.isEmpty()) { - FTDCBSONUtil::extractMetricsFromDocument(sample, sample, &_metrics); + auto swMatchesReference = + FTDCBSONUtil::extractMetricsFromDocument(sample, sample, &_metrics); + if (!swMatchesReference.isOK()) { + return swMatchesReference.getStatus(); + } + _reset(sample, date); return {boost::none}; } diff --git a/src/mongo/db/ftdc/compressor_test.cpp b/src/mongo/db/ftdc/compressor_test.cpp index 0c01bd58040..10914eba102 100644 --- a/src/mongo/db/ftdc/compressor_test.cpp +++ b/src/mongo/db/ftdc/compressor_test.cpp @@ -48,17 +48,17 @@ namespace mongo { ASSERT_TRUE(st.isOK()); \ ASSERT_FALSE(st.getValue().is_initialized()); -#define ASSERT_SCHEMA_CHANGED(st) \ - ASSERT_TRUE(st.isOK()); \ - ASSERT_TRUE(std::get<1>(st.getValue().get()) == \ - FTDCCompressor::CompressorState::kSchemaChanged); \ - ASSERT_TRUE(st.getValue().is_initialized()); - -#define ASSERT_FULL(st) \ - ASSERT_TRUE(st.isOK()); \ - ASSERT_TRUE(std::get<1>(st.getValue().get()) == \ - FTDCCompressor::CompressorState::kCompressorFull); \ - ASSERT_TRUE(st.getValue().is_initialized()); +#define ASSERT_SCHEMA_CHANGED(st) \ + ASSERT_TRUE(st.isOK()); \ + ASSERT_TRUE(st.getValue().is_initialized()); \ + ASSERT_TRUE(std::get<1>(st.getValue().get()) == \ + FTDCCompressor::CompressorState::kSchemaChanged); + +#define ASSERT_FULL(st) \ + ASSERT_TRUE(st.isOK()); \ + ASSERT_TRUE(st.getValue().is_initialized()); \ + ASSERT_TRUE(std::get<1>(st.getValue().get()) == \ + FTDCCompressor::CompressorState::kCompressorFull); // Sanity check TEST(FTDCCompressor, TestBasic) { @@ -125,7 +125,8 @@ TEST(FTDCCompressor, TestStrings) { */ class TestTie { public: - TestTie() : _compressor(&_config) {} + TestTie(FTDCValidationMode mode = FTDCValidationMode::kStrict) + : _compressor(&_config), _mode(mode) {} ~TestTie() { validate(boost::none); @@ -169,7 +170,7 @@ public: list = sw.getValue(); } - ValidateDocumentList(list, _docs); + ValidateDocumentList(list, _docs, _mode); } private: @@ -177,6 +178,7 @@ private: FTDCConfig _config; FTDCCompressor _compressor; FTDCDecompressor _decompressor; + FTDCValidationMode _mode; }; // Test various schema changes @@ -340,6 +342,114 @@ TEST(FTDCCompressor, TestSchemaChanges) { ASSERT_SCHEMA_CHANGED(st); } +// Test various schema changes with strings +TEST(FTDCCompressorTest, TestStringSchemaChanges) { + TestTie c(FTDCValidationMode::kWeak); + + auto st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 42)); + ASSERT_HAS_SPACE(st); + st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 45)); + ASSERT_HAS_SPACE(st); + + // Add string field + st = c.addSample(BSON("str1" + << "joe" + << "str2" + << "smith" + << "int1" + << 47)); + ASSERT_HAS_SPACE(st); + + // Reset schema by renaming a int field + st = c.addSample(BSON("str1" + << "joe" + << "str2" + << "smith" + << "int2" + << 48)); + ASSERT_SCHEMA_CHANGED(st); + + // Remove string field + st = c.addSample(BSON("str1" + << "joe" + << "int2" + << 49)); + ASSERT_HAS_SPACE(st); + + + // Add string field as last element + st = c.addSample(BSON("str1" + << "joe" + << "int2" + << 50 + << "str3" + << "bar")); + ASSERT_HAS_SPACE(st); + + // Reset schema by renaming a int field + st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 51 + << "str3" + << "bar")); + ASSERT_SCHEMA_CHANGED(st); + + // Remove string field as last element + st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 52)); + ASSERT_HAS_SPACE(st); + + + // Add 2 string fields + st = c.addSample(BSON("str1" + << "joe" + << "str2" + << "smith" + << "str3" + << "foo" + << "int1" + << 53)); + ASSERT_HAS_SPACE(st); + + // Reset schema by renaming a int field + st = c.addSample(BSON("str1" + << "joe" + << "str2" + << "smith" + << "str3" + << "foo" + << "int2" + << 54)); + ASSERT_SCHEMA_CHANGED(st); + + // Remove 2 string fields + st = c.addSample(BSON("str1" + << "joe" + << "int2" + << 55)); + ASSERT_HAS_SPACE(st); + + // Change string to number + st = c.addSample(BSON("str1" << 12 << "int1" << 56)); + ASSERT_SCHEMA_CHANGED(st); + + // Change number to string + st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 67)); + ASSERT_SCHEMA_CHANGED(st); +} + // Ensure changing between the various number formats is considered compatible TEST(FTDCCompressor, TestNumbersCompat) { TestTie c; diff --git a/src/mongo/db/ftdc/controller_test.cpp b/src/mongo/db/ftdc/controller_test.cpp index c75a3fe44d6..94b028845d4 100644 --- a/src/mongo/db/ftdc/controller_test.cpp +++ b/src/mongo/db/ftdc/controller_test.cpp @@ -202,7 +202,7 @@ TEST(FTDCControllerTest, TestFull) { auto alog = files[0]; - ValidateDocumentList(alog, allDocs); + ValidateDocumentList(alog, allDocs, FTDCValidationMode::kStrict); } // Test we can start and stop the controller in quick succession, make sure it succeeds without @@ -274,7 +274,7 @@ TEST(FTDCControllerTest, TestStartAsDisabled) { auto alog = files[0]; - ValidateDocumentList(alog, allDocs); + ValidateDocumentList(alog, allDocs, FTDCValidationMode::kStrict); } } // namespace mongo diff --git a/src/mongo/db/ftdc/file_manager_test.cpp b/src/mongo/db/ftdc/file_manager_test.cpp index 6c2e5c220a6..8ac115df584 100644 --- a/src/mongo/db/ftdc/file_manager_test.cpp +++ b/src/mongo/db/ftdc/file_manager_test.cpp @@ -378,11 +378,11 @@ TEST(FTDCFileManagerTest, TestNormalCrashInterim) { // Validate old file std::vector<BSONObj> docs1 = {mdoc1, sdoc1, sdoc1}; - ValidateDocumentList(files[0], docs1); + ValidateDocumentList(files[0], docs1, FTDCValidationMode::kStrict); // Validate new file std::vector<BSONObj> docs2 = {sdoc2, sdoc2, sdoc2, sdoc2}; - ValidateDocumentList(files[1], docs2); + ValidateDocumentList(files[1], docs2, FTDCValidationMode::kStrict); } } // namespace mongo diff --git a/src/mongo/db/ftdc/file_writer_test.cpp b/src/mongo/db/ftdc/file_writer_test.cpp index 138d7c850f6..a182d52fdfe 100644 --- a/src/mongo/db/ftdc/file_writer_test.cpp +++ b/src/mongo/db/ftdc/file_writer_test.cpp @@ -196,7 +196,7 @@ private: _writer.close(); - ValidateDocumentList(_path, _docs); + ValidateDocumentList(_path, _docs, FTDCValidationMode::kStrict); } private: diff --git a/src/mongo/db/ftdc/ftdc_test.cpp b/src/mongo/db/ftdc/ftdc_test.cpp index 7e4020b3979..49be73bb8cc 100644 --- a/src/mongo/db/ftdc/ftdc_test.cpp +++ b/src/mongo/db/ftdc/ftdc_test.cpp @@ -48,7 +48,23 @@ namespace mongo { -void ValidateDocumentList(const boost::filesystem::path& p, const std::vector<BSONObj>& docs) { +namespace { + +BSONObj filteredFTDCCopy(const BSONObj& obj) { + BSONObjBuilder builder; + for (const auto& f : obj) { + if (FTDCBSONUtil::isFTDCType(f.type())) { + builder.append(f); + } + } + return builder.obj(); +} +} // namespace + + +void ValidateDocumentList(const boost::filesystem::path& p, + const std::vector<BSONObj>& docs, + FTDCValidationMode mode) { FTDCFileReader reader; ASSERT_OK(reader.open(p)); @@ -62,20 +78,32 @@ void ValidateDocumentList(const boost::filesystem::path& p, const std::vector<BS ASSERT_OK(sw); - ValidateDocumentList(list, docs); + ValidateDocumentList(list, docs, mode); } -void ValidateDocumentList(const std::vector<BSONObj>& docs1, const std::vector<BSONObj>& docs2) { +void ValidateDocumentList(const std::vector<BSONObj>& docs1, + const std::vector<BSONObj>& docs2, + FTDCValidationMode mode) { ASSERT_EQUALS(docs1.size(), docs2.size()); auto ai = docs1.begin(); auto bi = docs2.begin(); while (ai != docs1.end() && bi != docs2.end()) { - if (SimpleBSONObjComparator::kInstance.evaluate(*ai != *bi)) { - std::cout << *ai << " vs " << *bi << std::endl; - ASSERT_BSONOBJ_EQ(*ai, *bi); + if (mode == FTDCValidationMode::kStrict) { + if (SimpleBSONObjComparator::kInstance.evaluate(*ai != *bi)) { + std::cout << *ai << " vs " << *bi << std::endl; + ASSERT_BSONOBJ_EQ(*ai, *bi); + } + } else { + BSONObj left = filteredFTDCCopy(*ai); + BSONObj right = filteredFTDCCopy(*bi); + if (SimpleBSONObjComparator::kInstance.evaluate(left != right)) { + std::cout << left << " vs " << right << std::endl; + ASSERT_BSONOBJ_EQ(left, right); + } } + ++ai; ++bi; } diff --git a/src/mongo/db/ftdc/ftdc_test.h b/src/mongo/db/ftdc/ftdc_test.h index afbca103b4f..d3b7b52d647 100644 --- a/src/mongo/db/ftdc/ftdc_test.h +++ b/src/mongo/db/ftdc/ftdc_test.h @@ -34,18 +34,38 @@ namespace mongo { /** + * Validation mode for tests, strict by default + */ +enum class FTDCValidationMode { + /** + * Compare BSONObjs exactly. + */ + kStrict, + + /** + * Compare BSONObjs by only comparing types FTDC compares about. FTDC ignores somes changes in + * the shapes of documents and therefore no longer reconstructs the shapes of documents exactly. + */ + kWeak, +}; + +/** * Validate the documents in a file match the specified vector. * * Unit Test ASSERTs if there is mismatch. */ -void ValidateDocumentList(const boost::filesystem::path& p, const std::vector<BSONObj>& docs); +void ValidateDocumentList(const boost::filesystem::path& p, + const std::vector<BSONObj>& docs, + FTDCValidationMode mode); /** * Validate that two lists of documents are equal. * * Unit Test ASSERTs if there is mismatch. */ -void ValidateDocumentList(const std::vector<BSONObj>& docs1, const std::vector<BSONObj>& docs2); +void ValidateDocumentList(const std::vector<BSONObj>& docs1, + const std::vector<BSONObj>& docs2, + FTDCValidationMode mode); /** * Delete a file if it exists. diff --git a/src/mongo/db/ftdc/util.cpp b/src/mongo/db/ftdc/util.cpp index 243ba90b666..ecfc4db5f37 100644 --- a/src/mongo/db/ftdc/util.cpp +++ b/src/mongo/db/ftdc/util.cpp @@ -123,6 +123,47 @@ namespace FTDCBSONUtil { namespace { +/** + * Iterate a BSONObj but only return fields that have types that FTDC cares about. + */ +class FTDCBSONObjIterator { +public: + FTDCBSONObjIterator(const BSONObj& obj) : _iterator(obj) { + advance(); + } + + bool more() { + return !_current.eoo(); + } + + BSONElement next() { + auto ret = _current; + advance(); + return ret; + } + +private: + /** + * Find the next element that is a valid FTDC type. + */ + void advance() { + _current = BSONElement(); + + while (_iterator.more()) { + + auto elem = _iterator.next(); + if (isFTDCType(elem.type())) { + _current = elem; + break; + } + } + } + +private: + BSONObjIterator _iterator; + BSONElement _current; +}; + StatusWith<bool> extractMetricsFromDocument(const BSONObj& referenceDoc, const BSONObj& currentDoc, std::vector<std::uint64_t>* metrics, @@ -132,15 +173,14 @@ StatusWith<bool> extractMetricsFromDocument(const BSONObj& referenceDoc, return {ErrorCodes::BadValue, "Recursion limit reached."}; } - BSONObjIterator itCurrent(currentDoc); - BSONObjIterator itReference(referenceDoc); + FTDCBSONObjIterator itCurrent(currentDoc); + FTDCBSONObjIterator itReference(referenceDoc); while (itCurrent.more()) { // Schema mismatch if current document is longer than reference document if (matches && !itReference.more()) { LOG(4) << "full-time diagnostic data capture schema change: currrent document is " - "longer than " - "reference document"; + "longer than reference document"; matches = false; } @@ -230,6 +270,24 @@ StatusWith<bool> extractMetricsFromDocument(const BSONObj& referenceDoc, } // namespace +bool isFTDCType(BSONType type) { + switch (type) { + case NumberDouble: + case NumberInt: + case NumberLong: + case NumberDecimal: + case Bool: + case Date: + case bsonTimestamp: + case Object: + case Array: + return true; + + default: + return false; + } +} + StatusWith<bool> extractMetricsFromDocument(const BSONObj& referenceDoc, const BSONObj& currentDoc, std::vector<std::uint64_t>* metrics) { diff --git a/src/mongo/db/ftdc/util.h b/src/mongo/db/ftdc/util.h index 4816c534f96..04d453139ce 100644 --- a/src/mongo/db/ftdc/util.h +++ b/src/mongo/db/ftdc/util.h @@ -168,6 +168,11 @@ StatusWith<BSONObj> getBSONDocumentFromMetadataDoc(const BSONObj& obj); */ StatusWith<std::vector<BSONObj>> getMetricsFromMetricDoc(const BSONObj& obj, FTDCDecompressor* decompressor); + +/** + * Is this a type that FTDC find's interesting? I.e. is this a numeric or container type? + */ +bool isFTDCType(BSONType type); } // namespace FTDCBSONUtil diff --git a/src/mongo/db/initialize_server_global_state.cpp b/src/mongo/db/initialize_server_global_state.cpp index f552a238109..4993fbf7bd8 100644 --- a/src/mongo/db/initialize_server_global_state.cpp +++ b/src/mongo/db/initialize_server_global_state.cpp @@ -373,7 +373,7 @@ bool initializeServerGlobalState() { << saslCommandUserDBFieldName << "$external" << saslCommandUserFieldName - << getSSLManager()->getSSLConfiguration().clientSubjectName)); + << getSSLManager()->getSSLConfiguration().clientSubjectName.toString())); } #endif return true; diff --git a/src/mongo/db/ops/write_ops_exec.cpp b/src/mongo/db/ops/write_ops_exec.cpp index 6ea4d88293a..ad307b508bf 100644 --- a/src/mongo/db/ops/write_ops_exec.cpp +++ b/src/mongo/db/ops/write_ops_exec.cpp @@ -52,6 +52,7 @@ #include "mongo/db/ops/parsed_update.h" #include "mongo/db/ops/update_lifecycle_impl.h" #include "mongo/db/ops/update_request.h" +#include "mongo/db/ops/write_ops.h" #include "mongo/db/ops/write_ops_exec.h" #include "mongo/db/query/get_executor.h" #include "mongo/db/query/plan_summary_stats.h" @@ -301,12 +302,13 @@ static WriteResult performCreateIndexes(OperationContext* txn, const InsertOp& w static void insertDocuments(OperationContext* txn, Collection* collection, std::vector<BSONObj>::const_iterator begin, - std::vector<BSONObj>::const_iterator end) { + std::vector<BSONObj>::const_iterator end, + bool fromMigrate) { // Intentionally not using a WRITE_CONFLICT_RETRY_LOOP. That is handled by the caller so it can // react to oversized batches. WriteUnitOfWork wuow(txn); uassertStatusOK(collection->insertDocuments( - txn, begin, end, &CurOp::get(txn)->debug(), /*enforceQuota*/ true)); + txn, begin, end, &CurOp::get(txn)->debug(), /*enforceQuota*/ true, fromMigrate)); wuow.commit(); } @@ -317,7 +319,8 @@ static bool insertBatchAndHandleErrors(OperationContext* txn, const InsertOp& wholeOp, const std::vector<BSONObj>& batch, LastOpFixer* lastOpFixer, - WriteResult* out) { + WriteResult* out, + bool fromMigrate) { if (batch.empty()) return true; @@ -350,7 +353,8 @@ static bool insertBatchAndHandleErrors(OperationContext* txn, // First try doing it all together. If all goes well, this is all we need to do. // See Collection::_insertDocuments for why we do all capped inserts one-at-a-time. lastOpFixer->startingOp(); - insertDocuments(txn, collection->getCollection(), batch.begin(), batch.end()); + insertDocuments( + txn, collection->getCollection(), batch.begin(), batch.end(), fromMigrate); lastOpFixer->finishedOpSuccessfully(); globalOpCounters.gotInserts(batch.size()); std::fill_n( @@ -374,7 +378,7 @@ static bool insertBatchAndHandleErrors(OperationContext* txn, if (!collection) acquireCollection(); lastOpFixer->startingOp(); - insertDocuments(txn, collection->getCollection(), it, it + 1); + insertDocuments(txn, collection->getCollection(), it, it + 1, fromMigrate); lastOpFixer->finishedOpSuccessfully(); out->results.emplace_back(WriteResult::SingleResult{1}); curOp.debug().ninserted++; @@ -396,7 +400,7 @@ static bool insertBatchAndHandleErrors(OperationContext* txn, return true; } -WriteResult performInserts(OperationContext* txn, const InsertOp& wholeOp) { +WriteResult performInserts(OperationContext* txn, const InsertOp& wholeOp, bool fromMigrate) { invariant(!txn->lockState()->inAWriteUnitOfWork()); // Does own retries. auto& curOp = *CurOp::get(txn); ON_BLOCK_EXIT([&] { @@ -453,7 +457,8 @@ WriteResult performInserts(OperationContext* txn, const InsertOp& wholeOp) { continue; // Add more to batch before inserting. } - bool canContinue = insertBatchAndHandleErrors(txn, wholeOp, batch, &lastOpFixer, &out); + bool canContinue = + insertBatchAndHandleErrors(txn, wholeOp, batch, &lastOpFixer, &out, fromMigrate); batch.clear(); // We won't need the current batch any more. bytesInBatch = 0; diff --git a/src/mongo/db/ops/write_ops_exec.h b/src/mongo/db/ops/write_ops_exec.h index 49d3d2e0cf1..362bf5a4782 100644 --- a/src/mongo/db/ops/write_ops_exec.h +++ b/src/mongo/db/ops/write_ops_exec.h @@ -75,8 +75,10 @@ struct WriteResult { * LastError is updated for failures of individual writes, but not for batch errors reported by an * exception being thrown from these functions. Callers are responsible for managing LastError in * that case. This should generally be combined with LastError handling from parse failures. + * + * 'fromMigrate' indicates whether the operation was induced by a chunk migration */ -WriteResult performInserts(OperationContext* txn, const InsertOp& op); +WriteResult performInserts(OperationContext* txn, const InsertOp& op, bool fromMigrate = false); WriteResult performUpdates(OperationContext* txn, const UpdateOp& op); WriteResult performDeletes(OperationContext* txn, const DeleteOp& op); diff --git a/src/mongo/db/pipeline/value_internal.h b/src/mongo/db/pipeline/value_internal.h index 51d76e6cf73..79a378c2a6a 100644 --- a/src/mongo/db/pipeline/value_internal.h +++ b/src/mongo/db/pipeline/value_internal.h @@ -76,7 +76,6 @@ public: const Decimal128 decimalValue; }; -#pragma pack(1) class ValueStorage { public: // Note: it is important the memory is zeroed out (by calling zero()) at the start of every @@ -311,6 +310,7 @@ public: // This data is public because this should only be used by Value which would be a friend union { +#pragma pack(1) struct { // byte 1 signed char type; @@ -354,11 +354,16 @@ public: }; }; }; +#pragma pack() // covers the whole ValueStorage long long i64[2]; + + // Forces the ValueStorage type to have at least pointer alignment. Can't use alignas on the + // type since that causes issues on MSVC. + void* forcePointerAlignment; }; }; MONGO_STATIC_ASSERT(sizeof(ValueStorage) == 16); -#pragma pack() +MONGO_STATIC_ASSERT(alignof(ValueStorage) >= alignof(void*)); } diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index d6c0cc2bd8e..93d529704c9 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -407,7 +407,7 @@ StatusWith<PrepareExecutionResult> prepareExecution(OperationContext* opCtx, root.reset(rawRoot); LOG(2) << "Using fast count: " << redact(canonicalQuery->toStringShort()) - << ", planSummary: " << redact(Explain::getPlanSummary(root.get())); + << ", planSummary: " << Explain::getPlanSummary(root.get()); querySolution.reset(solutions[i]); return PrepareExecutionResult( @@ -425,7 +425,7 @@ StatusWith<PrepareExecutionResult> prepareExecution(OperationContext* opCtx, LOG(2) << "Only one plan is available; it will be run but will not be cached. " << redact(canonicalQuery->toStringShort()) - << ", planSummary: " << redact(Explain::getPlanSummary(root.get())); + << ", planSummary: " << Explain::getPlanSummary(root.get()); querySolution.reset(solutions[0]); return PrepareExecutionResult( @@ -1536,7 +1536,7 @@ StatusWith<unique_ptr<PlanExecutor>> getExecutorDistinct(OperationContext* txn, unique_ptr<PlanStage> root(rawRoot); LOG(2) << "Using fast distinct: " << redact(cq->toStringShort()) - << ", planSummary: " << redact(Explain::getPlanSummary(root.get())); + << ", planSummary: " << Explain::getPlanSummary(root.get()); return PlanExecutor::make(txn, std::move(ws), @@ -1572,7 +1572,7 @@ StatusWith<unique_ptr<PlanExecutor>> getExecutorDistinct(OperationContext* txn, unique_ptr<PlanStage> root(rawRoot); LOG(2) << "Using fast distinct: " << redact(cq->toStringShort()) - << ", planSummary: " << redact(Explain::getPlanSummary(root.get())); + << ", planSummary: " << Explain::getPlanSummary(root.get()); return PlanExecutor::make(txn, std::move(ws), diff --git a/src/mongo/db/query/index_bounds.cpp b/src/mongo/db/query/index_bounds.cpp index ef8eeb6d8b8..b8b7e4bce74 100644 --- a/src/mongo/db/query/index_bounds.cpp +++ b/src/mongo/db/query/index_bounds.cpp @@ -32,6 +32,7 @@ #include <tuple> #include <utility> +#include "mongo/base/simple_string_data_comparator.h" #include "mongo/bson/simple_bsonobj_comparator.h" namespace mongo { @@ -163,6 +164,21 @@ BoundInclusion IndexBounds::makeBoundInclusionFromBoundBools(bool startKeyInclus } } +BoundInclusion IndexBounds::reverseBoundInclusion(BoundInclusion b) { + switch (b) { + case BoundInclusion::kIncludeStartKeyOnly: + return BoundInclusion::kIncludeEndKeyOnly; + case BoundInclusion::kIncludeEndKeyOnly: + return BoundInclusion::kIncludeStartKeyOnly; + case BoundInclusion::kIncludeBothStartAndEndKeys: + case BoundInclusion::kExcludeBothStartAndEndKeys: + // These are both symmetric. + return b; + default: + MONGO_UNREACHABLE; + } +} + bool OrderedIntervalList::operator==(const OrderedIntervalList& other) const { if (this->name != other.name) { @@ -186,6 +202,38 @@ bool OrderedIntervalList::operator!=(const OrderedIntervalList& other) const { return !(*this == other); } +void OrderedIntervalList::reverse() { + for (size_t i = 0; i < (intervals.size() + 1) / 2; i++) { + const size_t otherIdx = intervals.size() - i - 1; + intervals[i].reverse(); + if (i != otherIdx) { + intervals[otherIdx].reverse(); + std::swap(intervals[i], intervals[otherIdx]); + } + } +} + +OrderedIntervalList OrderedIntervalList::reverseClone() const { + OrderedIntervalList clone(name); + + for (auto it = intervals.rbegin(); it != intervals.rend(); ++it) { + clone.intervals.push_back(it->reverseClone()); + } + + return clone; +} + +Interval::Direction OrderedIntervalList::computeDirection() const { + for (auto&& iv : intervals) { + const auto dir = iv.getDirection(); + if (dir != Interval::Direction::kDirectionNone) { + return dir; + } + } + + return Interval::Direction::kDirectionNone; +} + // static void OrderedIntervalList::complement() { BSONObjBuilder minBob; @@ -299,6 +347,40 @@ BSONObj IndexBounds::toBSON() const { return bob.obj(); } +IndexBounds IndexBounds::forwardize() const { + IndexBounds newBounds; + newBounds.isSimpleRange = isSimpleRange; + + if (isSimpleRange) { + const int cmpRes = startKey.woCompare(endKey); + if (cmpRes <= 0) { + newBounds.startKey = startKey; + newBounds.endKey = endKey; + newBounds.boundInclusion = boundInclusion; + } else { + // Swap start and end key. + newBounds.endKey = startKey; + newBounds.startKey = endKey; + newBounds.boundInclusion = IndexBounds::reverseBoundInclusion(boundInclusion); + } + + return newBounds; + } + + newBounds.fields.reserve(fields.size()); + std::transform(fields.begin(), + fields.end(), + std::back_inserter(newBounds.fields), + [](const OrderedIntervalList& oil) -> OrderedIntervalList { + if (oil.computeDirection() == Interval::Direction::kDirectionDescending) { + return oil.reverseClone(); + } + return oil; + }); + + return newBounds; +} + // // Validity checking for bounds // diff --git a/src/mongo/db/query/index_bounds.h b/src/mongo/db/query/index_bounds.h index 2dfcc122b26..4b04f374b16 100644 --- a/src/mongo/db/query/index_bounds.h +++ b/src/mongo/db/query/index_bounds.h @@ -74,6 +74,15 @@ struct OrderedIntervalList { bool operator==(const OrderedIntervalList& other) const; bool operator!=(const OrderedIntervalList& other) const; + + void reverse(); + + /** + * Return a clone of this OIL, that is reversed. + */ + OrderedIntervalList reverseClone() const; + + Interval::Direction computeDirection() const; }; /** @@ -126,6 +135,11 @@ struct IndexBounds { static BoundInclusion makeBoundInclusionFromBoundBools(bool startKeyInclusive, bool endKeyInclusive); + /** + * Reverse the BoundInclusion. + */ + static BoundInclusion reverseBoundInclusion(BoundInclusion b); + /** * BSON format for explain. The format is an array of strings for each field. @@ -137,6 +151,12 @@ struct IndexBounds { */ BSONObj toBSON() const; + /** + * Return a copy of the index bounds, but with each of the OILs going in the ascending + * direction. + */ + IndexBounds forwardize() const; + // TODO: we use this for max/min scan. Consider migrating that. bool isSimpleRange; BSONObj startKey; diff --git a/src/mongo/db/query/index_bounds_builder.cpp b/src/mongo/db/query/index_bounds_builder.cpp index eeb1c2f1114..a326b960192 100644 --- a/src/mongo/db/query/index_bounds_builder.cpp +++ b/src/mongo/db/query/index_bounds_builder.cpp @@ -54,6 +54,23 @@ namespace mongo { namespace { +// Helper for checking that an OIL "appears" to be ascending given one interval. +void assertOILIsAscendingLocally(const vector<Interval>& intervals, size_t idx) { + // Each individual interval being examined should be ascending or none. + const auto dir = intervals[idx].getDirection(); + + // Should be either ascending, or have no direction (be a point/null/empty interval). + invariant(dir == Interval::Direction::kDirectionAscending || + dir == Interval::Direction::kDirectionNone); + + // The previous OIL's end value should be <= the next OIL's start value. + if (idx > 0) { + // Pass 'false' to avoid comparing the field names. + const int res = intervals[idx - 1].end.woCompare(intervals[idx].start, false); + invariant(res <= 0); + } +} + // Tightness rules are shared for $lt, $lte, $gt, $gte. IndexBoundsBuilder::BoundsTightness getInequalityPredicateTightness(const BSONElement& dataElt, const IndexEntry& index) { @@ -624,52 +641,57 @@ Interval IndexBoundsBuilder::makeRangeInterval(const BSONObj& obj, BoundInclusio } // static -void IndexBoundsBuilder::intersectize(const OrderedIntervalList& arg, OrderedIntervalList* oilOut) { - verify(arg.name == oilOut->name); +void IndexBoundsBuilder::intersectize(const OrderedIntervalList& oilA, OrderedIntervalList* oilB) { + invariant(oilB); + invariant(oilA.name == oilB->name); - size_t argidx = 0; - const vector<Interval>& argiv = arg.intervals; + size_t oilAIdx = 0; + const vector<Interval>& oilAIntervals = oilA.intervals; - size_t ividx = 0; - vector<Interval>& iv = oilOut->intervals; + size_t oilBIdx = 0; + vector<Interval>& oilBIntervals = oilB->intervals; vector<Interval> result; - while (argidx < argiv.size() && ividx < iv.size()) { - Interval::IntervalComparison cmp = argiv[argidx].compare(iv[ividx]); + while (oilAIdx < oilAIntervals.size() && oilBIdx < oilBIntervals.size()) { + if (kDebugBuild) { + // Ensure that both OILs are ascending. + assertOILIsAscendingLocally(oilAIntervals, oilAIdx); + assertOILIsAscendingLocally(oilBIntervals, oilBIdx); + } + Interval::IntervalComparison cmp = oilAIntervals[oilAIdx].compare(oilBIntervals[oilBIdx]); verify(Interval::INTERVAL_UNKNOWN != cmp); if (cmp == Interval::INTERVAL_PRECEDES || cmp == Interval::INTERVAL_PRECEDES_COULD_UNION) { - // argiv is before iv. move argiv forward. - ++argidx; + // oilAIntervals is before oilBIntervals. move oilAIntervals forward. + ++oilAIdx; } else if (cmp == Interval::INTERVAL_SUCCEEDS) { - // iv is before argiv. move iv forward. - ++ividx; + // oilBIntervals is before oilAIntervals. move oilBIntervals forward. + ++oilBIdx; } else { - // argiv[argidx] (cmpresults) iv[ividx] - Interval newInt = argiv[argidx]; - newInt.intersect(iv[ividx], cmp); + Interval newInt = oilAIntervals[oilAIdx]; + newInt.intersect(oilBIntervals[oilBIdx], cmp); result.push_back(newInt); if (Interval::INTERVAL_EQUALS == cmp) { - ++argidx; - ++ividx; + ++oilAIdx; + ++oilBIdx; } else if (Interval::INTERVAL_WITHIN == cmp) { - ++argidx; + ++oilAIdx; } else if (Interval::INTERVAL_CONTAINS == cmp) { - ++ividx; + ++oilBIdx; } else if (Interval::INTERVAL_OVERLAPS_BEFORE == cmp) { - ++argidx; + ++oilAIdx; } else if (Interval::INTERVAL_OVERLAPS_AFTER == cmp) { - ++ividx; + ++oilBIdx; } else { - verify(0); + MONGO_UNREACHABLE; } } } - oilOut->intervals.swap(result); + oilB->intervals.swap(result); } // static @@ -892,13 +914,7 @@ void IndexBoundsBuilder::alignBounds(IndexBounds* bounds, const BSONObj& kp, int int direction = (elt.number() >= 0) ? 1 : -1; direction *= scanDir; if (-1 == direction) { - vector<Interval>& iv = bounds->fields[oilIdx].intervals; - // Step 1: reverse the list. - std::reverse(iv.begin(), iv.end()); - // Step 2: reverse each interval. - for (size_t i = 0; i < iv.size(); ++i) { - iv[i].reverse(); - } + bounds->fields[oilIdx].reverse(); } ++oilIdx; } diff --git a/src/mongo/db/query/index_bounds_builder_test.cpp b/src/mongo/db/query/index_bounds_builder_test.cpp index a8ae3651d32..77d447094e5 100644 --- a/src/mongo/db/query/index_bounds_builder_test.cpp +++ b/src/mongo/db/query/index_bounds_builder_test.cpp @@ -2187,4 +2187,19 @@ TEST(IndexBoundsBuilderTest, CanUseCoveredMatchingForExistsTrueWithSparseIndex) ASSERT_TRUE(IndexBoundsBuilder::canUseCoveredMatching(expr.get(), testIndex)); } +TEST(IndexBoundsBuilderTest, IntersectizeBasic) { + OrderedIntervalList oil1("xyz"); + oil1.intervals = {Interval(BSON("" << 0 << "" << 5), false, false)}; + + OrderedIntervalList oil2("xyz"); + oil2.intervals = {Interval(BSON("" << 1 << "" << 6), false, false)}; + + IndexBoundsBuilder::intersectize(oil1, &oil2); + + OrderedIntervalList expectedIntersection("xyz"); + expectedIntersection.intervals = {Interval(BSON("" << 1 << "" << 5), false, false)}; + + ASSERT_TRUE(oil2 == expectedIntersection); +} + } // namespace diff --git a/src/mongo/db/query/index_bounds_test.cpp b/src/mongo/db/query/index_bounds_test.cpp index 1fca089e584..5597eaa9f6f 100644 --- a/src/mongo/db/query/index_bounds_test.cpp +++ b/src/mongo/db/query/index_bounds_test.cpp @@ -122,6 +122,58 @@ TEST(IndexBoundsTest, ValidOverlapOnlyWhenBothOpen) { ASSERT(bounds.isValidFor(BSON("foo" << 1), 1)); } +TEST(IndexBoundsCheckerTest, CheckOILReverse) { + // Check that the reverse of an empty list is empty. + OrderedIntervalList emptyList("someField"); + emptyList.reverse(); + OrderedIntervalList expectedReversedEmptyList("someField"); + ASSERT_TRUE(emptyList == expectedReversedEmptyList); + + // The reverse of a single-interval OIL is just an OIL with that interval reversed. + OrderedIntervalList singleEltList("xyz"); + singleEltList.intervals = {Interval(BSON("" << 5 << "" << 0), false, false)}; + singleEltList.reverse(); + + OrderedIntervalList expectedReversedSingleEltList("xyz"); + expectedReversedSingleEltList.intervals = {Interval(BSON("" << 0 << "" << 5), false, false)}; + ASSERT_TRUE(singleEltList == expectedReversedSingleEltList); + + // List with a few elements + OrderedIntervalList fooList("foo"); + fooList.intervals = {Interval(BSON("" << 40 << "" << 35), false, true), + Interval(BSON("" << 30 << "" << 21), true, true), + Interval(BSON("" << 20 << "" << 7), true, false)}; + fooList.reverse(); + + OrderedIntervalList expectedReverseFooList("foo"); + expectedReverseFooList.intervals = {Interval(BSON("" << 7 << "" << 20), false, true), + Interval(BSON("" << 21 << "" << 30), true, true), + Interval(BSON("" << 35 << "" << 40), true, false)}; + + ASSERT_TRUE(fooList == expectedReverseFooList); +} + +TEST(IndexBoundsTest, OILReverseClone) { + OrderedIntervalList emptyA("foo"); + OrderedIntervalList emptyB = emptyA.reverseClone(); + + ASSERT(emptyA == emptyB); + ASSERT(emptyA.computeDirection() == Interval::Direction::kDirectionNone); + ASSERT(emptyB.computeDirection() == Interval::Direction::kDirectionNone); + + OrderedIntervalList list("foo"); + + list.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, false)); + list.intervals.push_back(Interval(BSON("" << 20 << "" << 25), false, true)); + + OrderedIntervalList listClone = list.reverseClone(); + OrderedIntervalList reverseList("foo"); + reverseList.intervals = {Interval(BSON("" << 25 << "" << 20), true, false), + Interval(BSON("" << 20 << "" << 7), false, true)}; + ASSERT(reverseList == listClone); + ASSERT(listClone.computeDirection() == Interval::Direction::kDirectionDescending); +} + // // Tests for OrderedIntervalList::complement() // @@ -519,6 +571,47 @@ TEST(IndexBoundsTest, SimpleRangeBoundsNotEqualDifferentEndKeyInclusive) { ASSERT_TRUE(bounds1 != bounds2); } +TEST(IndexBoundsTest, ForwardizeSimpleRange) { + IndexBounds bounds1; + bounds1.isSimpleRange = true; + bounds1.startKey = BSON("" << 2 << "" << 4); + bounds1.endKey = BSON("" << 1 << "" << 3); + bounds1.boundInclusion = BoundInclusion::kIncludeStartKeyOnly; + + IndexBounds expectedBounds1; + expectedBounds1.isSimpleRange = true; + expectedBounds1.startKey = bounds1.endKey; + expectedBounds1.endKey = bounds1.startKey; + expectedBounds1.boundInclusion = BoundInclusion::kIncludeEndKeyOnly; + ASSERT(bounds1.forwardize() == expectedBounds1); + + IndexBounds bounds2; + bounds1.isSimpleRange = true; + bounds1.startKey = BSON("" << 1 << "" << 3); + bounds1.endKey = BSON("" << 2 << "" << 4); + bounds1.boundInclusion = BoundInclusion::kIncludeStartKeyOnly; + ASSERT(bounds2 == bounds2.forwardize()); +} + + +TEST(IndexBoundsTest, ForwardizeOnNonSimpleRangeShouldOnlyReverseDescendingRanges) { + OrderedIntervalList fooList("foo"); + fooList.intervals = {Interval(BSON("" << 7 << "" << 20), true, true)}; + + OrderedIntervalList barList("bar"); + barList.intervals = {Interval(BSON("" << 10 << "" << 5), false, false), + Interval(BSON("" << 4 << "" << 3), false, false)}; + + IndexBounds bounds; + bounds.fields = {fooList, barList}; + + IndexBounds forwardizedBounds = bounds.forwardize(); + + IndexBounds expectedBounds; + expectedBounds.fields = {fooList, barList.reverseClone()}; + ASSERT(expectedBounds == forwardizedBounds); +} + // // Iteration over // diff --git a/src/mongo/db/query/interval.cpp b/src/mongo/db/query/interval.cpp index df80321eb60..187d0704a20 100644 --- a/src/mongo/db/query/interval.cpp +++ b/src/mongo/db/query/interval.cpp @@ -66,6 +66,19 @@ bool Interval::isNull() const { return (!startInclusive || !endInclusive) && 0 == start.woCompare(end, false); } +Interval::Direction Interval::getDirection() const { + if (isEmpty() || isPoint() || isNull()) { + return Direction::kDirectionNone; + } + + // 'false' to not consider the field name. + const int res = start.woCompare(end, false); + + invariant(res != 0); + return res < 0 ? Direction::kDirectionAscending : Direction::kDirectionDescending; +} + + // // Comparison // @@ -93,6 +106,17 @@ bool Interval::equals(const Interval& other) const { } bool Interval::intersects(const Interval& other) const { + if (kDebugBuild) { + // This function assumes that both intervals are ascending (or are empty/point intervals). + // Determining this may be expensive, so we only do these checks when in a debug build. + const auto thisDir = getDirection(); + invariant(thisDir == Direction::kDirectionAscending || + thisDir == Direction::kDirectionNone); + const auto otherDir = other.getDirection(); + invariant(otherDir == Direction::kDirectionAscending || + otherDir == Direction::kDirectionNone); + } + int res = this->start.woCompare(other.end, false); if (res > 0) { return false; @@ -261,4 +285,15 @@ void Interval::reverse() { std::swap(startInclusive, endInclusive); } +Interval Interval::reverseClone() const { + Interval reversed; + reversed.start = end; + reversed.end = start; + reversed.startInclusive = endInclusive; + reversed.endInclusive = startInclusive; + reversed._intervalData = _intervalData; + + return reversed; +} + } // namespace mongo diff --git a/src/mongo/db/query/interval.h b/src/mongo/db/query/interval.h index 66767036f95..21b2cfce4ab 100644 --- a/src/mongo/db/query/interval.h +++ b/src/mongo/db/query/interval.h @@ -98,6 +98,18 @@ struct Interval { */ bool isNull() const; + enum class Direction { + // Point intervals, empty intervals, and null intervals have no direction. + kDirectionNone, + kDirectionAscending, + kDirectionDescending + }; + + /** + * Compute the direction. + */ + Direction getDirection() const; + // // Comparison with other intervals // @@ -169,6 +181,11 @@ struct Interval { void reverse(); /** + * Return a new Interval that's a reverse of this one. + */ + Interval reverseClone() const; + + /** * Updates 'this' with the intersection of 'this' and 'other'. If 'this' and 'other' * have been compare()d before, that result can be optionally passed in 'cmp' */ @@ -182,7 +199,7 @@ struct Interval { }; inline bool operator==(const Interval& lhs, const Interval& rhs) { - return lhs.compare(rhs) == Interval::INTERVAL_EQUALS; + return lhs.equals(rhs); } inline bool operator!=(const Interval& lhs, const Interval& rhs) { diff --git a/src/mongo/db/query/interval_test.cpp b/src/mongo/db/query/interval_test.cpp index d9e829a254b..608f7e25459 100644 --- a/src/mongo/db/query/interval_test.cpp +++ b/src/mongo/db/query/interval_test.cpp @@ -293,4 +293,41 @@ TEST(Union, Succeds) { ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); } +TEST(Introspection, GetDirection) { + // Empty/uninitialized Interval. + boost::optional<Interval> i; + i.emplace(); + ASSERT(i->getDirection() == Interval::Direction::kDirectionNone); + + // Empty Interval. + i.emplace(BSON("" << 10 << "" << 10), false, false); + ASSERT(i->getDirection() == Interval::Direction::kDirectionNone); + + // Point bound Interval. + i.emplace(BSON("" << 10 << "" << 10), true, true); + ASSERT(i->getDirection() == Interval::Direction::kDirectionNone); + + // Ascending interval. + i.emplace(BSON("" << 10 << "" << 20), true, true); + ASSERT(i->getDirection() == Interval::Direction::kDirectionAscending); + + // Descending interval. + i.emplace(BSON("" << 11 << "" << 10), true, true); + ASSERT(i->getDirection() == Interval::Direction::kDirectionDescending); +} + +TEST(Copying, ReverseClone) { + Interval a(BSON("" << 10 << "" << 20), false, true); + ASSERT(a.reverseClone() == Interval(BSON("" << 20 << "" << 10), true, false)); + ASSERT(a.reverseClone() != a); + + Interval b(BSON("" << 10 << "" << 5), true, true); + ASSERT(b.reverseClone() == Interval(BSON("" << 5 << "" << 10), true, true)); + ASSERT(b.reverseClone() != b); + + Interval c(BSON("" << 1 << "" << 1), true, true); + ASSERT(c.reverseClone() == c); +} + + } // unnamed namespace diff --git a/src/mongo/db/query/plan_ranker.cpp b/src/mongo/db/query/plan_ranker.cpp index ce943e117a7..9377e1382fc 100644 --- a/src/mongo/db/query/plan_ranker.cpp +++ b/src/mongo/db/query/plan_ranker.cpp @@ -95,7 +95,7 @@ size_t PlanRanker::pickBestPlan(const vector<CandidatePlan>& candidates, PlanRan LOG(5) << "Scoring plan " << i << ":" << endl << redact(candidates[i].solution->toString()) << "Stats:\n" << redact(Explain::statsToBSON(*statTrees[i]).jsonString(Strict, true)); - LOG(2) << "Scoring query plan: " << redact(Explain::getPlanSummary(candidates[i].root)) + LOG(2) << "Scoring query plan: " << Explain::getPlanSummary(candidates[i].root) << " planHitEOF=" << statTrees[i]->common.isEOF; double score = scoreTree(statTrees[i]); diff --git a/src/mongo/db/query/query_planner_common.cpp b/src/mongo/db/query/query_planner_common.cpp index 337f3a045fc..1347332c077 100644 --- a/src/mongo/db/query/query_planner_common.cpp +++ b/src/mongo/db/query/query_planner_common.cpp @@ -46,27 +46,11 @@ void QueryPlannerCommon::reverseScans(QuerySolutionNode* node) { if (isn->bounds.isSimpleRange) { std::swap(isn->bounds.startKey, isn->bounds.endKey); // If only one bound is included, swap which one is included. - switch (isn->bounds.boundInclusion) { - case BoundInclusion::kIncludeStartKeyOnly: - isn->bounds.boundInclusion = BoundInclusion::kIncludeEndKeyOnly; - break; - case BoundInclusion::kIncludeEndKeyOnly: - isn->bounds.boundInclusion = BoundInclusion::kIncludeStartKeyOnly; - break; - case BoundInclusion::kIncludeBothStartAndEndKeys: - case BoundInclusion::kExcludeBothStartAndEndKeys: - // These are both symmetric so no change needed. - break; - } + isn->bounds.boundInclusion = + IndexBounds::reverseBoundInclusion(isn->bounds.boundInclusion); } else { for (size_t i = 0; i < isn->bounds.fields.size(); ++i) { - std::vector<Interval>& iv = isn->bounds.fields[i].intervals; - // Step 1: reverse the list. - std::reverse(iv.begin(), iv.end()); - // Step 2: reverse each interval. - for (size_t j = 0; j < iv.size(); ++j) { - iv[j].reverse(); - } + isn->bounds.fields[i].reverse(); } } diff --git a/src/mongo/db/query/query_solution.cpp b/src/mongo/db/query/query_solution.cpp index aa2426954c4..9ab324ab3bd 100644 --- a/src/mongo/db/query/query_solution.cpp +++ b/src/mongo/db/query/query_solution.cpp @@ -594,9 +594,13 @@ bool IndexScanNode::sortedByDiskLoc() const { } // static -std::set<StringData> IndexScanNode::getFieldsWithStringBounds(const IndexBounds& bounds, +std::set<StringData> IndexScanNode::getFieldsWithStringBounds(const IndexBounds& inputBounds, const BSONObj& indexKeyPattern) { - BSONObjIterator keyPatternIterator = indexKeyPattern.begin(); + // Produce a copy of the bounds which are all ascending, as we can only compute intersections + // of ascending bounds. + IndexBounds bounds = inputBounds.forwardize(); + + BSONObjIterator keyPatternIterator(indexKeyPattern); if (bounds.isSimpleRange) { // With a simple range, the only cases we can say for sure do not contain strings diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index fbbf1c56efd..ac49485ede0 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -427,7 +427,7 @@ void BackgroundSync::_produce() { source, NamespaceString(rsOplogName), _replCoord->getConfig(), - _replicationCoordinatorExternalState->getOplogFetcherMaxFetcherRestarts(), + _replicationCoordinatorExternalState->getOplogFetcherSteadyStateMaxFetcherRestarts(), syncSourceResp.rbid, true /* requireFresherSyncSource */, &dataReplicatorExternalState, diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 76df271ed78..13769c5334d 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -658,12 +658,16 @@ std::map<std::string, ApplyOpMetadata> opsMap = { return applyOps(txn, nsToDatabase(ns), cmd, &resultWeDontCareAbout); }, {ErrorCodes::UnknownError}}}, - {"convertToCapped", {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { - return convertToCapped(txn, parseNs(ns, cmd), cmd["size"].number()); - }}}, - {"emptycapped", {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { - return emptyCapped(txn, parseNs(ns, cmd)); - }}}, + {"convertToCapped", + {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { + return convertToCapped(txn, parseNs(ns, cmd), cmd["size"].number()); + }, + {ErrorCodes::NamespaceNotFound}}}, + {"emptycapped", + {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { + return emptyCapped(txn, parseNs(ns, cmd)); + }, + {ErrorCodes::NamespaceNotFound}}}, }; } // namespace diff --git a/src/mongo/db/repl/oplog_fetcher.cpp b/src/mongo/db/repl/oplog_fetcher.cpp index c9b96ab77f7..36a6e237e73 100644 --- a/src/mongo/db/repl/oplog_fetcher.cpp +++ b/src/mongo/db/repl/oplog_fetcher.cpp @@ -58,8 +58,15 @@ MONGO_FP_DECLARE(stopReplProducer); namespace { // Number of seconds for the `maxTimeMS` on the initial `find` command. +// +// For the initial 'find' request, we provide a generous timeout, to account for the potentially +// slow process of a sync source finding the lastApplied optime provided in a node's query in its +// oplog. MONGO_EXPORT_SERVER_PARAMETER(oplogInitialFindMaxSeconds, int, 60); +// Number of seconds for the `maxTimeMS` on any retried `find` commands. +MONGO_EXPORT_SERVER_PARAMETER(oplogRetriedFindMaxSeconds, int, 2); + // Number of milliseconds to add to the `find` and `getMore` timeouts to calculate the network // timeout for the requests. const Milliseconds kNetworkTimeoutBufferMS{5000}; @@ -373,7 +380,7 @@ OplogFetcher::OplogFetcher(executor::TaskExecutor* executor, uassert(ErrorCodes::BadValue, "null onShutdownCallback function", onShutdownCallbackFn); auto currentTerm = dataReplicatorExternalState->getCurrentTermAndLastCommittedOpTime().value; - _fetcher = _makeFetcher(currentTerm, _lastFetched.opTime); + _fetcher = _makeFetcher(currentTerm, _lastFetched.opTime, _getInitialFindMaxTime()); } OplogFetcher::~OplogFetcher() { @@ -463,10 +470,14 @@ Milliseconds OplogFetcher::getAwaitDataTimeout_forTest() const { return _getGetMoreMaxTime(); } -Milliseconds OplogFetcher::_getFindMaxTime() const { +Milliseconds OplogFetcher::_getInitialFindMaxTime() const { return Milliseconds(oplogInitialFindMaxSeconds.load() * 1000); } +Milliseconds OplogFetcher::_getRetriedFindMaxTime() const { + return Milliseconds(oplogRetriedFindMaxSeconds.load() * 1000); +} + Milliseconds OplogFetcher::_getGetMoreMaxTime() const { return _awaitDataTimeout; } @@ -511,7 +522,7 @@ void OplogFetcher::_callback(const Fetcher::QueryResponseStatus& result, // Move the old fetcher into the shutting down instance. _shuttingDownFetcher.swap(_fetcher); // Create and start fetcher with current term and new starting optime. - _fetcher = _makeFetcher(currentTerm, _lastFetched.opTime); + _fetcher = _makeFetcher(currentTerm, _lastFetched.opTime, _getRetriedFindMaxTime()); auto scheduleStatus = _scheduleFetcher_inlock(); if (scheduleStatus.isOK()) { log() << "Scheduled new oplog query " << _fetcher->toString(); @@ -704,15 +715,16 @@ void OplogFetcher::_finishCallback(Status status, OpTimeWithHash opTimeWithHash) } std::unique_ptr<Fetcher> OplogFetcher::_makeFetcher(long long currentTerm, - OpTime lastFetchedOpTime) { + OpTime lastFetchedOpTime, + Milliseconds findMaxTime) { return stdx::make_unique<Fetcher>( _executor, _source, _nss.db().toString(), - makeFindCommandObject(_nss, currentTerm, lastFetchedOpTime, _getFindMaxTime()), + makeFindCommandObject(_nss, currentTerm, lastFetchedOpTime, findMaxTime), stdx::bind(&OplogFetcher::_callback, this, stdx::placeholders::_1, stdx::placeholders::_3), _metadataObject, - _getFindMaxTime() + kNetworkTimeoutBufferMS, + findMaxTime + kNetworkTimeoutBufferMS, _getGetMoreMaxTime() + kNetworkTimeoutBufferMS); } diff --git a/src/mongo/db/repl/oplog_fetcher.h b/src/mongo/db/repl/oplog_fetcher.h index 54bfbabbf8d..c567598a66f 100644 --- a/src/mongo/db/repl/oplog_fetcher.h +++ b/src/mongo/db/repl/oplog_fetcher.h @@ -237,7 +237,16 @@ private: /** * Returns how long the `find` command should wait before timing out. */ - virtual Milliseconds _getFindMaxTime() const; + virtual Milliseconds _getInitialFindMaxTime() const; + + /** + * Returns how long the `find` command should wait before timing out, if we are retrying the + * 'find' due to an error. This timeout should be considerably smaller than our initial oplog + * find time, since a communication failure with an upstream node may indicate it is + * unreachable. + */ + virtual Milliseconds _getRetriedFindMaxTime() const; + /** * Returns how long the `getMore` command should wait before timing out. @@ -247,7 +256,9 @@ private: /** * Creates a new instance of the fetcher to tail the remote oplog starting at the given optime. */ - std::unique_ptr<Fetcher> _makeFetcher(long long currentTerm, OpTime lastFetchedOpTime); + std::unique_ptr<Fetcher> _makeFetcher(long long currentTerm, + OpTime lastFetchedOpTime, + Milliseconds findMaxTime); /** * Returns whether the oplog fetcher is in shutdown. diff --git a/src/mongo/db/repl/oplog_fetcher_test.cpp b/src/mongo/db/repl/oplog_fetcher_test.cpp index 9170c203695..2b25d0bbe02 100644 --- a/src/mongo/db/repl/oplog_fetcher_test.cpp +++ b/src/mongo/db/repl/oplog_fetcher_test.cpp @@ -204,6 +204,11 @@ BSONObj OplogFetcherTest::makeOplogQueryMetadataObject(OpTime lastAppliedOpTime, HostAndPort source("localhost:12345"); NamespaceString nss("local.oplog.rs"); +// For testing, set these network timeouts to match the defaults in the OplogFetcher. +const Milliseconds kNetworkTimeoutBufferMS{5000}; +const Milliseconds initialFindMaxTime = Milliseconds(60000); +const Milliseconds retriedFindMaxTime = Milliseconds(2000); + ReplSetConfig _createConfig(bool isV1ElectionProtocol) { BSONObjBuilder bob; bob.append("_id", "myset"); @@ -1478,6 +1483,99 @@ TEST_F(OplogFetcherTest, OplogFetcherAbortsWithOriginalResponseErrorOnFailureToS ASSERT_EQUALS(_getOpTimeWithHash(ops[2]), shutdownState->getLastFetched()); } +TEST_F(OplogFetcherTest, OplogFetcherTimesOutCorrectlyOnInitialFindRequests) { + auto ops = _generateOplogEntries(2U); + std::size_t maxFetcherRestarts = 0U; + auto shutdownState = stdx::make_unique<ShutdownState>(); + OplogFetcher oplogFetcher(&getExecutor(), + _getOpTimeWithHash(ops[0]), + source, + nss, + _createConfig(true), + maxFetcherRestarts, + rbid, + true, + dataReplicatorExternalState.get(), + enqueueDocumentsFn, + stdx::ref(*shutdownState)); + + ON_BLOCK_EXIT([this] { getExecutor().shutdown(); }); + + ASSERT_OK(oplogFetcher.startup()); + ASSERT_TRUE(oplogFetcher.isActive()); + + auto net = getNet(); + + // Schedule a response at a time that would exceed the initial find request network timeout. + net->enterNetwork(); + auto when = net->now() + initialFindMaxTime + kNetworkTimeoutBufferMS + Milliseconds(10); + auto noi = getNet()->getNextReadyRequest(); + RemoteCommandResponse response = { + {makeCursorResponse(1, {ops[0], ops[1]})}, rpc::makeEmptyMetadata(), Milliseconds(0)}; + auto request = net->scheduleSuccessfulResponse(noi, when, response); + net->runUntil(when); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + oplogFetcher.join(); + + // The fetcher should have shut down after its last request timed out. + ASSERT_EQUALS(ErrorCodes::NetworkTimeout, shutdownState->getStatus()); +} + +TEST_F(OplogFetcherTest, OplogFetcherTimesOutCorrectlyOnRetriedFindRequests) { + auto ops = _generateOplogEntries(2U); + std::size_t maxFetcherRestarts = 1U; + auto shutdownState = stdx::make_unique<ShutdownState>(); + OplogFetcher oplogFetcher(&getExecutor(), + _getOpTimeWithHash(ops[0]), + source, + nss, + _createConfig(true), + maxFetcherRestarts, + rbid, + true, + dataReplicatorExternalState.get(), + enqueueDocumentsFn, + stdx::ref(*shutdownState)); + + + ON_BLOCK_EXIT([this] { getExecutor().shutdown(); }); + + ASSERT_OK(oplogFetcher.startup()); + ASSERT_TRUE(oplogFetcher.isActive()); + + auto net = getNet(); + + // Schedule a response at a time that would exceed the initial find request network timeout. + net->enterNetwork(); + auto when = net->now() + initialFindMaxTime + kNetworkTimeoutBufferMS + Milliseconds(10); + auto noi = getNet()->getNextReadyRequest(); + RemoteCommandResponse response = { + {makeCursorResponse(1, {ops[0], ops[1]})}, rpc::makeEmptyMetadata(), Milliseconds(0)}; + auto request = net->scheduleSuccessfulResponse(noi, when, response); + net->runUntil(when); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + // Schedule a response at a time that would exceed the retried find request network timeout. + net->enterNetwork(); + when = net->now() + retriedFindMaxTime + kNetworkTimeoutBufferMS + Milliseconds(10); + noi = getNet()->getNextReadyRequest(); + response = { + {makeCursorResponse(1, {ops[0], ops[1]})}, rpc::makeEmptyMetadata(), Milliseconds(0)}; + request = net->scheduleSuccessfulResponse(noi, when, response); + net->runUntil(when); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + oplogFetcher.join(); + + // The fetcher should have shut down after its last request timed out. + ASSERT_EQUALS(ErrorCodes::NetworkTimeout, shutdownState->getStatus()); +} + + bool sharedCallbackStateDestroyed = false; class SharedCallbackState { MONGO_DISALLOW_COPYING(SharedCallbackState); diff --git a/src/mongo/db/repl/repl_set_request_votes.cpp b/src/mongo/db/repl/repl_set_request_votes.cpp index 02eb5311cb3..8c1fe8adece 100644 --- a/src/mongo/db/repl/repl_set_request_votes.cpp +++ b/src/mongo/db/repl/repl_set_request_votes.cpp @@ -83,8 +83,10 @@ private: ReplSetRequestVotesResponse response; status = getGlobalReplicationCoordinator()->processReplSetRequestVotes( txn, parsedArgs, &response); + uassertStatusOK(status); + response.addToBSON(&result); - return appendCommandStatus(result, status); + return true; } } cmdReplSetRequestVotes; diff --git a/src/mongo/db/repl/replication_coordinator_external_state.h b/src/mongo/db/repl/replication_coordinator_external_state.h index 8776bfa8330..dadaaff2d2d 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state.h +++ b/src/mongo/db/repl/replication_coordinator_external_state.h @@ -348,9 +348,15 @@ public: /** * Returns maximum number of times that the oplog fetcher will consecutively restart the oplog - * tailing query on non-cancellation errors. + * tailing query on non-cancellation errors during steady state replication. */ - virtual std::size_t getOplogFetcherMaxFetcherRestarts() const = 0; + virtual std::size_t getOplogFetcherSteadyStateMaxFetcherRestarts() const = 0; + + /** + * Returns maximum number of times that the oplog fetcher will consecutively restart the oplog + * tailing query on non-cancellation errors during initial sync. + */ + virtual std::size_t getOplogFetcherInitialSyncMaxFetcherRestarts() const = 0; /* * Creates noop writer instance. Setting the _noopWriter member is not protected by a guard, diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index 05d070c08fe..597c9b47799 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -37,6 +37,9 @@ #include "mongo/base/init.h" #include "mongo/base/status_with.h" #include "mongo/bson/oid.h" +#include "mongo/db/auth/auth_index_d.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_holder.h" #include "mongo/db/client.h" @@ -128,29 +131,59 @@ MONGO_EXPORT_STARTUP_SERVER_PARAMETER(initialSyncOplogBuffer, // Set this to specify size of read ahead buffer in the OplogBufferCollection. MONGO_EXPORT_STARTUP_SERVER_PARAMETER(initialSyncOplogBufferPeekCacheSize, int, 10000); -// Set this to specify maximum number of times the oplog fetcher will consecutively restart the -// oplog tailing query on non-cancellation errors. +// Set this to specify the maximum number of times the oplog fetcher will consecutively restart the +// oplog tailing query on non-cancellation errors during steady state replication. server_parameter_storage_type<int, ServerParameterType::kStartupAndRuntime>::value_type - oplogFetcherMaxFetcherRestarts(3); -class ExportedOplogFetcherMaxFetcherRestartsServerParameter + oplogFetcherSteadyStateMaxFetcherRestarts(1); +class ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter : public ExportedServerParameter<int, ServerParameterType::kStartupAndRuntime> { public: - ExportedOplogFetcherMaxFetcherRestartsServerParameter(); + ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter(); Status validate(const int& potentialNewValue) override; -} _exportedOplogFetcherMaxFetcherRestartsServerParameter; +} _exportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter; -ExportedOplogFetcherMaxFetcherRestartsServerParameter:: - ExportedOplogFetcherMaxFetcherRestartsServerParameter() +ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter:: + ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter() : ExportedServerParameter<int, ServerParameterType::kStartupAndRuntime>( ServerParameterSet::getGlobal(), - "oplogFetcherMaxFetcherRestarts", - &oplogFetcherMaxFetcherRestarts) {} + "oplogFetcherSteadyStateMaxFetcherRestarts", + &oplogFetcherSteadyStateMaxFetcherRestarts) {} -Status ExportedOplogFetcherMaxFetcherRestartsServerParameter::validate( +Status ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter::validate( const int& potentialNewValue) { if (potentialNewValue < 0) { - return Status(ErrorCodes::BadValue, - "oplogFetcherMaxFetcherRestarts must be greater than or equal to 0"); + return Status( + ErrorCodes::BadValue, + "oplogFetcherSteadyStateMaxFetcherRestarts must be greater than or equal to 0"); + } + return Status::OK(); +} + +// Set this to specify the maximum number of times the oplog fetcher will consecutively restart the +// oplog tailing query on non-cancellation errors during initial sync. By default we provide a +// generous amount of restarts to avoid potentially restarting an entire initial sync from scratch. +server_parameter_storage_type<int, ServerParameterType::kStartupAndRuntime>::value_type + oplogFetcherInitialSyncMaxFetcherRestarts(10); +class ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter + : public ExportedServerParameter<int, ServerParameterType::kStartupAndRuntime> { +public: + ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter(); + Status validate(const int& potentialNewValue) override; +} _exportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter; + +ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter:: + ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter() + : ExportedServerParameter<int, ServerParameterType::kStartupAndRuntime>( + ServerParameterSet::getGlobal(), + "oplogFetcherInitialSyncMaxFetcherRestarts", + &oplogFetcherInitialSyncMaxFetcherRestarts) {} + +Status ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter::validate( + const int& potentialNewValue) { + if (potentialNewValue < 0) { + return Status( + ErrorCodes::BadValue, + "oplogFetcherInitialSyncMaxFetcherRestarts must be greater than or equal to 0"); } return Status::OK(); } @@ -454,6 +487,16 @@ OpTime ReplicationCoordinatorExternalStateImpl::onTransitionToPrimary(OperationC _shardingOnTransitionToPrimaryHook(txn); _dropAllTempCollections(txn); + // It is only necessary to check the system indexes on the first transition to master. + // On subsequent transitions to master the indexes will have already been created. + static std::once_flag verifySystemIndexesOnce; + std::call_once(verifySystemIndexesOnce, [txn] { + const auto globalAuthzManager = AuthorizationManager::get(txn->getServiceContext()); + if (globalAuthzManager->shouldValidateAuthSchemaOnStartup()) { + fassert(65536, authindex::verifySystemIndexes(txn)); + } + }); + serverGlobalParams.featureCompatibility.validateFeaturesAsMaster.store(true); return opTimeToReturn; @@ -944,8 +987,14 @@ bool ReplicationCoordinatorExternalStateImpl::shouldUseDataReplicatorInitialSync return !use3dot2InitialSync; } -std::size_t ReplicationCoordinatorExternalStateImpl::getOplogFetcherMaxFetcherRestarts() const { - return oplogFetcherMaxFetcherRestarts; +std::size_t ReplicationCoordinatorExternalStateImpl::getOplogFetcherSteadyStateMaxFetcherRestarts() + const { + return oplogFetcherSteadyStateMaxFetcherRestarts.load(); +} + +std::size_t ReplicationCoordinatorExternalStateImpl::getOplogFetcherInitialSyncMaxFetcherRestarts() + const { + return oplogFetcherInitialSyncMaxFetcherRestarts.load(); } JournalListener::Token ReplicationCoordinatorExternalStateImpl::getToken() { diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.h b/src/mongo/db/repl/replication_coordinator_external_state_impl.h index 8926f378829..ff2fa102982 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.h +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.h @@ -116,7 +116,8 @@ public: virtual std::unique_ptr<OplogBuffer> makeSteadyStateOplogBuffer( OperationContext* txn) const override; virtual bool shouldUseDataReplicatorInitialSync() const override; - virtual std::size_t getOplogFetcherMaxFetcherRestarts() const override; + virtual std::size_t getOplogFetcherSteadyStateMaxFetcherRestarts() const override; + virtual std::size_t getOplogFetcherInitialSyncMaxFetcherRestarts() const override; // Methods from JournalListener. virtual JournalListener::Token getToken(); diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp index 6b832728f24..cef211451a6 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp @@ -288,7 +288,13 @@ bool ReplicationCoordinatorExternalStateMock::shouldUseDataReplicatorInitialSync return true; } -std::size_t ReplicationCoordinatorExternalStateMock::getOplogFetcherMaxFetcherRestarts() const { +std::size_t ReplicationCoordinatorExternalStateMock::getOplogFetcherSteadyStateMaxFetcherRestarts() + const { + return 0; +} + +std::size_t ReplicationCoordinatorExternalStateMock::getOplogFetcherInitialSyncMaxFetcherRestarts() + const { return 0; } diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.h b/src/mongo/db/repl/replication_coordinator_external_state_mock.h index c22e053bf35..7120433bd73 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.h +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.h @@ -109,7 +109,8 @@ public: virtual std::unique_ptr<OplogBuffer> makeSteadyStateOplogBuffer( OperationContext* txn) const override; virtual bool shouldUseDataReplicatorInitialSync() const override; - virtual std::size_t getOplogFetcherMaxFetcherRestarts() const override; + virtual std::size_t getOplogFetcherSteadyStateMaxFetcherRestarts() const override; + virtual std::size_t getOplogFetcherInitialSyncMaxFetcherRestarts() const override; /** * Adds "host" to the list of hosts that this mock will match when responding to "isSelf" diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 10b60b7a205..eec1b8989ae 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -289,7 +289,8 @@ InitialSyncerOptions createInitialSyncerOptions( options.getSlaveDelay = [replCoord]() { return replCoord->getSlaveDelaySecs(); }; options.syncSourceSelector = replCoord; options.replBatchLimitBytes = dur::UncommittedBytesLimit; - options.oplogFetcherMaxFetcherRestarts = externalState->getOplogFetcherMaxFetcherRestarts(); + options.oplogFetcherMaxFetcherRestarts = + externalState->getOplogFetcherInitialSyncMaxFetcherRestarts(); return options; } } // namespace diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp index 09953633f7e..f9fea88bf0b 100644 --- a/src/mongo/db/repl/sync_tail_test.cpp +++ b/src/mongo/db/repl/sync_tail_test.cpp @@ -1530,4 +1530,38 @@ TEST_F(IdempotencyTest, ResyncOnRenameCollection) { ASSERT_EQUALS(runOp(op), ErrorCodes::OplogOperationUnsupported); } +TEST_F(IdempotencyTest, EmptyCappedNamespaceNotFound) { + // Create a BSON "emptycapped" command. + auto emptyCappedCmd = BSON("emptycapped" << nss.coll()); + + // Create an "emptycapped" oplog entry. + auto emptyCappedOp = makeCommandOplogEntry(nextOpTime(), nss, emptyCappedCmd); + + // Ensure that NamespaceNotFound is acceptable. + ASSERT_OK(runOps({emptyCappedOp})); + + AutoGetCollectionForRead autoColl(_opCtx.get(), nss); + + // Ensure that autoColl.getCollection() and autoColl.getDb() are both null. + ASSERT_FALSE(autoColl.getCollection()); + ASSERT_FALSE(autoColl.getDb()); +} + +TEST_F(IdempotencyTest, ConvertToCappedNamespaceNotFound) { + // Create a BSON "convertToCapped" command. + auto convertToCappedCmd = BSON("convertToCapped" << nss.coll()); + + // Create a "convertToCapped" oplog entry. + auto convertToCappedOp = makeCommandOplogEntry(nextOpTime(), nss, convertToCappedCmd); + + // Ensure that NamespaceNotFound is acceptable. + ASSERT_OK(runOps({convertToCappedOp})); + + AutoGetCollectionForRead autoColl(_opCtx.get(), nss); + + // Ensure that autoColl.getCollection() and autoColl.getDb() are both null. + ASSERT_FALSE(autoColl.getCollection()); + ASSERT_FALSE(autoColl.getDb()); +} + } // namespace diff --git a/src/mongo/db/repl/task_runner.cpp b/src/mongo/db/repl/task_runner.cpp index 210718bba3e..134160bbdf1 100644 --- a/src/mongo/db/repl/task_runner.cpp +++ b/src/mongo/db/repl/task_runner.cpp @@ -131,20 +131,17 @@ void TaskRunner::join() { } void TaskRunner::_runTasks() { - Client* client = nullptr; + // We initialize cc() because ServiceContextMongoD::_newOpCtx() expects cc() to be equal to the + // client used to create the operation context. + Client::initThreadIfNotAlready(); + Client* client = &cc(); + if (AuthorizationManager::get(client->getServiceContext())->isAuthEnabled()) { + AuthorizationSession::get(client)->grantInternalAuthorization(); + } ServiceContext::UniqueOperationContext txn; while (Task task = _waitForNextTask()) { if (!txn) { - if (!client) { - // We initialize cc() because ServiceContextMongoD::_newOpCtx() expects cc() - // to be equal to the client used to create the operation context. - Client::initThreadIfNotAlready(); - client = &cc(); - if (getGlobalAuthorizationManager()->isAuthEnabled()) { - AuthorizationSession::get(client)->grantInternalAuthorization(); - } - } txn = client->makeOperationContext(); } diff --git a/src/mongo/db/run_commands.cpp b/src/mongo/db/run_commands.cpp index 41adbdfb507..9068b8c9adb 100644 --- a/src/mongo/db/run_commands.cpp +++ b/src/mongo/db/run_commands.cpp @@ -62,7 +62,7 @@ void runCommands(OperationContext* txn, } LOG(2) << "run command " << request.getDatabase() << ".$cmd" << ' ' - << c->getRedactedCopyForLogging(request.getCommandArgs()); + << redact(c->getRedactedCopyForLogging(request.getCommandArgs())); { // Try to set this as early as possible, as soon as we have figured out the command. diff --git a/src/mongo/db/s/metadata_manager.cpp b/src/mongo/db/s/metadata_manager.cpp index 0bba1ff478e..afda09358f2 100644 --- a/src/mongo/db/s/metadata_manager.cpp +++ b/src/mongo/db/s/metadata_manager.cpp @@ -192,8 +192,6 @@ void MetadataManager::beginReceive(const ChunkRange& range) { auto itRecv = _receivingChunks.find(overlapChunkMin.first); invariant(itRecv != _receivingChunks.end()); - const ChunkRange receivingRange(itRecv->first, itRecv->second.getMaxKey()); - _receivingChunks.erase(itRecv); } diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp index 8dae501c57e..3c62381a1f8 100644 --- a/src/mongo/db/s/migration_destination_manager.cpp +++ b/src/mongo/db/s/migration_destination_manager.cpp @@ -47,6 +47,8 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/ops/delete.h" +#include "mongo/db/ops/write_ops.h" +#include "mongo/db/ops/write_ops_exec.h" #include "mongo/db/range_deleter_service.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/replication_coordinator_global.h" @@ -56,6 +58,7 @@ #include "mongo/db/s/move_timing_helper.h" #include "mongo/db/s/sharded_connection_info.h" #include "mongo/db/s/sharding_state.h" +#include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" #include "mongo/logger/ramlog.h" #include "mongo/s/catalog/type_chunk.h" @@ -349,7 +352,7 @@ Status MigrationDestinationManager::start(const NamespaceString& nss, void MigrationDestinationManager::cloneDocumentsFromDonor( OperationContext* txn, - stdx::function<void(OperationContext*, BSONObjIterator)> insertBatchFn, + stdx::function<void(OperationContext*, BSONObj)> insertBatchFn, stdx::function<BSONObj(OperationContext*)> fetchBatchFn) { ProducerConsumerQueue<BSONObj> batches(1); @@ -364,7 +367,7 @@ void MigrationDestinationManager::cloneDocumentsFromDonor( if (arr.isEmpty()) { return; } - insertBatchFn(inserterTxn.get(), BSONObjIterator(arr)); + insertBatchFn(inserterTxn.get(), arr); } } catch (...) { stdx::lock_guard<Client> lk(*txn->getClient()); @@ -506,6 +509,17 @@ void MigrationDestinationManager::_migrateThread(BSONObj min, _isActiveCV.notify_all(); } +// The maximum number of documents to insert in a single batch during migration clone. +// secondaryThrottle and migrateCloneInsertionBatchDelayMS apply between each batch. +// 0 or negative values (the default) means no limit to batch size. +// 1 corresponds to 3.4.16 (and earlier) behavior. +MONGO_EXPORT_SERVER_PARAMETER(migrateCloneInsertionBatchSize, int, 0); + +// Time in milliseconds between batches of insertions during migration clone. +// This is in addition to any time spent waiting for replication (secondaryThrottle). +// Defaults to 0. +MONGO_EXPORT_SERVER_PARAMETER(migrateCloneInsertionBatchDelayMS, int, 0); + void MigrationDestinationManager::_migrateDriver(OperationContext* txn, const BSONObj& min, const BSONObj& max, @@ -665,6 +679,12 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* txn, wunit.commit(); } + Status status = _notePending(txn, _nss, min, max, epoch); + if (!status.isOK()) { + setState(FAIL); + return; + } + timing.done(1); MONGO_FAIL_POINT_PAUSE_WHILE_SET(migrateThreadHangAtStep1); } @@ -680,7 +700,12 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* txn, // results deleterOptions.waitForOpenCursors = false; deleterOptions.fromMigrate = true; - deleterOptions.onlyRemoveOrphanedDocs = true; + + // There is no need to perform checking for orphaned docs as part of the range deletion, + // because the call above (to _notePending) will ensure that the chunk which is coming in is + // not currently owned by this shard and the scopedRegisterReceiveChunk would prevent this + // chunk from getting received while range deletion is running. + deleterOptions.onlyRemoveOrphanedDocs = false; deleterOptions.removeSaverReason = "preCleanup"; if (!getDeleter()->deleteNow(txn, deleterOptions, &errmsg)) { @@ -689,12 +714,6 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* txn, return; } - Status status = _notePending(txn, _nss, min, max, epoch); - if (!status.isOK()) { - setState(FAIL); - return; - } - timing.done(2); MONGO_FAIL_POINT_PAUSE_WHILE_SET(migrateThreadHangAtStep2); } @@ -705,57 +724,61 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* txn, const BSONObj migrateCloneRequest = createMigrateCloneRequest(_nss, *_sessionId); - auto insertBatchFn = [&](OperationContext* txn, BSONObjIterator docs) { - while (docs.more()) { - txn->checkForInterrupt(); + auto assertNotAborted = [&](OperationContext* opCtx) { + opCtx->checkForInterrupt(); + uassert(40655, "Migration aborted while copying documents", getState() != ABORT); + }; - if (getState() == ABORT) { - auto message = "Migration aborted while copying documents"; - log() << message << migrateLog; - uasserted(40655, message); + auto insertBatchFn = [&](OperationContext* opCtx, BSONObj arr) { + auto it = arr.begin(); + while (it != arr.end()) { + int batchNumCloned = 0; + int batchClonedBytes = 0; + int batchMaxCloned = migrateCloneInsertionBatchSize.load(); + + assertNotAborted(opCtx); + + std::vector<BSONObj> toInsert; + while (it != arr.end() && + (batchMaxCloned <= 0 || batchNumCloned < batchMaxCloned)) { + const auto& doc = *it; + BSONObj docToClone = doc.Obj(); + toInsert.push_back(docToClone); + batchNumCloned++; + batchClonedBytes += docToClone.objsize(); + it++; } + InsertOp insertOp; + insertOp.ns = _nss; + insertOp.documents = toInsert; - BSONObj docToClone = docs.next().Obj(); - { - OldClientWriteContext cx(txn, _nss.ns()); - BSONObj localDoc; - if (willOverrideLocalId(txn, - _nss.ns(), - min, - max, - shardKeyPattern, - cx.db(), - docToClone, - &localDoc)) { - const std::string errMsg = str::stream() - << "cannot migrate chunk, local document " << redact(localDoc) - << " has same _id as cloned " - << "remote document " << redact(docToClone); - warning() << errMsg; - - // Exception will abort migration cleanly - uasserted(16976, errMsg); - } - Helpers::upsert(txn, _nss.ns(), docToClone, true); + const WriteResult reply = performInserts(opCtx, insertOp, true); + + for (unsigned long i = 0; i < reply.results.size(); ++i) { + uassertStatusOK(reply.results[i]); } + { stdx::lock_guard<stdx::mutex> statsLock(_mutex); - _numCloned++; - _clonedBytes += docToClone.objsize(); + _numCloned += batchNumCloned; + _clonedBytes += batchClonedBytes; } + if (writeConcern.shouldWaitForOtherNodes()) { repl::ReplicationCoordinator::StatusAndDuration replStatus = - repl::ReplicationCoordinator::get(txn)->awaitReplication( - txn, - repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(), + repl::ReplicationCoordinator::get(opCtx)->awaitReplication( + opCtx, + repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(), writeConcern); if (replStatus.status.code() == ErrorCodes::WriteConcernFailed) { warning() << "secondaryThrottle on, but doc insert timed out; " "continuing"; } else { - massertStatusOK(replStatus.status); + uassertStatusOK(replStatus.status); } } + + sleepmillis(migrateCloneInsertionBatchDelayMS.load()); } }; diff --git a/src/mongo/db/s/migration_destination_manager.h b/src/mongo/db/s/migration_destination_manager.h index 836761530d4..791aa393116 100644 --- a/src/mongo/db/s/migration_destination_manager.h +++ b/src/mongo/db/s/migration_destination_manager.h @@ -105,7 +105,7 @@ public: */ static void cloneDocumentsFromDonor( OperationContext* txn, - stdx::function<void(OperationContext*, BSONObjIterator)> insertBatchFn, + stdx::function<void(OperationContext*, BSONObj)> insertBatchFn, stdx::function<BSONObj(OperationContext*)> fetchBatchFn); /** diff --git a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp index ac9752fd96d..4c28de35b44 100644 --- a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp +++ b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp @@ -111,11 +111,16 @@ public: } } - const NamespaceString nss(cmdObj.firstElement().String()); - const auto chunkRange = uassertStatusOK(ChunkRange::fromBSON(cmdObj)); + const MigrationSessionId migrationSessionId( + uassertStatusOK(MigrationSessionId::extractFromBSON(cmdObj))); + + // Ensure this shard is not currently receiving or donating any chunks. + auto scopedRegisterReceiveChunk( + uassertStatusOK(shardingState->registerReceiveChunk(nss, chunkRange, fromShard))); + // Refresh our collection manager from the config server, we need a collection manager to // start registering pending chunks. We force the remote refresh here to make the behavior // consistent and predictable, generally we'd refresh anyway, and to be paranoid. @@ -147,13 +152,6 @@ public: return false; } - const MigrationSessionId migrationSessionId( - uassertStatusOK(MigrationSessionId::extractFromBSON(cmdObj))); - - // Ensure this shard is not currently receiving or donating any chunks. - auto scopedRegisterReceiveChunk( - uassertStatusOK(shardingState->registerReceiveChunk(nss, chunkRange, fromShard))); - // Even if this shard is not currently donating any chunks, it may still have pending // deletes from a previous migration, particularly if there are still open cursors on the // range pending deletion. diff --git a/src/mongo/db/s/migration_destination_manager_test.cpp b/src/mongo/db/s/migration_destination_manager_test.cpp index f1ce14d3147..b9daad4ad9f 100644 --- a/src/mongo/db/s/migration_destination_manager_test.cpp +++ b/src/mongo/db/s/migration_destination_manager_test.cpp @@ -84,9 +84,9 @@ TEST_F(MigrationDestinationManagerTest, CloneDocumentsFromDonorWorksCorrectly) { std::vector<BSONObj> resultDocs; - auto insertBatchFn = [&](OperationContext* opCtx, BSONObjIterator docs) { - while (docs.more()) { - resultDocs.push_back(docs.next().Obj().getOwned()); + auto insertBatchFn = [&](OperationContext* opCtx, BSONObj docs) { + for (auto&& docToClone : docs) { + resultDocs.push_back(docToClone.Obj().getOwned()); } }; @@ -122,7 +122,7 @@ TEST_F(MigrationDestinationManagerTest, CloneDocumentsThrowsFetchErrors) { return fetchBatchResultBuilder.obj(); }; - auto insertBatchFn = [&](OperationContext* opCtx, BSONObjIterator docs) {}; + auto insertBatchFn = [&](OperationContext* opCtx, BSONObj docs) {}; ASSERT_THROWS_CODE_AND_WHAT(MigrationDestinationManager::cloneDocumentsFromDonor( operationContext(), insertBatchFn, fetchBatchFn), @@ -140,7 +140,7 @@ TEST_F(MigrationDestinationManagerTest, CloneDocumentsCatchesInsertErrors) { return fetchBatchResultBuilder.obj(); }; - auto insertBatchFn = [&](OperationContext* opCtx, BSONObjIterator docs) { + auto insertBatchFn = [&](OperationContext* opCtx, BSONObj docs) { uasserted(ErrorCodes::FailedToParse, "insertion error"); }; diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp index 035d21cbb82..6d522bcbd26 100644 --- a/src/mongo/db/s/migration_source_manager.cpp +++ b/src/mongo/db/s/migration_source_manager.cpp @@ -393,17 +393,40 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* txn << redact(status)}); } - // Do a best effort attempt to incrementally refresh the metadata. If this fails, just clear it - // up so that subsequent requests will try to do a full refresh. - ChunkVersion unusedShardVersion; - Status refreshStatus = - ShardingState::get(txn)->refreshMetadataNow(txn, getNss(), &unusedShardVersion); + // Because the CatalogCache's WithRefresh methods (on which forceShardFilteringMetadataRefresh + // depends) are not causally consistent, we need to perform up to two refresh rounds if refresh + // returns that the shard still owns the chunk + ChunkVersion collectionVersionAfterRefresh; + + for (int retriesLeft = 1;; --retriesLeft) { + ChunkVersion unusedShardVersion; + Status refreshStatus = + ShardingState::get(txn)->refreshMetadataNow(txn, getNss(), &unusedShardVersion); + + // If the refresh fails, there is no way to confirm whether the migration commit actually + // went through or not. Because of that, the collection's metadata is reset to UNSHARDED so + // that subsequent versioned requests will get StaleShardVersion and will retry the refresh. + if (!refreshStatus.isOK()) { + ScopedTransaction scopedXact(txn, MODE_IX); + AutoGetCollection autoColl(txn, getNss(), MODE_IX, MODE_X); + + CollectionShardingState::get(txn, getNss())->refreshMetadata(txn, nullptr); + + log() + << "Failed to refresh metadata after a failed commit attempt. Metadata was cleared " + "so it will get a full refresh when accessed again" + << causedBy(redact(refreshStatus)); - if (refreshStatus.isOK()) { - ScopedTransaction scopedXact(txn, MODE_IS); - AutoGetCollection autoColl(txn, getNss(), MODE_IS); + return {migrationCommitStatus.code(), + str::stream() << "Failed to refresh metadata after migration commit due to " + << refreshStatus.toString()}; + } - auto refreshedMetadata = CollectionShardingState::get(txn, getNss())->getMetadata(); + auto refreshedMetadata = [&] { + ScopedTransaction scopedXact(txn, MODE_IS); + AutoGetCollection autoColl(txn, getNss(), MODE_IS); + return CollectionShardingState::get(txn, getNss())->getMetadata(); + }(); if (!refreshedMetadata) { return {ErrorCodes::NamespaceNotSharded, @@ -412,32 +435,43 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* txn << migrationCommitStatus.toString()}; } - if (refreshedMetadata->keyBelongsToMe(_args.getMinKey())) { - // The chunk modification was not applied, so report the original error - return {migrationCommitStatus.code(), - str::stream() << "Chunk move was not successful due to " - << migrationCommitStatus.reason()}; + // If after a successful refresh the metadata indicates that the node still owns the chunk, + // we must do one more refresh in order to ensure that the previous refresh round didn't + // join an already active catalog cache refresh and missed its own commit + if (!refreshedMetadata->keyBelongsToMe(_args.getMinKey())) { + collectionVersionAfterRefresh = refreshedMetadata->getCollVersion(); + break; } - // Migration succeeded - log() << "Migration succeeded and updated collection version to " - << refreshedMetadata->getCollVersion(); - } else { - ScopedTransaction scopedXact(txn, MODE_IX); - AutoGetCollection autoColl(txn, getNss(), MODE_IX, MODE_X); - - CollectionShardingState::get(txn, getNss())->refreshMetadata(txn, nullptr); + if (retriesLeft) + continue; - log() << "Failed to refresh metadata after a failed commit attempt. Metadata was cleared " - "so it will get a full refresh when accessed again" - << causedBy(redact(refreshStatus)); + // This condition may only happen if the migration commit has failed for any reason + if (migrationCommitStatus.isOK()) { + severe() << "The migration commit succeeded, but the new chunk placement was not " + "reflected after metadata refresh, which is an indication of an " + "afterOpTime bug."; + severe() << "The current config server opTime is " << grid.configOpTime(); + severe() << "The commit response contained:"; + severe() << " metadata: " + << redact(commitChunkMigrationResponse.getValue().metadata.toString()); + severe() << " response: " + << redact(commitChunkMigrationResponse.getValue().response.toString()); + + fassertFailed(50878); + } - // We don't know whether migration succeeded or failed return {migrationCommitStatus.code(), - str::stream() << "Failed to refresh metadata after migration commit due to " - << refreshStatus.toString()}; + str::stream() << "Chunk move was not successful due to " + << migrationCommitStatus.reason()}; } + invariant(collectionVersionAfterRefresh.isSet()); + + // Migration succeeded + log() << "Migration succeeded and updated collection version to " + << collectionVersionAfterRefresh; + MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangBeforeLeavingCriticalSection); scopedGuard.Dismiss(); diff --git a/src/mongo/db/s/operation_sharding_state.cpp b/src/mongo/db/s/operation_sharding_state.cpp index 0f92bbd5492..2f0911bc9ac 100644 --- a/src/mongo/db/s/operation_sharding_state.cpp +++ b/src/mongo/db/s/operation_sharding_state.cpp @@ -40,7 +40,7 @@ const OperationContext::Decoration<OperationShardingState> shardingMetadataDecor OperationContext::declareDecoration<OperationShardingState>(); // Max time to wait for the migration critical section to complete -const Microseconds kMaxWaitForMigrationCriticalSection = Minutes(5); +const Milliseconds kMaxWaitForMigrationCriticalSection = Minutes(5); } // namespace @@ -109,7 +109,7 @@ bool OperationShardingState::waitForMigrationCriticalSectionSignal(OperationCont _migrationCriticalSectionSignal->waitFor( txn, txn->hasDeadline() - ? std::min(txn->getRemainingMaxTimeMicros(), kMaxWaitForMigrationCriticalSection) + ? std::min(txn->getRemainingMaxTimeMillis(), kMaxWaitForMigrationCriticalSection) : kMaxWaitForMigrationCriticalSection); _migrationCriticalSectionSignal = nullptr; return true; diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.cpp b/src/mongo/db/storage/bson_collection_catalog_entry.cpp index 7837e898b56..f0e5589a946 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/bson_collection_catalog_entry.cpp @@ -143,6 +143,16 @@ void BSONCollectionCatalogEntry::getAllIndexes(OperationContext* txn, } } +void BSONCollectionCatalogEntry::getReadyIndexes(OperationContext* txn, + std::vector<std::string>* names) const { + MetaData md = _getMetaData(txn); + + for (unsigned i = 0; i < md.indexes.size(); i++) { + if (md.indexes[i].ready) + names->push_back(md.indexes[i].spec["name"].String()); + } +} + bool BSONCollectionCatalogEntry::isIndexMultikey(OperationContext* txn, StringData indexName, MultikeyPaths* multikeyPaths) const { diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.h b/src/mongo/db/storage/bson_collection_catalog_entry.h index 83c2238fc17..c42908f5f8c 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.h +++ b/src/mongo/db/storage/bson_collection_catalog_entry.h @@ -58,6 +58,8 @@ public: virtual void getAllIndexes(OperationContext* txn, std::vector<std::string>* names) const; + virtual void getReadyIndexes(OperationContext* txn, std::vector<std::string>* names) const; + virtual bool isIndexMultikey(OperationContext* txn, StringData indexName, MultikeyPaths* multikeyPaths) const; diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp index a2421962853..abf43946697 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp @@ -106,6 +106,19 @@ void NamespaceDetailsCollectionCatalogEntry::getAllIndexes(OperationContext* txn } } +void NamespaceDetailsCollectionCatalogEntry::getReadyIndexes( + OperationContext* txn, std::vector<std::string>* names) const { + NamespaceDetails::IndexIterator i = _details->ii(true); + while (i.more()) { + const IndexDetails& id = i.next(); + const BSONObj obj = _indexRecordStore->dataFor(txn, id.info.toRecordId()).toBson(); + const char* idxName = obj.getStringField("name"); + if (isIndexReady(txn, StringData(idxName))) { + names->push_back(idxName); + } + } +} + bool NamespaceDetailsCollectionCatalogEntry::isIndexMultikey(OperationContext* txn, StringData idxName, MultikeyPaths* multikeyPaths) const { diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h index 0f8940be756..8d57825d0c3 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h @@ -67,6 +67,8 @@ public: BSONObj getIndexSpec(OperationContext* txn, StringData idxName) const final; + void getReadyIndexes(OperationContext* txn, std::vector<std::string>* names) const final; + bool isIndexMultikey(OperationContext* txn, StringData indexName, MultikeyPaths* multikeyPaths) const final; diff --git a/src/mongo/db/update_index_data.cpp b/src/mongo/db/update_index_data.cpp index 2b1144d40a3..ade76f0c50d 100644 --- a/src/mongo/db/update_index_data.cpp +++ b/src/mongo/db/update_index_data.cpp @@ -149,12 +149,35 @@ bool getCanonicalIndexField(StringData fullName, string* out) { while (j + 1 < fullName.size() && isdigit(fullName[j + 1])) j++; - if (j + 1 == fullName.size() || fullName[j + 1] == '.') { + if (j + 1 == fullName.size()) { // only digits found, skip forward i = j; modified = true; continue; } + + // Check for consecutive digits separated by a period. + if (fullName[j + 1] == '.') { + // Peek ahead to see if the next set of characters are also numeric. + size_t k = j + 2; + while (k < fullName.size() && isdigit(fullName[k])) { + k++; + } + + // The second set of digits may end at the end of the path or a '.'. + if (k == fullName.size() || fullName[k] == '.') { + // Found consecutive numerical path components. Since this implies a numeric + // field name, return the prefix as the canonical index field. This is meant to + // fix SERVER-37058. + modified = true; + break; + } + + // Only one numerical path component, skip forward. + i = j; + modified = true; + continue; + } } buf << c; diff --git a/src/mongo/db/update_index_data_test.cpp b/src/mongo/db/update_index_data_test.cpp index 167cb632ab1..f01f495ed1f 100644 --- a/src/mongo/db/update_index_data_test.cpp +++ b/src/mongo/db/update_index_data_test.cpp @@ -126,4 +126,26 @@ TEST(UpdateIndexDataTest, getCanonicalIndexField1) { ASSERT_FALSE(getCanonicalIndexField("a.", &x)); } + +TEST(UpdateIndexDataTest, CanonicalIndexFieldForConsecutiveDigits) { + std::string indexField; + + ASSERT_TRUE(getCanonicalIndexField("a.0.0", &indexField)); + ASSERT_EQ(indexField, "a"); + + ASSERT_TRUE(getCanonicalIndexField("a.55.01", &indexField)); + ASSERT_EQ(indexField, "a"); + + ASSERT_TRUE(getCanonicalIndexField("a.0.0.b.1", &indexField)); + ASSERT_EQ(indexField, "a"); + + ASSERT_TRUE(getCanonicalIndexField("a.0b.1", &indexField)); + ASSERT_EQ(indexField, "a.0b"); + + ASSERT_TRUE(getCanonicalIndexField("a.0.b.1.2", &indexField)); + ASSERT_EQ(indexField, "a.b"); + + ASSERT_TRUE(getCanonicalIndexField("a.0.11b", &indexField)); + ASSERT_EQ(indexField, "a.11b"); +} } diff --git a/src/mongo/executor/network_interface_asio_auth.cpp b/src/mongo/executor/network_interface_asio_auth.cpp index 571a1be1f81..f5f9ec19b68 100644 --- a/src/mongo/executor/network_interface_asio_auth.cpp +++ b/src/mongo/executor/network_interface_asio_auth.cpp @@ -184,7 +184,7 @@ void NetworkInterfaceASIO::_authenticate(AsyncOp* op) { std::string clientName; #ifdef MONGO_CONFIG_SSL if (getSSLManager()) { - clientName = getSSLManager()->getSSLConfiguration().clientSubjectName; + clientName = getSSLManager()->getSSLConfiguration().clientSubjectName.toString(); } #endif diff --git a/src/mongo/gotools/common.yml b/src/mongo/gotools/common.yml index ee3c741a010..44c64b6870d 100644 --- a/src/mongo/gotools/common.yml +++ b/src/mongo/gotools/common.yml @@ -1429,21 +1429,21 @@ buildvariants: ####################################### - name: amazonlinux64 - display_name: Amazon Linux 64 (Go 1.8) + display_name: Amazon Linux 64 (Go 1.10) run_on: - linux-64-amzn-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist - name: amazon2 - display_name: Amazon Linux 64 v2 (Go 1.8) + display_name: Amazon Linux 64 v2 (Go 1.10) run_on: - amazon2-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1463,11 +1463,11 @@ buildvariants: - name: dist - name: debian81 - display_name: Debian 8.1 (Go 1.8) + display_name: Debian 8.1 (Go 1.10) run_on: - debian81-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1477,7 +1477,7 @@ buildvariants: ####################################### - name: macOS-1012 - display_name: MacOS 10.12 (Go 1.8) + display_name: MacOS 10.12 (Go 1.10) run_on: - macos-1012 expansions: @@ -1486,11 +1486,11 @@ buildvariants: mongo_os: "osx" arch: "osx/x86_64" excludes: requires_many_files - gorootvars: 'PATH="/usr/local/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/usr/local/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' tasks: *macos_1012_tasks - name: macOS-1012-ssl - display_name: MacOS 10.12 SSL (Go 1.8) + display_name: MacOS 10.12 SSL (Go 1.10) run_on: - macos-1012 expansions: @@ -1501,7 +1501,7 @@ buildvariants: arch: "osx/x86_64" build_tags: "ssl openssl_pre_1.0" excludes: requires_many_files - gorootvars: 'PATH="/usr/local/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/usr/local/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' tasks: *macos_1012_ssl_tasks ####################################### @@ -1509,21 +1509,21 @@ buildvariants: ####################################### - name: rhel62 - display_name: RHEL 6.2 (Go 1.8) + display_name: RHEL 6.2 (Go 1.10) run_on: - rhel62-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist - name: rhel70 - display_name: RHEL 7.0 (Go 1.8) + display_name: RHEL 7.0 (Go 1.10) run_on: - rhel70 expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1533,11 +1533,11 @@ buildvariants: ####################################### - name: suse12 - display_name: SUSE 12 (Go 1.8) + display_name: SUSE 12 (Go 1.10) run_on: - suse12-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1547,7 +1547,7 @@ buildvariants: ####################################### - name: ubuntu1404 - display_name: Ubuntu 14.04 (Go 1.8) + display_name: Ubuntu 14.04 (Go 1.10) run_on: - ubuntu1404-test expansions: @@ -1555,15 +1555,15 @@ buildvariants: <<: *mongo_default_startup_args mongo_os: "ubuntu1404" mongo_edition: "targeted" - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' - build_tags: "ssl" + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' + build_tags: "sasl ssl" arch: "linux/x86_64" integration_test_args: integration resmoke_args: --jobs $(grep -c ^processor /proc/cpuinfo) tasks: *ubuntu1404_tasks - name: ubuntu1404-ssl - display_name: Ubuntu 14.04 SSL (Go 1.8) + display_name: Ubuntu 14.04 SSL (Go 1.10) run_on: - ubuntu1404-test expansions: @@ -1571,7 +1571,7 @@ buildvariants: <<: *mongo_ssl_startup_args mongo_os: "ubuntu1404" mongo_edition: "enterprise" - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" edition: ssl arch: "linux/x86_64" @@ -1590,7 +1590,7 @@ buildvariants: <<: *mongo_default_startup_args mongo_os: "ubuntu1404" mongo_edition: "enterprise" - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "ssl sasl" smoke_use_ssl: --use-ssl resmoke_use_ssl: _ssl @@ -1602,11 +1602,11 @@ buildvariants: tasks: *ubuntu1404_enterprise_tasks - name: ubuntu1604 - display_name: Ubuntu 16.04 (Go 1.8) + display_name: Ubuntu 16.04 (Go 1.10) run_on: - ubuntu1604-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1616,7 +1616,7 @@ buildvariants: ####################################### - name: windows-64 - display_name: Windows 64-bit (Go 1.8) + display_name: Windows 64-bit (Go 1.10) run_on: - windows-64-vs2013-test expansions: @@ -1629,11 +1629,11 @@ buildvariants: arch: "win32/x86_64" preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' " integration_test_args: "integration" - gorootvars: 'PATH="/cygdrive/c/go1.8/go/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/go1.8/go"' + gorootvars: 'PATH="/cygdrive/c/golang/go1.10/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/golang/go1.10"' tasks: *windows_64_tasks - name: windows-64-ssl - display_name: Windows 64-bit SSL (Go 1.8) + display_name: Windows 64-bit SSL (Go 1.10) run_on: - windows-64-vs2013-compile expansions: @@ -1649,13 +1649,13 @@ buildvariants: multiversion_override: "2.6" extension: .exe arch: "win32/x86_64" - gorootvars: 'PATH="/cygdrive/c/go1.8/go/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/go1.8/go"' + gorootvars: 'PATH="/cygdrive/c/golang/go1.10/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/golang/go1.10"' preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' " integration_test_args: "integration,ssl" tasks: *windows_64_ssl_tasks - name: windows-64-enterprise - display_name: Windows 64-bit Enterprise (Go 1.8) + display_name: Windows 64-bit Enterprise (Go 1.10) run_on: - windows-64-vs2013-compile expansions: @@ -1672,7 +1672,7 @@ buildvariants: edition: enterprise extension: .exe arch: "win32/x86_64" - gorootvars: 'PATH="/cygdrive/c/go1.8/go/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/go1.8/go"' + gorootvars: 'PATH="/cygdrive/c/golang/go1.10/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/golang/go1.10"' preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' " integration_test_args: "integration" tasks: *windows_64_enterprise_tasks @@ -1682,7 +1682,7 @@ buildvariants: ####################################### - name: ubuntu1604-arm64 - display_name: ZAP ARM64 Ubuntu 16.04 SSL (gccgo 1.4) + display_name: ZAP ARM64 Ubuntu 16.04 SSL (Go 1.10) run_on: - ubuntu1604-arm64-small stepback: false @@ -1693,10 +1693,9 @@ buildvariants: mongo_os: "ubuntu1604" mongo_edition: "targeted" mongo_arch: "arm64" - args: -gccgoflags "$(pkg-config --libs --cflags libcrypto libssl)" build_tags: "ssl" resmoke_use_ssl: _ssl - gorootvars: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH" + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/aarch64-mongodb-linux-gcc' excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30 resmoke_args: -j 2 multiversion_override: "skip" @@ -1710,7 +1709,7 @@ buildvariants: ####################################### - name: rhel71-ppc64le-enterprise - display_name: ZAP PPC64LE RHEL 7.1 Enterprise (Go 1.8) + display_name: ZAP PPC64LE RHEL 7.1 Enterprise (Go 1.10) run_on: - rhel71-power8-test stepback: false @@ -1725,7 +1724,7 @@ buildvariants: #args: ... libsasl2; build_tags "sasl ssl" build_tags: 'ssl' resmoke_use_ssl: _ssl - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' resmoke_args: -j 4 excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30 multiversion_override: "skip" @@ -1736,14 +1735,14 @@ buildvariants: tasks: *rhel71_enterprise_tasks - name: ubuntu1604-ppc64le-enterprise - display_name: ZAP PPC64LE Ubuntu 16.04 Enterprise (Go 1.8) + display_name: ZAP PPC64LE Ubuntu 16.04 Enterprise (Go 1.10) run_on: - ubuntu1604-power8-test stepback: false batchtime: 10080 # weekly expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' - build_tags: 'ssl' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' + build_tags: 'ssl sasl' tasks: - name: dist @@ -1752,7 +1751,7 @@ buildvariants: ####################################### - name: rhel67-s390x-enterprise - display_name: ZAP s390x RHEL 6.7 Enterprise (Go 1.8) + display_name: ZAP s390x RHEL 6.7 Enterprise (Go 1.10) run_on: - rhel67-zseries-test stepback: false @@ -1777,7 +1776,7 @@ buildvariants: mongo_arch: "s390x" build_tags: "sasl ssl" resmoke_use_ssl: _ssl - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' excludes: requires_mmap_available,requires_mongo_24,requires_mongo_26,requires_mongo_30 resmoke_args: -j 2 multiversion_override: "skip" @@ -1800,13 +1799,13 @@ buildvariants: - name: dist - name: ubuntu1604-s390x-enterprise - display_name: ZAP s390x Ubuntu 16.04 Enterprise (Go 1.8) + display_name: ZAP s390x Ubuntu 16.04 Enterprise (Go 1.10) run_on: - ubuntu1604-zseries-small stepback: false batchtime: 10080 # weekly expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' build_tags: "sasl ssl" tasks: - name: dist @@ -1818,7 +1817,7 @@ buildvariants: - name: ubuntu-race stepback: false batchtime: 1440 # daily - display_name: z Race Detector Ubuntu 14.04 (Go 1.8) + display_name: z Race Detector Ubuntu 14.04 (Go 1.10) run_on: - ubuntu1404-test expansions: @@ -1826,8 +1825,8 @@ buildvariants: <<: *mongo_default_startup_args mongo_os: "ubuntu1404" mongo_edition: "enterprise" - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' - build_tags: "ssl" + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' + build_tags: "sasl ssl" arch: "linux/x86_64" args: "-race" excludes: requires_large_ram diff --git a/src/mongo/gotools/import.data b/src/mongo/gotools/import.data index d1d91264483..4b4c75c79f0 100644 --- a/src/mongo/gotools/import.data +++ b/src/mongo/gotools/import.data @@ -1,5 +1,5 @@ { - "commit": "4c5314b404c2d7aac7ceb50133faa3ac4fc3d2ea", + "commit": "38376e791d2c264b377ba3115344c860c146e0b2", "github": "mongodb/mongo-tools.git", "vendor": "tools", "branch": "v3.4" diff --git a/src/mongo/gotools/mongoreplay/stat_format.go b/src/mongo/gotools/mongoreplay/stat_format.go index ad64ef7bd95..613087fbc43 100644 --- a/src/mongo/gotools/mongoreplay/stat_format.go +++ b/src/mongo/gotools/mongoreplay/stat_format.go @@ -28,10 +28,10 @@ type OpStat struct { Ns string `json:"ns,omitempty"` // Data represents the payload of the request operation. - RequestData interface{} `json:"request_data, omitempty"` + RequestData interface{} `json:"request_data,omitempty"` // Data represents the payload of the reply operation. - ReplyData interface{} `json:"reply_data, omitempty"` + ReplyData interface{} `json:"reply_data,omitempty"` // NumReturned is the number of documents that were fetched as a result of this operation. NumReturned int `json:"nreturned,omitempty"` @@ -71,7 +71,7 @@ type OpStat struct { // RequestID is the ID of the mongodb operation as taken from the header. // The RequestID for a request operation is the same as the ResponseID for // the corresponding reply, so this field will be the same for request/reply pairs. - RequestID int32 `json:"request_id, omitempty"` + RequestID int32 `json:"request_id,omitempty"` } // jsonGet retrieves serialized json req/res via the channel-like arg; diff --git a/src/mongo/gotools/mongorestore/oplog.go b/src/mongo/gotools/mongorestore/oplog.go index 68e24d20b8f..010f16f410e 100644 --- a/src/mongo/gotools/mongorestore/oplog.go +++ b/src/mongo/gotools/mongorestore/oplog.go @@ -93,6 +93,9 @@ func (restore *MongoRestore) RestoreOplog() error { } log.Logvf(log.Info, "applied %v ops", totalOps) + if err := bsonSource.Err(); err != nil { + return fmt.Errorf("error reading oplog bson input: %v", err) + } return nil } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.evergreen/config.yml b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.evergreen/config.yml new file mode 100644 index 00000000000..b8bbabba9a9 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.evergreen/config.yml @@ -0,0 +1,377 @@ +# default command type +command_type: system + +# run the same task in the previous revision if the current task fails +stepback: true + +functions: + + "set shell vars": + - command: shell.exec + params: + script: | + set -o errexit + set -o xtrace + export RAWGOPATH="$(pwd)/gopath" + export GOPATH="$RAWGOPATH" + if [ "Windows_NT" = "$OS" ]; then + set -o igncr + export GOPATH=$(echo $GOPATH | sed -e 's|/cygdrive/c|c:|') + fi + cat <<EOT > expansion.yml + rawgopath: $RAWGOPATH + repopath: $RAWGOPATH/src/github.com/10gen/openssl + prepare_shell: | + export GOPATH="$GOPATH" + set -o errexit + set -o xtrace + EOT + cat expansion.yml + exit 0 + - command: expansions.update + params: + file: expansion.yml + + "setup gopath" : + - command: shell.exec + params: + silent: false + script: | + ${prepare_shell} + ${gorootvars} go get github.com/spacemonkeygo/spacelog + exit 0 + + "fetch source" : + - command: git.get_project + params: + directory: src + - command: shell.exec + params: + script: | + ${prepare_shell} + mkdir -p $(dirname "${repopath}") + mv src "${repopath}" + exit 0 + + "go build" : + - command: shell.exec + type: test + params: + script: | + ${prepare_shell} + cd ${repopath} + ${gorootvars} go build ${args} -v -x -tags '${build_tags}' + exit 0 + + "go test" : + - command: shell.exec + type: test + params: + script: | + ${prepare_shell} + cd ${repopath} + ${gorootvars} go test ${args} -v -x -tags '${build_tags}' + exit 0 + +post: + - command: shell.exec + params: + silent: true + script: | + ${prepare_shell} + rm -rf "${rawgopath}" + exit 0 + +tasks: + +- name: "build" + commands: + - func: "set shell vars" + - func: "setup gopath" + - func: "fetch source" + - func: "go build" + +- name: "test" + depends_on: + - name: "build" + commands: + - func: "set shell vars" + - func: "setup gopath" + - func: "fetch source" + - func: "go test" + +buildvariants: + +####################################### +# Amazon Buildvariants # +####################################### + +- name: amazonlinux64 + display_name: Amazon Linux 64 (Go 1.8) + run_on: + - linux-64-amzn-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: amazon2 + display_name: Amazon Linux 64 v2 (Go 1.8) + run_on: + - amazon2-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Debian Buildvariants # +####################################### + +- name: debian71 + display_name: Debian 7.1 (Go 1.8) + run_on: + - debian71-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: debian81 + display_name: Debian 8.1 (Go 1.8) + run_on: + - debian81-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: debian92 + display_name: Debian 9.2 (Go 1.8) + run_on: + - debian92-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# macOS Buildvariant # +####################################### + +- name: macOS-1012 + display_name: MacOS 10.12 (Go 1.8) + run_on: + - macos-1012 + expansions: + gorootvars: 'PATH="/usr/local/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/usr/local/go1.8/go CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.10 CGO_LDFLAGS=-mmacosx-version-min=10.10' + build_tags: "openssl_pre_1.0" + tasks: + - name: build + - name: test + +####################################### +# RHEL Buildvariants # +####################################### + +- name: rhel62 + display_name: RHEL 6.2 (Go 1.8) + run_on: + - rhel62-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: rhel70 + display_name: RHEL 7.0 (Go 1.8) + run_on: + - rhel70 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# SUSE Buildvariants # +####################################### + +- name: suse11 + display_name: SUSE 11 (Go 1.8) + run_on: + - suse11-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "openssl_pre_1.0" + tasks: + - name: build + - name: test + +- name: suse12 + display_name: SUSE 12 (Go 1.8) + run_on: + - suse12-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Ubuntu Buildvariants # +####################################### + +- name: ubuntu1404 + display_name: Ubuntu 14.04 (Go 1.8) + run_on: + - ubuntu1404-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: ubuntu1604 + display_name: Ubuntu 16.04 (Go 1.8) + run_on: + - ubuntu1604-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Windows Buildvariants # +####################################### + +- name: windows-64 + display_name: Windows 64-bit (Go 1.8) + run_on: + - windows-64-vs2015-test + expansions: + gorootvars: 'PATH="/cygdrive/c/go1.8/go/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/go1.8/go"' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# ARM Buildvariants # +####################################### + +- name: ubuntu1604-arm64-go1.8 + display_name: ZAP ARM64 Ubuntu 16.04 SSL (Go 1.8) + run_on: + - ubuntu1604-arm64-small + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/aarch64-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Power Buildvariants # +####################################### + +- name: rhel71-ppc64le-enterprise-go1.8 + display_name: ZAP PPC64LE RHEL 7.1 Enterprise (Go 1.8) + run_on: + - rhel71-power8-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +- name: ubuntu1604-ppc64le-enterprise-go1.8 + display_name: ZAP PPC64LE Ubuntu 16.04 Enterprise (Go 1.8) + run_on: + - ubuntu1604-power8-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Z (s390x) Buildvariants # +####################################### + +- name: rhel67-s390x-enterprise-go1.8 + display_name: ZAP s390x RHEL 6.7 Enterprise (Go 1.8) + run_on: + - rhel67-zseries-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +- name: rhel72-s390x-enterprise-go1.8 + display_name: ZAP s390x RHEL 7.2 Enterprise (Go 1.8) + run_on: + - rhel72-zseries-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +- name: suse12-s390x-enterprise-go1.8 + display_name: ZAP s390x SUSE 12 Enterprise (Go 1.8) + run_on: + - suse12-zseries-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +- name: ubuntu1604-s390x-enterprise-go1.8 + display_name: ZAP s390x Ubuntu 16.04 Enterprise (Go 1.8) + run_on: + - ubuntu1604-zseries-small + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.gitignore b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.gitignore new file mode 100644 index 00000000000..805d350b7e5 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.gitignore @@ -0,0 +1 @@ +openssl.test diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/AUTHORS b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/AUTHORS new file mode 100644 index 00000000000..bc88546999e --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/AUTHORS @@ -0,0 +1,23 @@ +Andrew Brampton <github@bramp.net> +Anton Baklanov <antonbaklanov@gmail.com> +Carlos MartÃn Nieto <cmn@dwim.me> +Charles Strahan <charles@cstrahan.com> +Christopher Dudley <chris@github.chrisdudley.xyz> +Christopher Fredericks <cfredmakecode@gmail.com> +Colin Misare +dequis <dx@dxzone.com.ar> +Gabriel Russell <gabriel.russell@mongodb.com> +Giulio <programmatore@ditieri.it> +Jakob Unterwurzacher <jakobunt@gmail.com> +Juuso Haavisto <juuso@mail.com> +kujenga <ataylor0123@gmail.com> +MongoDB, Inc. +Phus Lu <phuslu@hotmail.com> +Russ Egan <russ@safemonk.com> +Ryan Hileman <lunixbochs@gmail.com> +Scott J. Goldman <scottjg@github.com> +Scott Kidder <skidder@brightcove.com> +Space Monkey, Inc <hello@spacemonkey.com> +Stephen Gallagher <sgallagh@redhat.com> +Viacheslav Biriukov <v.v.biriukov@gmail.com> +Zack Owens <zowens2009@gmail.com> diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/README.md b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/README.md index 6bd3383a0e8..2785366f5e1 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/README.md +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/README.md @@ -4,7 +4,7 @@ Please see http://godoc.org/github.com/spacemonkeygo/openssl for more info ### License -Copyright (C) 2014 Space Monkey, Inc. +Copyright (C) 2017. See AUTHORS. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,9 +18,33 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -### Using on Windows -1. Install [mingw-w64](http://mingw-w64.sourceforge.net/) -2. Install [pkg-config-lite](http://sourceforge.net/projects/pkgconfiglite) -3. Build (or install precompiled) openssl for mingw32-w64 -4. Set __PKG\_CONFIG\_PATH__ to the directory containing openssl.pc - (i.e. c:\mingw64\mingw64\lib\pkgconfig) +### Installing on a Unix-ish system with pkg-config + +1. (If necessary) install the openssl C library with a package manager + that provides an openssl.pc file OR install openssl manually and create + an openssl.pc file. + +2. Ensure that `pkg-config --cflags --libs openssl` finds your openssl + library. If it doesn't, try setting `PKG_CONFIG_PATH` to the directory + containing your openssl.pc file. E.g. for darwin: with MacPorts, + `PKG_CONFIG_PATH=/opt/local/lib/pkgconfig` or for Homebrew, + `PKG_CONFIG_PATH=/usr/local/Cellar/openssl/1.0.2l/lib/pkgconfig` + +### Installing on a Unix-ish system without pkg-config + +1. (If necessary) install the openssl C library in your customary way + +2. Set the `CGO_CPP_FLAGS`, `CGO_CFLAGS` and `CGO_LDFLAGS` as necessary to + provide `-I`, `-L` and other options to the compiler. E.g. on darwin, + MongoDB's darwin build servers use the native libssl, but provide the + missing headers in a custom directory, so it the build hosts set + `CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include` + +### Installing on Windows + +1. Install [mingw-w64](http://mingw-w64.sourceforge.net/) and add it to + your `PATH` + +2. Install the C openssl into `C:\openssl`. (Unfortunately, this is still + hard-coded.) You should have directories like `C:\openssl\include` and + `C:\openssl\bin`. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/bio.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/bio.go index 8d0da8998eb..9fe32aa8032 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/bio.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/bio.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,56 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <string.h> -#include <openssl/bio.h> - -extern int cbioNew(BIO *b); -static int cbioFree(BIO *b) { - return 1; -} - -extern int writeBioWrite(BIO *b, char *buf, int size); -extern long writeBioCtrl(BIO *b, int cmd, long arg1, void *arg2); -static int writeBioPuts(BIO *b, const char *str) { - return writeBioWrite(b, (char*)str, (int)strlen(str)); -} - -extern int readBioRead(BIO *b, char *buf, int size); -extern long readBioCtrl(BIO *b, int cmd, long arg1, void *arg2); - -static BIO_METHOD writeBioMethod = { - BIO_TYPE_SOURCE_SINK, - "Go Write BIO", - (int (*)(BIO *, const char *, int))writeBioWrite, - NULL, - writeBioPuts, - NULL, - writeBioCtrl, - cbioNew, - cbioFree, - NULL}; - -static BIO_METHOD* BIO_s_writeBio() { return &writeBioMethod; } - -static BIO_METHOD readBioMethod = { - BIO_TYPE_SOURCE_SINK, - "Go Read BIO", - NULL, - readBioRead, - NULL, - NULL, - readBioCtrl, - cbioNew, - cbioFree, - NULL}; - -static BIO_METHOD* BIO_s_readBio() { return &readBioMethod; } -*/ +// #include "shim.h" import "C" import ( @@ -89,16 +42,6 @@ func nonCopyCString(data *C.char, size C.int) []byte { return nonCopyGoBytes(uintptr(unsafe.Pointer(data)), int(size)) } -//export cbioNew -func cbioNew(b *C.BIO) C.int { - b.shutdown = 1 - b.init = 1 - b.num = -1 - b.ptr = nil - b.flags = 0 - return 1 -} - var writeBioMapping = newMapping() type writeBio struct { @@ -109,21 +52,20 @@ type writeBio struct { } func loadWritePtr(b *C.BIO) *writeBio { - return (*writeBio)(writeBioMapping.Get(token(b.ptr))) + t := token(C.X_BIO_get_data(b)) + return (*writeBio)(writeBioMapping.Get(t)) } func bioClearRetryFlags(b *C.BIO) { - // from BIO_clear_retry_flags and BIO_clear_flags - b.flags &= ^(C.BIO_FLAGS_RWS | C.BIO_FLAGS_SHOULD_RETRY) + C.X_BIO_clear_flags(b, C.BIO_FLAGS_RWS|C.BIO_FLAGS_SHOULD_RETRY) } func bioSetRetryRead(b *C.BIO) { - // from BIO_set_retry_read and BIO_set_flags - b.flags |= (C.BIO_FLAGS_READ | C.BIO_FLAGS_SHOULD_RETRY) + C.X_BIO_set_flags(b, C.BIO_FLAGS_READ|C.BIO_FLAGS_SHOULD_RETRY) } -//export writeBioWrite -func writeBioWrite(b *C.BIO, data *C.char, size C.int) (rc C.int) { +//export go_write_bio_write +func go_write_bio_write(b *C.BIO, data *C.char, size C.int) (rc C.int) { defer func() { if err := recover(); err != nil { logger.Critf("openssl: writeBioWrite panic'd: %v", err) @@ -141,8 +83,8 @@ func writeBioWrite(b *C.BIO, data *C.char, size C.int) (rc C.int) { return size } -//export writeBioCtrl -func writeBioCtrl(b *C.BIO, cmd C.int, arg1 C.long, arg2 unsafe.Pointer) ( +//export go_write_bio_ctrl +func go_write_bio_ctrl(b *C.BIO, cmd C.int, arg1 C.long, arg2 unsafe.Pointer) ( rc C.long) { defer func() { if err := recover(); err != nil { @@ -197,15 +139,15 @@ func (b *writeBio) WriteTo(w io.Writer) (rv int64, err error) { func (self *writeBio) Disconnect(b *C.BIO) { if loadWritePtr(b) == self { - writeBioMapping.Del(token(b.ptr)) - b.ptr = nil + writeBioMapping.Del(token(C.X_BIO_get_data(b))) + C.X_BIO_set_data(b, nil) } } func (b *writeBio) MakeCBIO() *C.BIO { - rv := C.BIO_new(C.BIO_s_writeBio()) + rv := C.X_BIO_new_write_bio() token := writeBioMapping.Add(unsafe.Pointer(b)) - rv.ptr = unsafe.Pointer(token) + C.X_BIO_set_data(rv, unsafe.Pointer(token)) return rv } @@ -220,14 +162,14 @@ type readBio struct { } func loadReadPtr(b *C.BIO) *readBio { - return (*readBio)(readBioMapping.Get(token(b.ptr))) + return (*readBio)(readBioMapping.Get(token(C.X_BIO_get_data(b)))) } -//export readBioRead -func readBioRead(b *C.BIO, data *C.char, size C.int) (rc C.int) { +//export go_read_bio_read +func go_read_bio_read(b *C.BIO, data *C.char, size C.int) (rc C.int) { defer func() { if err := recover(); err != nil { - logger.Critf("openssl: readBioRead panic'd: %v", err) + logger.Critf("openssl: go_read_bio_read panic'd: %v", err) rc = -1 } }() @@ -256,8 +198,8 @@ func readBioRead(b *C.BIO, data *C.char, size C.int) (rc C.int) { return C.int(n) } -//export readBioCtrl -func readBioCtrl(b *C.BIO, cmd C.int, arg1 C.long, arg2 unsafe.Pointer) ( +//export go_read_bio_ctrl +func go_read_bio_ctrl(b *C.BIO, cmd C.int, arg1 C.long, arg2 unsafe.Pointer) ( rc C.long) { defer func() { @@ -316,16 +258,16 @@ func (b *readBio) ReadFromOnce(r io.Reader) (n int, err error) { } func (b *readBio) MakeCBIO() *C.BIO { - rv := C.BIO_new(C.BIO_s_readBio()) + rv := C.X_BIO_new_read_bio() token := readBioMapping.Add(unsafe.Pointer(b)) - rv.ptr = unsafe.Pointer(token) + C.X_BIO_set_data(rv, unsafe.Pointer(token)) return rv } func (self *readBio) Disconnect(b *C.BIO) { if loadReadPtr(b) == self { - readBioMapping.Del(token(b.ptr)) - b.ptr = nil + readBioMapping.Del(token(C.X_BIO_get_data(b))) + C.X_BIO_set_data(b, nil) } } @@ -343,7 +285,7 @@ func (b *anyBio) Read(buf []byte) (n int, err error) { if len(buf) == 0 { return 0, nil } - n = int(C.BIO_read((*C.BIO)(b), unsafe.Pointer(&buf[0]), C.int(len(buf)))) + n = int(C.X_BIO_read((*C.BIO)(b), unsafe.Pointer(&buf[0]), C.int(len(buf)))) if n <= 0 { return 0, io.EOF } @@ -354,7 +296,7 @@ func (b *anyBio) Write(buf []byte) (written int, err error) { if len(buf) == 0 { return 0, nil } - n := int(C.BIO_write((*C.BIO)(b), unsafe.Pointer(&buf[0]), + n := int(C.X_BIO_write((*C.BIO)(b), unsafe.Pointer(&buf[0]), C.int(len(buf)))) if n != len(buf) { return n, errors.New("BIO write failed") diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build.go index f71e285639a..d286163ffcb 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo +// +build !openssl_static package openssl -// #cgo linux pkg-config: openssl -// #cgo windows CFLAGS: -DWIN32_LEAN_AND_MEAN -// #cgo windows LDFLAGS: -lcrypt32 -// #cgo darwin CFLAGS: -Wno-deprecated-declarations -I/usr/include -I/usr/local/opt/openssl/include -// #cgo darwin LDFLAGS: -L/usr/local/opt/openssl/lib -lssl -lcrypto -framework CoreFoundation -framework Foundation -framework Security +// #cgo linux darwin pkg-config: openssl +// #cgo CFLAGS: -Wno-deprecated-declarations +// #cgo windows CFLAGS: -DWIN32_LEAN_AND_MEAN -I"c:/openssl/include" +// #cgo windows LDFLAGS: -lssleay32 -llibeay32 -lcrypt32 -L "c:/openssl/bin" +// #cgo darwin LDFLAGS: -framework CoreFoundation -framework Foundation -framework Security import "C" diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build_static.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build_static.go new file mode 100644 index 00000000000..1450d52e1a9 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build_static.go @@ -0,0 +1,24 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build openssl_static + +package openssl + +// #cgo linux windows darwin pkg-config: --static libssl libcrypto +// #cgo CFLAGS: -Wno-deprecated-declarations +// #cgo windows CFLAGS: -DWIN32_LEAN_AND_MEAN -I"c:/openssl/include" +// #cgo windows LDFLAGS: -lssleay32 -llibeay32 -lcrypt32 -L "c:/openssl/bin" +// #cgo darwin LDFLAGS: -framework CoreFoundation -framework Foundation -framework Security +import "C" diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert.go index 61637c649fa..d3df63507e3 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,16 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -// #include <openssl/conf.h> -// #include <openssl/ssl.h> -// #include <openssl/x509v3.h> -// -// void OPENSSL_free_not_a_macro(void *ref) { OPENSSL_free(ref); } -// +// #include "shim.h" import "C" import ( @@ -229,7 +222,7 @@ func (c *Certificate) SetSerial(serial *big.Int) error { // SetIssueDate sets the certificate issue date relative to the current time. func (c *Certificate) SetIssueDate(when time.Duration) error { offset := C.long(when / time.Second) - result := C.X509_gmtime_adj(c.x.cert_info.validity.notBefore, offset) + result := C.X509_gmtime_adj(C.X_X509_get0_notBefore(c.x), offset) if result == nil { return errors.New("failed to set issue date") } @@ -239,7 +232,7 @@ func (c *Certificate) SetIssueDate(when time.Duration) error { // SetExpireDate sets the certificate issue date relative to the current time. func (c *Certificate) SetExpireDate(when time.Duration) error { offset := C.long(when / time.Second) - result := C.X509_gmtime_adj(c.x.cert_info.validity.notAfter, offset) + result := C.X509_gmtime_adj(C.X_X509_get0_notAfter(c.x), offset) if result == nil { return errors.New("failed to set expire date") } @@ -270,37 +263,41 @@ func (c *Certificate) Sign(privKey PrivateKey, digest EVP_MD) error { } func (c *Certificate) insecureSign(privKey PrivateKey, digest EVP_MD) error { - var md *C.EVP_MD + var md *C.EVP_MD = getDigestFunction(digest) + if C.X509_sign(c.x, privKey.evpPKey(), md) <= 0 { + return errors.New("failed to sign certificate") + } + return nil +} + +func getDigestFunction(digest EVP_MD) (md *C.EVP_MD) { switch digest { // please don't use these digest functions case EVP_NULL: - md = C.EVP_md_null() + md = C.X_EVP_md_null() case EVP_MD5: - md = C.EVP_md5() + md = C.X_EVP_md5() case EVP_SHA: - md = C.EVP_sha() + md = C.X_EVP_sha() case EVP_SHA1: - md = C.EVP_sha1() + md = C.X_EVP_sha1() case EVP_DSS: - md = C.EVP_dss() + md = C.X_EVP_dss() case EVP_DSS1: - md = C.EVP_dss1() + md = C.X_EVP_dss1() case EVP_RIPEMD160: - md = C.EVP_ripemd160() + md = C.X_EVP_ripemd160() case EVP_SHA224: - md = C.EVP_sha224() + md = C.X_EVP_sha224() // you actually want one of these case EVP_SHA256: - md = C.EVP_sha256() + md = C.X_EVP_sha256() case EVP_SHA384: - md = C.EVP_sha384() + md = C.X_EVP_sha384() case EVP_SHA512: - md = C.EVP_sha512() - } - if C.X509_sign(c.x, privKey.evpPKey(), md) <= 0 { - return errors.New("failed to sign certificate") + md = C.X_EVP_sha512() } - return nil + return md } // Add an extension to a certificate. @@ -388,7 +385,7 @@ func (c *Certificate) GetSerialNumberHex() (serial string) { hex := C.BN_bn2hex(bignum) serial = C.GoString(hex) C.BN_free(bignum) - C.OPENSSL_free_not_a_macro(unsafe.Pointer(hex)) + C.X_OPENSSL_free(unsafe.Pointer(hex)) return } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert_test.go index c32883ba4eb..96083260507 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Ryan Hileman +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers.go index 12662707f54..e4f5771f8dc 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,43 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -// #include <openssl/evp.h> -// -// int EVP_CIPHER_block_size_not_a_macro(EVP_CIPHER *c) { -// return EVP_CIPHER_block_size(c); -// } -// -// int EVP_CIPHER_key_length_not_a_macro(EVP_CIPHER *c) { -// return EVP_CIPHER_key_length(c); -// } -// -// int EVP_CIPHER_iv_length_not_a_macro(EVP_CIPHER *c) { -// return EVP_CIPHER_iv_length(c); -// } -// -// int EVP_CIPHER_nid_not_a_macro(EVP_CIPHER *c) { -// return EVP_CIPHER_nid(c); -// } -// -// int EVP_CIPHER_CTX_block_size_not_a_macro(EVP_CIPHER_CTX *ctx) { -// return EVP_CIPHER_CTX_block_size(ctx); -// } -// -// int EVP_CIPHER_CTX_key_length_not_a_macro(EVP_CIPHER_CTX *ctx) { -// return EVP_CIPHER_CTX_key_length(ctx); -// } -// -// int EVP_CIPHER_CTX_iv_length_not_a_macro(EVP_CIPHER_CTX *ctx) { -// return EVP_CIPHER_CTX_iv_length(ctx); -// } -// -// const EVP_CIPHER *EVP_CIPHER_CTX_cipher_not_a_macro(EVP_CIPHER_CTX *ctx) { -// return EVP_CIPHER_CTX_cipher(ctx); -// } +// #include "shim.h" import "C" import ( @@ -74,7 +40,7 @@ type Cipher struct { } func (c *Cipher) Nid() NID { - return NID(C.EVP_CIPHER_nid_not_a_macro(c.ptr)) + return NID(C.X_EVP_CIPHER_nid(c.ptr)) } func (c *Cipher) ShortName() (string, error) { @@ -82,15 +48,15 @@ func (c *Cipher) ShortName() (string, error) { } func (c *Cipher) BlockSize() int { - return int(C.EVP_CIPHER_block_size_not_a_macro(c.ptr)) + return int(C.X_EVP_CIPHER_block_size(c.ptr)) } func (c *Cipher) KeySize() int { - return int(C.EVP_CIPHER_key_length_not_a_macro(c.ptr)) + return int(C.X_EVP_CIPHER_key_length(c.ptr)) } func (c *Cipher) IVSize() int { - return int(C.EVP_CIPHER_iv_length_not_a_macro(c.ptr)) + return int(C.X_EVP_CIPHER_iv_length(c.ptr)) } func Nid2ShortName(nid NID) (string, error) { @@ -154,7 +120,7 @@ func (ctx *cipherCtx) applyKeyAndIV(key, iv []byte) error { } if kptr != nil || iptr != nil { var res C.int - if ctx.ctx.encrypt != 0 { + if C.X_EVP_CIPHER_CTX_encrypting(ctx.ctx) != 0 { res = C.EVP_EncryptInit_ex(ctx.ctx, nil, nil, kptr, iptr) } else { res = C.EVP_DecryptInit_ex(ctx.ctx, nil, nil, kptr, iptr) @@ -167,19 +133,19 @@ func (ctx *cipherCtx) applyKeyAndIV(key, iv []byte) error { } func (ctx *cipherCtx) Cipher() *Cipher { - return &Cipher{ptr: C.EVP_CIPHER_CTX_cipher_not_a_macro(ctx.ctx)} + return &Cipher{ptr: C.X_EVP_CIPHER_CTX_cipher(ctx.ctx)} } func (ctx *cipherCtx) BlockSize() int { - return int(C.EVP_CIPHER_CTX_block_size_not_a_macro(ctx.ctx)) + return int(C.X_EVP_CIPHER_CTX_block_size(ctx.ctx)) } func (ctx *cipherCtx) KeySize() int { - return int(C.EVP_CIPHER_CTX_key_length_not_a_macro(ctx.ctx)) + return int(C.X_EVP_CIPHER_CTX_key_length(ctx.ctx)) } func (ctx *cipherCtx) IVSize() int { - return int(C.EVP_CIPHER_CTX_iv_length_not_a_macro(ctx.ctx)) + return int(C.X_EVP_CIPHER_CTX_iv_length(ctx.ctx)) } func (ctx *cipherCtx) setCtrl(code, arg int) error { diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers_test.go index 9f5d27ab1c3..463b30dfe55 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build !darwin // +build !openssl_pre_1.0 package openssl @@ -91,6 +90,10 @@ func doDecryption(key, iv, aad, ciphertext, tag []byte, blocksize, if err != nil { return nil, fmt.Errorf("Failed making GCM decryption ctx: %s", err) } + err = dctx.SetTag(tag) + if err != nil { + return nil, fmt.Errorf("Failed to set expected GCM tag: %s", err) + } aadbuf := bytes.NewBuffer(aad) for aadbuf.Len() > 0 { err = dctx.ExtraData(aadbuf.Next(bufsize)) @@ -107,10 +110,6 @@ func doDecryption(key, iv, aad, ciphertext, tag []byte, blocksize, } plainb.Write(moar) } - err = dctx.SetTag(tag) - if err != nil { - return nil, fmt.Errorf("Failed to set expected GCM tag: %s", err) - } moar, err := dctx.DecryptFinal() if err != nil { return nil, fmt.Errorf("Failed to finalize decryption: %s", err) diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/conn.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/conn.go index f77fb4d61b9..2d2f208489d 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/conn.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/conn.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,30 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <stdlib.h> -#include <openssl/ssl.h> -#include <openssl/conf.h> -#include <openssl/err.h> - -int sk_X509_num_not_a_macro(STACK_OF(X509) *sk) { return sk_X509_num(sk); } -X509 *sk_X509_value_not_a_macro(STACK_OF(X509)* sk, int i) { - return sk_X509_value(sk, i); -} -long SSL_set_tlsext_host_name_not_a_macro(SSL *ssl, const char *name) { - return SSL_set_tlsext_host_name(ssl, name); -} -const char * SSL_get_cipher_name_not_a_macro(const SSL *ssl) { - return SSL_get_cipher_name(ssl); -} -static int SSL_session_reused_not_a_macro(SSL *ssl) { - return SSL_session_reused(ssl); -} -*/ +// #include "shim.h" import "C" import ( @@ -59,8 +38,9 @@ var ( ) type Conn struct { + *SSL + conn net.Conn - ssl *C.SSL ctx *Ctx // for gc into_ssl *readBio from_ssl *writeBio @@ -156,9 +136,13 @@ func newConn(conn net.Conn, ctx *Ctx) (*Conn, error) { // the ssl object takes ownership of these objects now C.SSL_set_bio(ssl, into_ssl_cbio, from_ssl_cbio) + s := &SSL{ssl: ssl} + C.SSL_set_ex_data(s.ssl, get_ssl_idx(), unsafe.Pointer(s)) + c := &Conn{ + SSL: s, + conn: conn, - ssl: ssl, ctx: ctx, into_ssl: into_ssl, from_ssl: from_ssl} @@ -203,8 +187,10 @@ func Server(conn net.Conn, ctx *Ctx) (*Conn, error) { return c, nil } +func (c *Conn) GetCtx() *Ctx { return c.ctx } + func (c *Conn) CurrentCipher() (string, error) { - p := C.SSL_get_cipher_name_not_a_macro(c.ssl) + p := C.X_SSL_get_cipher_name(c.ssl) if p == nil { return "", errors.New("Session not established") } @@ -358,10 +344,10 @@ func (c *Conn) PeerCertificateChain() (rv []*Certificate, err error) { if sk == nil { return nil, errors.New("no peer certificates found") } - sk_num := int(C.sk_X509_num_not_a_macro(sk)) + sk_num := int(C.X_sk_X509_num(sk)) rv = make([]*Certificate, 0, sk_num) for i := 0; i < sk_num; i++ { - x := C.sk_X509_value_not_a_macro(sk, C.int(i)) + x := C.X_sk_X509_value(sk, C.int(i)) // ref holds on to the underlying connection memory so we don't need to // worry about incrementing refcounts manually or freeing the X509 rv = append(rv, &Certificate{x: x, ref: c}) @@ -578,7 +564,7 @@ func (c *Conn) SetTlsExtHostName(name string) error { defer C.free(unsafe.Pointer(cname)) runtime.LockOSThread() defer runtime.UnlockOSThread() - if C.SSL_set_tlsext_host_name_not_a_macro(c.ssl, cname) == 0 { + if C.X_SSL_set_tlsext_host_name(c.ssl, cname) == 0 { return errorFromErrorQueue() } return nil @@ -589,7 +575,7 @@ func (c *Conn) VerifyResult() VerifyResult { } func (c *Conn) SessionReused() bool { - return C.SSL_session_reused_not_a_macro(c.ssl) == 1 + return C.X_SSL_session_reused(c.ssl) == 1 } func (c *Conn) GetSession() ([]byte, error) { diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx.go index 8daa1bbbb1f..f67a95d6ea3 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,83 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl /* -#include <openssl/crypto.h> -#include <openssl/ssl.h> +#include "shim.h" #include <openssl/err.h> -#include <openssl/conf.h> -#include <openssl/x509.h> - -static long SSL_CTX_set_options_not_a_macro(SSL_CTX* ctx, long options) { - return SSL_CTX_set_options(ctx, options); -} - -static long SSL_CTX_clear_options_not_a_macro(SSL_CTX* ctx, long options) { - return SSL_CTX_clear_options(ctx, options); -} - -static long SSL_CTX_get_options_not_a_macro(SSL_CTX* ctx) { - return SSL_CTX_get_options(ctx); -} - -static long SSL_CTX_set_mode_not_a_macro(SSL_CTX* ctx, long modes) { - return SSL_CTX_set_mode(ctx, modes); -} - -static long SSL_CTX_get_mode_not_a_macro(SSL_CTX* ctx) { - return SSL_CTX_get_mode(ctx); -} - -static long SSL_CTX_set_session_cache_mode_not_a_macro(SSL_CTX* ctx, long modes) { - return SSL_CTX_set_session_cache_mode(ctx, modes); -} - -static long SSL_CTX_sess_set_cache_size_not_a_macro(SSL_CTX* ctx, long t) { - return SSL_CTX_sess_set_cache_size(ctx, t); -} - -static long SSL_CTX_sess_get_cache_size_not_a_macro(SSL_CTX* ctx) { - return SSL_CTX_sess_get_cache_size(ctx); -} - -static long SSL_CTX_set_timeout_not_a_macro(SSL_CTX* ctx, long t) { - return SSL_CTX_set_timeout(ctx, t); -} - -static long SSL_CTX_get_timeout_not_a_macro(SSL_CTX* ctx) { - return SSL_CTX_get_timeout(ctx); -} - -static int CRYPTO_add_not_a_macro(int *pointer,int amount,int type) { - return CRYPTO_add(pointer, amount, type); -} - -static long SSL_CTX_add_extra_chain_cert_not_a_macro(SSL_CTX* ctx, X509 *cert) { - return SSL_CTX_add_extra_chain_cert(ctx, cert); -} - -static long SSL_CTX_set_tlsext_servername_callback_not_a_macro( - SSL_CTX* ctx, int (*cb)(SSL *con, int *ad, void *args)) { - return SSL_CTX_set_tlsext_servername_callback(ctx, cb); -} - -#ifndef SSL_MODE_RELEASE_BUFFERS -#define SSL_MODE_RELEASE_BUFFERS 0 -#endif - -#ifndef SSL_OP_NO_COMPRESSION -#define SSL_OP_NO_COMPRESSION 0 -#endif - -#if defined SSL_CTRL_SET_TLSEXT_HOSTNAME - extern int sni_cb(SSL *ssl_conn, int *ad, void *arg); -#endif - -extern int verify_cb(int ok, X509_STORE_CTX* store); typedef STACK_OF(X509_NAME) *STACK_OF_X509_NAME_not_a_macro; @@ -97,6 +25,7 @@ static void sk_X509_NAME_pop_free_not_a_macro(STACK_OF_X509_NAME_not_a_macro st) } extern int password_cb(char *buf, int size, int rwflag, void *password); + */ import "C" @@ -114,7 +43,7 @@ import ( ) var ( - ssl_ctx_idx = C.SSL_CTX_get_ex_new_index(0, nil, nil, nil, nil) + ssl_ctx_idx = C.X_SSL_CTX_new_index() logger = spacelog.GetLogger() ) @@ -169,10 +98,16 @@ const ( func NewCtxWithVersion(version SSLVersion) (*Ctx, error) { var method *C.SSL_METHOD switch version { + case SSLv3: + method = C.X_SSLv3_method() case TLSv1: - method = C.TLSv1_method() + method = C.X_TLSv1_method() + case TLSv1_1: + method = C.X_TLSv1_1_method() + case TLSv1_2: + method = C.X_TLSv1_2_method() case AnyVersion: - method = C.SSLv23_method() + method = C.X_SSLv23_method() } if method == nil { return nil, errors.New("unknown ssl/tls version") @@ -255,6 +190,8 @@ const ( Prime256v1 EllipticCurve = C.NID_X9_62_prime256v1 // P-384: NIST/SECG curve over a 384 bit prime field Secp384r1 EllipticCurve = C.NID_secp384r1 + // P-521: NIST/SECG curve over a 521 bit prime field + Secp521r1 EllipticCurve = C.NID_secp521r1 ) // UseCertificate configures the context to present the given certificate to @@ -386,7 +323,7 @@ func (c *Ctx) AddChainCertificate(cert *Certificate) error { runtime.LockOSThread() defer runtime.UnlockOSThread() c.chain = append(c.chain, cert) - if int(C.SSL_CTX_add_extra_chain_cert_not_a_macro(c.ctx, cert.x)) != 1 { + if int(C.X_SSL_CTX_add_extra_chain_cert(c.ctx, cert.x)) != 1 { return errorFromErrorQueue() } // OpenSSL takes ownership via SSL_CTX_add_extra_chain_cert @@ -581,7 +518,9 @@ func (self *CertificateStoreCtx) GetCurrentCert() *Certificate { return nil } // add a ref - C.CRYPTO_add_not_a_macro(&x509.references, 1, C.CRYPTO_LOCK_X509) + if 1 != C.X_X509_add_ref(x509) { + return nil + } cert := &Certificate{ x: x509, } @@ -617,10 +556,13 @@ type Options uint const ( // NoCompression is only valid if you are using OpenSSL 1.0.1 or newer - NoCompression Options = C.SSL_OP_NO_COMPRESSION - NoSSLv2 Options = C.SSL_OP_NO_SSLv2 - NoSSLv3 Options = C.SSL_OP_NO_SSLv3 - NoTLSv1 Options = C.SSL_OP_NO_TLSv1 + NoCompression Options = C.SSL_OP_NO_COMPRESSION + NoSSLv2 Options = C.SSL_OP_NO_SSLv2 + NoSSLv3 Options = C.SSL_OP_NO_SSLv3 + NoTLSv1 Options = C.SSL_OP_NO_TLSv1 + // NoTLSv1_1 and NoTLSv1_2 are only valid if you are using OpenSSL 1.0.1 or newer + NoTLSv1_1 Options = C.SSL_OP_NO_TLSv1_1 + NoTLSv1_2 Options = C.SSL_OP_NO_TLSv1_2 CipherServerPreference Options = C.SSL_OP_CIPHER_SERVER_PREFERENCE NoSessionResumptionOrRenegotiation Options = C.SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION NoTicket Options = C.SSL_OP_NO_TICKET @@ -630,19 +572,19 @@ const ( // SetOptions sets context options. See // http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (c *Ctx) SetOptions(options Options) Options { - return Options(C.SSL_CTX_set_options_not_a_macro( + return Options(C.X_SSL_CTX_set_options( c.ctx, C.long(options))) } func (c *Ctx) ClearOptions(options Options) Options { - return Options(C.SSL_CTX_clear_options_not_a_macro( + return Options(C.X_SSL_CTX_clear_options( c.ctx, C.long(options))) } // GetOptions returns context options. See // https://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (c *Ctx) GetOptions() Options { - return Options(C.SSL_CTX_get_options_not_a_macro(c.ctx)) + return Options(C.X_SSL_CTX_get_options(c.ctx)) } type Modes int @@ -656,13 +598,13 @@ const ( // SetMode sets context modes. See // http://www.openssl.org/docs/ssl/SSL_CTX_set_mode.html func (c *Ctx) SetMode(modes Modes) Modes { - return Modes(C.SSL_CTX_set_mode_not_a_macro(c.ctx, C.long(modes))) + return Modes(C.X_SSL_CTX_set_mode(c.ctx, C.long(modes))) } // GetMode returns context modes. See // http://www.openssl.org/docs/ssl/SSL_CTX_set_mode.html func (c *Ctx) GetMode() Modes { - return Modes(C.SSL_CTX_get_mode_not_a_macro(c.ctx)) + return Modes(C.X_SSL_CTX_get_mode(c.ctx)) } type VerifyOptions int @@ -683,8 +625,8 @@ const ( type VerifyCallback func(ok bool, store *CertificateStoreCtx) bool -//export verify_cb_thunk -func verify_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { +//export go_ssl_ctx_verify_cb_thunk +func go_ssl_ctx_verify_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { defer func() { if err := recover(); err != nil { logger.Critf("openssl: verify callback panic'd: %v", err) @@ -709,7 +651,7 @@ func verify_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { func (c *Ctx) SetVerify(options VerifyOptions, verify_cb VerifyCallback) { c.verify_cb = verify_cb if verify_cb != nil { - C.SSL_CTX_set_verify(c.ctx, C.int(options), (*[0]byte)(C.verify_cb)) + C.SSL_CTX_set_verify(c.ctx, C.int(options), (*[0]byte)(C.X_SSL_CTX_verify_cb)) } else { C.SSL_CTX_set_verify(c.ctx, C.int(options), nil) } @@ -752,7 +694,7 @@ type TLSExtServernameCallback func(ssl *SSL) SSLTLSExtErr // http://stackoverflow.com/questions/22373332/serving-multiple-domains-in-one-box-with-sni func (c *Ctx) SetTLSExtServernameCallback(sni_cb TLSExtServernameCallback) { c.sni_cb = sni_cb - C.SSL_CTX_set_tlsext_servername_callback_not_a_macro(c.ctx, (*[0]byte)(C.sni_cb)) + C.X_SSL_CTX_set_tlsext_servername_callback(c.ctx, (*[0]byte)(C.sni_cb)) } func (c *Ctx) SetSessionId(session_id []byte) error { @@ -800,30 +742,30 @@ const ( // http://www.openssl.org/docs/ssl/SSL_CTX_set_session_cache_mode.html func (c *Ctx) SetSessionCacheMode(modes SessionCacheModes) SessionCacheModes { return SessionCacheModes( - C.SSL_CTX_set_session_cache_mode_not_a_macro(c.ctx, C.long(modes))) + C.X_SSL_CTX_set_session_cache_mode(c.ctx, C.long(modes))) } // Set session cache timeout. Returns previously set value. // See https://www.openssl.org/docs/ssl/SSL_CTX_set_timeout.html func (c *Ctx) SetTimeout(t time.Duration) time.Duration { - prev := C.SSL_CTX_set_timeout_not_a_macro(c.ctx, C.long(t/time.Second)) + prev := C.X_SSL_CTX_set_timeout(c.ctx, C.long(t/time.Second)) return time.Duration(prev) * time.Second } // Get session cache timeout. // See https://www.openssl.org/docs/ssl/SSL_CTX_set_timeout.html func (c *Ctx) GetTimeout() time.Duration { - return time.Duration(C.SSL_CTX_get_timeout_not_a_macro(c.ctx)) * time.Second + return time.Duration(C.X_SSL_CTX_get_timeout(c.ctx)) * time.Second } // Set session cache size. Returns previously set value. // https://www.openssl.org/docs/ssl/SSL_CTX_sess_set_cache_size.html func (c *Ctx) SessSetCacheSize(t int) int { - return int(C.SSL_CTX_sess_set_cache_size_not_a_macro(c.ctx, C.long(t))) + return int(C.X_SSL_CTX_sess_set_cache_size(c.ctx, C.long(t))) } // Get session cache size. // https://www.openssl.org/docs/ssl/SSL_CTX_sess_set_cache_size.html func (c *Ctx) SessGetCacheSize() int { - return int(C.SSL_CTX_sess_get_cache_size_not_a_macro(c.ctx)) + return int(C.X_SSL_CTX_sess_get_cache_size(c.ctx)) } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx_test.go index 9644e518bf3..cd2a82a5a66 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Ryan Hileman +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh.go new file mode 100644 index 00000000000..7d0cc703985 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh.go @@ -0,0 +1,68 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +// #include "shim.h" +import "C" +import ( + "errors" + "unsafe" +) + +// DeriveSharedSecret derives a shared secret using a private key and a peer's +// public key. +// The specific algorithm that is used depends on the types of the +// keys, but it is most commonly a variant of Diffie-Hellman. +func DeriveSharedSecret(private PrivateKey, public PublicKey) ([]byte, error) { + // Create context for the shared secret derivation + dhCtx := C.EVP_PKEY_CTX_new(private.evpPKey(), nil) + if dhCtx == nil { + return nil, errors.New("failed creating shared secret derivation context") + } + defer C.EVP_PKEY_CTX_free(dhCtx) + + // Initialize the context + if int(C.EVP_PKEY_derive_init(dhCtx)) != 1 { + return nil, errors.New("failed initializing shared secret derivation context") + } + + // Provide the peer's public key + if int(C.EVP_PKEY_derive_set_peer(dhCtx, public.evpPKey())) != 1 { + return nil, errors.New("failed adding peer public key to context") + } + + // Determine how large of a buffer we need for the shared secret + var buffLen C.size_t + if int(C.EVP_PKEY_derive(dhCtx, nil, &buffLen)) != 1 { + return nil, errors.New("failed determining shared secret length") + } + + // Allocate a buffer + buffer := C.X_OPENSSL_malloc(buffLen) + if buffer == nil { + return nil, errors.New("failed allocating buffer for shared secret") + } + defer C.X_OPENSSL_free(buffer) + + // Derive the shared secret + if int(C.EVP_PKEY_derive(dhCtx, (*C.uchar)(buffer), &buffLen)) != 1 { + return nil, errors.New("failed deriving the shared secret") + } + + secret := C.GoBytes(unsafe.Pointer(buffer), C.int(buffLen)) + return secret, nil +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh_test.go new file mode 100644 index 00000000000..e6b5ae59905 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh_test.go @@ -0,0 +1,51 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +import ( + "bytes" + "testing" +) + +func TestECDH(t *testing.T) { + t.Parallel() + if !HasECDH() { + t.Skip("ECDH not available") + } + + myKey, err := GenerateECKey(Prime256v1) + if err != nil { + t.Fatal(err) + } + peerKey, err := GenerateECKey(Prime256v1) + if err != nil { + t.Fatal(err) + } + + mySecret, err := DeriveSharedSecret(myKey, peerKey) + if err != nil { + t.Fatal(err) + } + theirSecret, err := DeriveSharedSecret(peerKey, myKey) + if err != nil { + t.Fatal(err) + } + + if bytes.Compare(mySecret, theirSecret) != 0 { + t.Fatal("shared secrets are different") + } +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dhparam.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dhparam.go index a698645c1ec..294d0645c03 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dhparam.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dhparam.go @@ -1,21 +1,20 @@ -// +build cgo +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package openssl -/* -#include <openssl/crypto.h> -#include <openssl/ssl.h> -#include <openssl/err.h> -#include <openssl/conf.h> -#include <openssl/dh.h> - -static long SSL_CTX_set_tmp_dh_not_a_macro(SSL_CTX* ctx, DH *dh) { - return SSL_CTX_set_tmp_dh(ctx, dh); -} -static long PEM_read_DHparams_not_a_macro(SSL_CTX* ctx, DH *dh) { - return SSL_CTX_set_tmp_dh(ctx, dh); -} -*/ +// #include "shim.h" import "C" import ( @@ -58,7 +57,7 @@ func (c *Ctx) SetDHParameters(dh *DH) error { runtime.LockOSThread() defer runtime.UnlockOSThread() - if int(C.SSL_CTX_set_tmp_dh_not_a_macro(c.ctx, dh.dh)) != 1 { + if int(C.X_SSL_CTX_set_tmp_dh(c.ctx, dh.dh)) != 1 { return errorFromErrorQueue() } return nil diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/digest.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/digest.go index 44d4d001b13..6d8d2635aee 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/digest.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/digest.go @@ -1,4 +1,4 @@ -// Copyright (C) 2015 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -// #include <openssl/evp.h> +// #include "shim.h" import "C" import ( @@ -34,7 +32,7 @@ type Digest struct { func GetDigestByName(name string) (*Digest, error) { cname := C.CString(name) defer C.free(unsafe.Pointer(cname)) - p := C.EVP_get_digestbyname(cname) + p := C.X_EVP_get_digestbyname(cname) if p == nil { return nil, fmt.Errorf("Digest %v not found", name) } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/engine.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/engine.go index 7a175b70f7c..78aef956fca 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/engine.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/engine.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl /* diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/features.go index 894c2676038..c091f0644e8 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.c +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/features.go @@ -1,4 +1,4 @@ -// Copyright (C) 2015 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,16 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include <openssl/ssl.h> -#include <openssl/evp.h> -#include "_cgo_export.h" +package openssl -int ticket_key_cb(SSL *s, unsigned char key_name[16], - unsigned char iv[EVP_MAX_IV_LENGTH], - EVP_CIPHER_CTX *cctx, HMAC_CTX *hctx, int enc) { +// #include "shim.h" +import "C" - SSL_CTX* ssl_ctx = SSL_get_SSL_CTX(s); - void* p = SSL_CTX_get_ex_data(ssl_ctx, get_ssl_ctx_idx()); - // get the pointer to the go Ctx object and pass it back into the thunk - return ticket_key_cb_thunk(p, s, key_name, iv, cctx, hctx, enc); +func HasECDH() bool { + return C.X_OPENSSL_NO_ECDH() == 0 } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips.go index fcccb000a36..77e1dc3eddf 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips.go @@ -1,19 +1,56 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build cgo -// +build !darwin package openssl /* -#include <openssl/ssl.h> +#include "shim.h" + +static int X_FIPS_defined() { +#ifdef OPENSSL_FIPS + return 1; +#else + return 0; +#endif +} + */ import "C" +import "runtime" +// FIPSModeDefined indicates if the openssl library has the FIPS +// module complied in, specifically if the "OPENSSL_FIPS" macro is defined. +func FIPSModeDefined() bool { + if C.X_FIPS_defined() == 1 { + return true + } + return false +} + +// FIPSModeSet enables a FIPS 140-2 validated mode of operation. +// https://wiki.openssl.org/index.php/FIPS_mode_set() func FIPSModeSet(mode bool) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + var r C.int if mode { - r = C.FIPS_mode_set(1) + r = C.X_FIPS_mode_set(1) } else { - r = C.FIPS_mode_set(0) + r = C.X_FIPS_mode_set(0) } if r != 1 { return errorFromErrorQueue() @@ -22,8 +59,8 @@ func FIPSModeSet(mode bool) error { } func FIPSMode() bool { - if C.FIPS_mode() == 0 { - return false + if FIPSModeDefined() && C.X_FIPS_mode() != 0 { + return true } - return true + return false } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips_test.go index 63d353b4a41..31218edb33b 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips_test.go @@ -1,5 +1,3 @@ -// +build !darwin - package openssl_test import ( @@ -9,8 +7,12 @@ import ( ) func TestSetFIPSMode(t *testing.T) { + if !openssl.FIPSModeDefined() { + t.Skip("OPENSSL_FIPS not defined in headers") + } + if openssl.FIPSMode() { - t.Fatal("Expected FIPS mode to be disabled, but was enabled") + t.Skip("FIPS mode already enabled") } err := openssl.FIPSModeSet(true) @@ -22,12 +24,4 @@ func TestSetFIPSMode(t *testing.T) { t.Fatal("Expected FIPS mode to be enabled, but was disabled") } - err = openssl.FIPSModeSet(false) - if err != nil { - t.Fatal(err) - } - - if openssl.FIPSMode() { - t.Fatal("Expected FIPS mode to be disabled, but was enabled") - } } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac.go new file mode 100644 index 00000000000..a8640cfac63 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac.go @@ -0,0 +1,91 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openssl + +// #include "shim.h" +import "C" + +import ( + "errors" + "runtime" + "unsafe" +) + +type HMAC struct { + ctx *C.HMAC_CTX + engine *Engine + md *C.EVP_MD +} + +func NewHMAC(key []byte, digestAlgorithm EVP_MD) (*HMAC, error) { + return NewHMACWithEngine(key, digestAlgorithm, nil) +} + +func NewHMACWithEngine(key []byte, digestAlgorithm EVP_MD, e *Engine) (*HMAC, error) { + var md *C.EVP_MD = getDigestFunction(digestAlgorithm) + h := &HMAC{engine: e, md: md} + h.ctx = C.X_HMAC_CTX_new() + if h.ctx == nil { + return nil, errors.New("unable to allocate HMAC_CTX") + } + + var c_e *C.ENGINE + if e != nil { + c_e = e.e + } + if rc := C.X_HMAC_Init_ex(h.ctx, + unsafe.Pointer(&key[0]), + C.int(len(key)), + md, + c_e); rc != 1 { + C.X_HMAC_CTX_free(h.ctx) + return nil, errors.New("failed to initialize HMAC_CTX") + } + + runtime.SetFinalizer(h, func(h *HMAC) { h.Close() }) + return h, nil +} + +func (h *HMAC) Close() { + C.X_HMAC_CTX_free(h.ctx) +} + +func (h *HMAC) Write(data []byte) (n int, err error) { + if len(data) == 0 { + return 0, nil + } + if rc := C.X_HMAC_Update(h.ctx, (*C.uchar)(unsafe.Pointer(&data[0])), + C.size_t(len(data))); rc != 1 { + return 0, errors.New("failed to update HMAC") + } + return len(data), nil +} + +func (h *HMAC) Reset() error { + if 1 != C.X_HMAC_Init_ex(h.ctx, nil, 0, nil, nil) { + return errors.New("failed to reset HMAC_CTX") + } + return nil +} + +func (h *HMAC) Final() (result []byte, err error) { + mdLength := C.X_EVP_MD_size(h.md) + result = make([]byte, mdLength) + if rc := C.X_HMAC_Final(h.ctx, (*C.uchar)(unsafe.Pointer(&result[0])), + (*C.uint)(unsafe.Pointer(&mdLength))); rc != 1 { + return nil, errors.New("failed to finalized HMAC") + } + return result, h.Reset() +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac_test.go new file mode 100644 index 00000000000..424720e2171 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac_test.go @@ -0,0 +1,74 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "testing" +) + +func TestSHA256HMAC(t *testing.T) { + key := []byte("d741787cc61851af045ccd37") + data := []byte("5912EEFD-59EC-43E3-ADB8-D5325AEC3271") + + h, err := NewHMAC(key, EVP_SHA256) + if err != nil { + t.Fatalf("Unable to create new HMAC: %s", err) + } + if _, err := h.Write(data); err != nil { + t.Fatalf("Unable to write data into HMAC: %s", err) + } + + var actualHMACBytes []byte + if actualHMACBytes, err = h.Final(); err != nil { + t.Fatalf("Error while finalizing HMAC: %s", err) + } + actualString := hex.EncodeToString(actualHMACBytes) + + // generate HMAC with built-in crypto lib + mac := hmac.New(sha256.New, key) + mac.Write(data) + expectedString := hex.EncodeToString(mac.Sum(nil)) + + if expectedString != actualString { + t.Errorf("HMAC was incorrect: expected=%s, actual=%s", expectedString, actualString) + } +} + +func BenchmarkSHA256HMAC(b *testing.B) { + key := []byte("d741787cc61851af045ccd37") + data := []byte("5912EEFD-59EC-43E3-ADB8-D5325AEC3271") + + h, err := NewHMAC(key, EVP_SHA256) + if err != nil { + b.Fatalf("Unable to create new HMAC: %s", err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := h.Write(data); err != nil { + b.Fatalf("Unable to write data into HMAC: %s", err) + } + + var err error + if _, err = h.Final(); err != nil { + b.Fatalf("Error while finalizing HMAC: %s", err) + } + } +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.c index 9a610292067..aef33355262 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.c +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.c @@ -1,7 +1,8 @@ -/* Go-OpenSSL notice: - This file is required for all OpenSSL versions prior to 1.1.0. This simply - provides the new 1.1.0 X509_check_* methods for hostname validation if they - don't already exist. +/* + * Go-OpenSSL notice: + * This file is required for all OpenSSL versions prior to 1.1.0. This simply + * provides the new 1.1.0 X509_check_* methods for hostname validation if they + * don't already exist. */ #include <openssl/x509.h> @@ -67,6 +68,7 @@ */ /* X509 v3 extension utilities */ +#include <string.h> #include <stdlib.h> #include <openssl/ssl.h> #include <openssl/conf.h> @@ -346,22 +348,26 @@ static int do_x509_check(X509 *x, const unsigned char *chk, size_t chklen, return 0; } -int _X509_check_host(X509 *x, const unsigned char *chk, size_t chklen, - unsigned int flags) +#if OPENSSL_VERSION_NUMBER < 0x1000200fL + +int X509_check_host(X509 *x, const unsigned char *chk, size_t chklen, + unsigned int flags, char **peername) { return do_x509_check(x, chk, chklen, flags, GEN_DNS); } -int _X509_check_email(X509 *x, const unsigned char *chk, size_t chklen, +int X509_check_email(X509 *x, const unsigned char *chk, size_t chklen, unsigned int flags) { return do_x509_check(x, chk, chklen, flags, GEN_EMAIL); } -int _X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, +int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, unsigned int flags) { return do_x509_check(x, chk, chklen, flags, GEN_IPADD); } +#endif /* OPENSSL_VERSION_NUMBER */ + #endif diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.go index c1d1202fb65..f0b36db678d 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl /* @@ -25,11 +23,11 @@ package openssl #define X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT 0x1 #define X509_CHECK_FLAG_NO_WILDCARDS 0x2 -extern int _X509_check_host(X509 *x, const unsigned char *chk, size_t chklen, - unsigned int flags); -extern int _X509_check_email(X509 *x, const unsigned char *chk, size_t chklen, +extern int X509_check_host(X509 *x, const unsigned char *chk, size_t chklen, + unsigned int flags, char **peername); +extern int X509_check_email(X509 *x, const unsigned char *chk, size_t chklen, unsigned int flags); -extern int _X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, +extern int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, unsigned int flags); #endif */ @@ -60,8 +58,9 @@ const ( func (c *Certificate) CheckHost(host string, flags CheckFlags) error { chost := unsafe.Pointer(C.CString(host)) defer C.free(chost) - rv := C._X509_check_host(c.x, (*C.uchar)(chost), C.size_t(len(host)), - C.uint(flags)) + + rv := C.X509_check_host(c.x, (*C.uchar)(chost), C.size_t(len(host)), + C.uint(flags), nil) if rv > 0 { return nil } @@ -79,7 +78,7 @@ func (c *Certificate) CheckHost(host string, flags CheckFlags) error { func (c *Certificate) CheckEmail(email string, flags CheckFlags) error { cemail := unsafe.Pointer(C.CString(email)) defer C.free(cemail) - rv := C._X509_check_email(c.x, (*C.uchar)(cemail), C.size_t(len(email)), + rv := C.X509_check_email(c.x, (*C.uchar)(cemail), C.size_t(len(email)), C.uint(flags)) if rv > 0 { return nil @@ -97,7 +96,7 @@ func (c *Certificate) CheckEmail(email string, flags CheckFlags) error { // there was no internal error. func (c *Certificate) CheckIP(ip net.IP, flags CheckFlags) error { cip := unsafe.Pointer(&ip[0]) - rv := C._X509_check_ip(c.x, (*C.uchar)(cip), C.size_t(len(ip)), + rv := C.X509_check_ip(c.x, (*C.uchar)(cip), C.size_t(len(ip)), C.uint(flags)) if rv > 0 { return nil diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/http.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/http.go index e3be32c264a..39bd5a28b5f 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/http.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/http.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init.go index 314e5415c18..ac2aa04327b 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,49 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - /* Package openssl is a light wrapper around OpenSSL for Go. -It strives to provide a near-drop-in replacement for the Go standard library -tls package, while allowing for: - -Performance - -OpenSSL is battle-tested and optimized C. While Go's built-in library shows -great promise, it is still young and in some places, inefficient. This simple -OpenSSL wrapper can often do at least 2x with the same cipher and protocol. - -On my lappytop, I get the following benchmarking speeds: - BenchmarkSHA1Large_openssl 1000 2611282 ns/op 401.56 MB/s - BenchmarkSHA1Large_stdlib 500 3963983 ns/op 264.53 MB/s - BenchmarkSHA1Small_openssl 1000000 3476 ns/op 0.29 MB/s - BenchmarkSHA1Small_stdlib 5000000 550 ns/op 1.82 MB/s - BenchmarkSHA256Large_openssl 200 8085314 ns/op 129.69 MB/s - BenchmarkSHA256Large_stdlib 100 18948189 ns/op 55.34 MB/s - BenchmarkSHA256Small_openssl 1000000 4262 ns/op 0.23 MB/s - BenchmarkSHA256Small_stdlib 1000000 1444 ns/op 0.69 MB/s - BenchmarkOpenSSLThroughput 100000 21634 ns/op 47.33 MB/s - BenchmarkStdlibThroughput 50000 58974 ns/op 17.36 MB/s - -Interoperability - -Many systems support OpenSSL with a variety of plugins and modules for things, -such as hardware acceleration in embedded devices. - -Greater flexibility and configuration - -OpenSSL allows for far greater configuration of corner cases and backwards -compatibility (such as support of SSLv2). You shouldn't be using SSLv2 if you -can help but, but sometimes you can't help it. - -Security - -Yeah yeah, Heartbleed. But according to the author of the standard library's -TLS implementation, Go's TLS library is vulnerable to timing attacks. And -whether or not OpenSSL received the appropriate amount of scrutiny -pre-Heartbleed, it sure is receiving it now. +This version has been forked from https://github.com/spacemonkeygo/openssl +for greater back-compatibility to older openssl libraries. Usage @@ -80,62 +42,26 @@ Making a client connection is straightforward too: } conn, err := openssl.Dial("tcp", "localhost:7777", ctx, 0) -Help wanted: To get this library to work with net/http's client, we -had to fork net/http. It would be nice if an alternate http client library -supported the generality needed to use OpenSSL instead of crypto/tls. */ package openssl -/* -#include <openssl/ssl.h> -#include <openssl/conf.h> -#include <openssl/err.h> -#include <openssl/evp.h> -#include <openssl/engine.h> - -extern int Goopenssl_init_locks(); -extern unsigned long Goopenssl_thread_id_callback(); -extern void Goopenssl_thread_locking_callback(int, int, const char*, int); - -static int Goopenssl_init_threadsafety() { - // Set up OPENSSL thread safety callbacks. - // TOOLS-1694 added setting of thread id callback for compatibility with openssl 0.9.8 - int rc = Goopenssl_init_locks(); - if (rc == 0) { - CRYPTO_set_locking_callback(Goopenssl_thread_locking_callback); - } - CRYPTO_set_id_callback(Goopenssl_thread_id_callback); - return rc; -} - -static void OpenSSL_add_all_algorithms_not_a_macro() { - OpenSSL_add_all_algorithms(); -} - -*/ +// #include "shim.h" import "C" import ( - "errors" "fmt" "strings" ) func init() { - C.ERR_load_crypto_strings() - C.OPENSSL_config(nil) - C.ENGINE_load_builtin_engines() - C.SSL_load_error_strings() - C.SSL_library_init() - C.OpenSSL_add_all_algorithms_not_a_macro() - rc := C.Goopenssl_init_threadsafety() - if rc != 0 { - panic(fmt.Errorf("Goopenssl_init_locks failed with %d", rc)) + if rc := C.X_shim_init(); rc != 0 { + panic(fmt.Errorf("X_shim_init failed with %d", rc)) } } // errorFromErrorQueue needs to run in the same OS thread as the operation -// that caused the possible error +// that caused the possible error. In some circumstances, ERR_get_error +// returns 0 when it shouldn't so we provide a message in that case. func errorFromErrorQueue() error { var errs []string for { @@ -143,10 +69,14 @@ func errorFromErrorQueue() error { if err == 0 { break } - errs = append(errs, fmt.Sprintf("%s:%s:%s", + errs = append(errs, fmt.Sprintf("%x:%s:%s:%s", + err, C.GoString(C.ERR_lib_error_string(err)), C.GoString(C.ERR_func_error_string(err)), C.GoString(C.ERR_reason_error_string(err)))) } - return errors.New(fmt.Sprintf("SSL errors: %s", strings.Join(errs, "\n"))) + if len(errs) == 0 { + errs = append(errs, "0:Error unavailable") + } + return fmt.Errorf("SSL errors: %s", strings.Join(errs, "\n")) } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_posix.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_posix.go index 99558298e3a..9e52b4e00be 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_posix.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_posix.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,13 +18,14 @@ package openssl /* +#if OPENSSL_VERSION_NUMBER < 0x10100000L #include <errno.h> #include <openssl/crypto.h> #include <pthread.h> pthread_mutex_t* goopenssl_locks; -int Goopenssl_init_locks() { +int go_init_locks() { int rc = 0; int nlock; int i; @@ -52,8 +53,7 @@ int Goopenssl_init_locks() { return rc; } -#if OPENSSL_VERSION_NUMBER < 0x10100000L -void Goopenssl_thread_locking_callback(int mode, int n, const char *file, +void go_thread_locking_callback(int mode, int n, const char *file, int line) { if (mode & CRYPTO_LOCK) { pthread_mutex_lock(&goopenssl_locks[n]); @@ -61,7 +61,8 @@ void Goopenssl_thread_locking_callback(int mode, int n, const char *file, pthread_mutex_unlock(&goopenssl_locks[n]); } } -unsigned long Goopenssl_thread_id_callback() { + +unsigned long go_thread_id_callback() { return (unsigned long) pthread_self(); } #endif diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_windows.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_windows.go index ec817926b7a..4a096899074 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_windows.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_windows.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,20 +17,14 @@ package openssl /* - -#cgo windows LDFLAGS: -lssleay32 -llibeay32 -L c:/openssl/bin -#cgo windows CFLAGS: -I"c:/openssl/include" - -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif +#if OPENSSL_VERSION_NUMBER < 0x10100000L #include <errno.h> #include <openssl/crypto.h> #include <windows.h> CRITICAL_SECTION* goopenssl_locks; -int Goopenssl_init_locks() { +int go_init_locks() { int rc = 0; int nlock; int i; @@ -48,7 +42,7 @@ int Goopenssl_init_locks() { return 0; } -void Goopenssl_thread_locking_callback(int mode, int n, const char *file, +void go_thread_locking_callback(int mode, int n, const char *file, int line) { if (mode & CRYPTO_LOCK) { EnterCriticalSection(&goopenssl_locks[n]); @@ -56,8 +50,8 @@ void Goopenssl_thread_locking_callback(int mode, int n, const char *file, LeaveCriticalSection(&goopenssl_locks[n]); } } -#if OPENSSL_VERSION_NUMBER < 0x10100000L -unsigned long Goopenssl_thread_id_callback() { + +unsigned long go_thread_id_callback() { return (unsigned long) GetCurrentThreadId(); } #endif diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key.go index cc17f5fcf7d..4e39a38a579 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,35 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -// #include <openssl/evp.h> -// #include <openssl/ssl.h> -// #include <openssl/conf.h> -// -// int EVP_SignInit_not_a_macro(EVP_MD_CTX *ctx, const EVP_MD *type) { -// return EVP_SignInit(ctx, type); -// } -// -// int EVP_SignUpdate_not_a_macro(EVP_MD_CTX *ctx, const void *d, -// unsigned int cnt) { -// return EVP_SignUpdate(ctx, d, cnt); -// } -// -// int EVP_VerifyInit_not_a_macro(EVP_MD_CTX *ctx, const EVP_MD *type) { -// return EVP_VerifyInit(ctx, type); -// } -// -// int EVP_VerifyUpdate_not_a_macro(EVP_MD_CTX *ctx, const void *d, -// unsigned int cnt) { -// return EVP_VerifyUpdate(ctx, d, cnt); -// } -// -// int EVP_PKEY_assign_charp(EVP_PKEY *pkey, int type, char *key) { -// return EVP_PKEY_assign(pkey, type, key); -// } +// #include "shim.h" import "C" import ( @@ -53,25 +27,30 @@ import ( type Method *C.EVP_MD var ( - SHA1_Method Method = C.EVP_sha1() - SHA256_Method Method = C.EVP_sha256() - SHA512_Method Method = C.EVP_sha512() + SHA1_Method Method = C.X_EVP_sha1() + SHA256_Method Method = C.X_EVP_sha256() + SHA512_Method Method = C.X_EVP_sha512() ) -type PublicKey interface { - // Verifies the data signature using PKCS1.15 - VerifyPKCS1v15(method Method, data, sig []byte) error - - // MarshalPKIXPublicKeyPEM converts the public key to PEM-encoded PKIX - // format - MarshalPKIXPublicKeyPEM() (pem_block []byte, err error) - - // MarshalPKIXPublicKeyDER converts the public key to DER-encoded PKIX - // format - MarshalPKIXPublicKeyDER() (der_block []byte, err error) - - evpPKey() *C.EVP_PKEY -} +// Constants for the various key types. +// Mapping of name -> NID taken from openssl/evp.h +const ( + KeyTypeNone = NID_undef + KeyTypeRSA = NID_rsaEncryption + KeyTypeRSA2 = NID_rsa + KeyTypeDSA = NID_dsa + KeyTypeDSA1 = NID_dsa_2 + KeyTypeDSA2 = NID_dsaWithSHA + KeyTypeDSA3 = NID_dsaWithSHA1 + KeyTypeDSA4 = NID_dsaWithSHA1_2 + KeyTypeDH = NID_dhKeyAgreement + KeyTypeDHX = NID_dhpublicnumber + KeyTypeEC = NID_X9_62_id_ecPublicKey + KeyTypeHMAC = NID_hmac + KeyTypeCMAC = NID_cmac + KeyTypeTLS1PRF = NID_tls1_prf + KeyTypeHKDF = NID_hkdf +) type PrivateKey interface { PublicKey @@ -95,22 +74,21 @@ type pKey struct { func (key *pKey) evpPKey() *C.EVP_PKEY { return key.key } func (key *pKey) SignPKCS1v15(method Method, data []byte) ([]byte, error) { - var ctx C.EVP_MD_CTX - C.EVP_MD_CTX_init(&ctx) - defer C.EVP_MD_CTX_cleanup(&ctx) + ctx := C.X_EVP_MD_CTX_new() + defer C.X_EVP_MD_CTX_free(ctx) - if 1 != C.EVP_SignInit_not_a_macro(&ctx, method) { + if 1 != C.X_EVP_SignInit(ctx, method) { return nil, errors.New("signpkcs1v15: failed to init signature") } if len(data) > 0 { - if 1 != C.EVP_SignUpdate_not_a_macro( - &ctx, unsafe.Pointer(&data[0]), C.uint(len(data))) { + if 1 != C.X_EVP_SignUpdate( + ctx, unsafe.Pointer(&data[0]), C.uint(len(data))) { return nil, errors.New("signpkcs1v15: failed to update signature") } } - sig := make([]byte, C.EVP_PKEY_size(key.key)) + sig := make([]byte, C.X_EVP_PKEY_size(key.key)) var sigblen C.uint - if 1 != C.EVP_SignFinal(&ctx, + if 1 != C.X_EVP_SignFinal(ctx, ((*C.uchar)(unsafe.Pointer(&sig[0]))), &sigblen, key.key) { return nil, errors.New("signpkcs1v15: failed to finalize signature") } @@ -118,45 +96,25 @@ func (key *pKey) SignPKCS1v15(method Method, data []byte) ([]byte, error) { } func (key *pKey) VerifyPKCS1v15(method Method, data, sig []byte) error { - var ctx C.EVP_MD_CTX - C.EVP_MD_CTX_init(&ctx) - defer C.EVP_MD_CTX_cleanup(&ctx) + ctx := C.X_EVP_MD_CTX_new() + defer C.X_EVP_MD_CTX_free(ctx) - if 1 != C.EVP_VerifyInit_not_a_macro(&ctx, method) { + if 1 != C.X_EVP_VerifyInit(ctx, method) { return errors.New("verifypkcs1v15: failed to init verify") } if len(data) > 0 { - if 1 != C.EVP_VerifyUpdate_not_a_macro( - &ctx, unsafe.Pointer(&data[0]), C.uint(len(data))) { + if 1 != C.X_EVP_VerifyUpdate( + ctx, unsafe.Pointer(&data[0]), C.uint(len(data))) { return errors.New("verifypkcs1v15: failed to update verify") } } - if 1 != C.EVP_VerifyFinal(&ctx, + if 1 != C.X_EVP_VerifyFinal(ctx, ((*C.uchar)(unsafe.Pointer(&sig[0]))), C.uint(len(sig)), key.key) { return errors.New("verifypkcs1v15: failed to finalize verify") } return nil } -func (key *pKey) MarshalPKCS1PrivateKeyPEM() (pem_block []byte, - err error) { - bio := C.BIO_new(C.BIO_s_mem()) - if bio == nil { - return nil, errors.New("failed to allocate memory BIO") - } - defer C.BIO_free(bio) - rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) - if rsa == nil { - return nil, errors.New("failed getting rsa key") - } - defer C.RSA_free(rsa) - if int(C.PEM_write_bio_RSAPrivateKey(bio, rsa, nil, nil, C.int(0), nil, - nil)) != 1 { - return nil, errors.New("failed dumping private key") - } - return ioutil.ReadAll(asAnyBio(bio)) -} - func (key *pKey) MarshalPKCS1PrivateKeyDER() (der_block []byte, err error) { bio := C.BIO_new(C.BIO_s_mem()) @@ -164,14 +122,11 @@ func (key *pKey) MarshalPKCS1PrivateKeyDER() (der_block []byte, return nil, errors.New("failed to allocate memory BIO") } defer C.BIO_free(bio) - rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) - if rsa == nil { - return nil, errors.New("failed getting rsa key") - } - defer C.RSA_free(rsa) - if int(C.i2d_RSAPrivateKey_bio(bio, rsa)) != 1 { + + if int(C.i2d_PrivateKey_bio(bio, key.key)) != 1 { return nil, errors.New("failed dumping private key der") } + return ioutil.ReadAll(asAnyBio(bio)) } @@ -182,14 +137,11 @@ func (key *pKey) MarshalPKIXPublicKeyPEM() (pem_block []byte, return nil, errors.New("failed to allocate memory BIO") } defer C.BIO_free(bio) - rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) - if rsa == nil { - return nil, errors.New("failed getting rsa key") - } - defer C.RSA_free(rsa) - if int(C.PEM_write_bio_RSA_PUBKEY(bio, rsa)) != 1 { + + if int(C.PEM_write_bio_PUBKEY(bio, key.key)) != 1 { return nil, errors.New("failed dumping public key pem") } + return ioutil.ReadAll(asAnyBio(bio)) } @@ -200,14 +152,11 @@ func (key *pKey) MarshalPKIXPublicKeyDER() (der_block []byte, return nil, errors.New("failed to allocate memory BIO") } defer C.BIO_free(bio) - rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) - if rsa == nil { - return nil, errors.New("failed getting rsa key") - } - defer C.RSA_free(rsa) - if int(C.i2d_RSA_PUBKEY_bio(bio, rsa)) != 1 { + + if int(C.i2d_PUBKEY_bio(bio, key.key)) != 1 { return nil, errors.New("failed dumping public key der") } + return ioutil.ReadAll(asAnyBio(bio)) } @@ -223,31 +172,20 @@ func LoadPrivateKeyFromPEM(pem_block []byte) (PrivateKey, error) { } defer C.BIO_free(bio) - rsakey := C.PEM_read_bio_RSAPrivateKey(bio, nil, nil, nil) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.PEM_read_bio_PrivateKey(bio, nil, nil, nil) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading private key") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } -// LoadPrivateKeyFromPEM loads a private key from a PEM-encoded block. -func LoadPrivateKeyFromPEMWidthPassword(pem_block []byte, password string) ( +// LoadPrivateKeyFromPEMWithPassword loads a private key from a PEM-encoded block. +func LoadPrivateKeyFromPEMWithPassword(pem_block []byte, password string) ( PrivateKey, error) { if len(pem_block) == 0 { return nil, errors.New("empty pem block") @@ -260,25 +198,14 @@ func LoadPrivateKeyFromPEMWidthPassword(pem_block []byte, password string) ( defer C.BIO_free(bio) cs := C.CString(password) defer C.free(unsafe.Pointer(cs)) - rsakey := C.PEM_read_bio_RSAPrivateKey(bio, nil, nil, unsafe.Pointer(cs)) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.PEM_read_bio_PrivateKey(bio, nil, nil, unsafe.Pointer(cs)) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading private key") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } @@ -295,29 +222,25 @@ func LoadPrivateKeyFromDER(der_block []byte) (PrivateKey, error) { } defer C.BIO_free(bio) - rsakey := C.d2i_RSAPrivateKey_bio(bio, nil) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.d2i_PrivateKey_bio(bio, nil) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading private key der") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } +// LoadPrivateKeyFromPEMWidthPassword loads a private key from a PEM-encoded block. +// Backwards-compatible with typo +func LoadPrivateKeyFromPEMWidthPassword(pem_block []byte, password string) ( + PrivateKey, error) { + return LoadPrivateKeyFromPEMWithPassword(pem_block, password) +} + // LoadPublicKeyFromPEM loads a public key from a PEM-encoded block. func LoadPublicKeyFromPEM(pem_block []byte) (PublicKey, error) { if len(pem_block) == 0 { @@ -330,25 +253,14 @@ func LoadPublicKeyFromPEM(pem_block []byte) (PublicKey, error) { } defer C.BIO_free(bio) - rsakey := C.PEM_read_bio_RSA_PUBKEY(bio, nil, nil, nil) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.PEM_read_bio_PUBKEY(bio, nil, nil, nil) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading public key der") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } @@ -365,25 +277,14 @@ func LoadPublicKeyFromDER(der_block []byte) (PublicKey, error) { } defer C.BIO_free(bio) - rsakey := C.d2i_RSA_PUBKEY_bio(bio, nil) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.d2i_PUBKEY_bio(bio, nil) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading public key der") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } @@ -399,17 +300,17 @@ func GenerateRSAKeyWithExponent(bits int, exponent int) (PrivateKey, error) { if rsa == nil { return nil, errors.New("failed to generate RSA key") } - key := C.EVP_PKEY_new() + key := C.X_EVP_PKEY_new() if key == nil { return nil, errors.New("failed to allocate EVP_PKEY") } - if C.EVP_PKEY_assign_charp(key, C.EVP_PKEY_RSA, (*C.char)(unsafe.Pointer(rsa))) != 1 { - C.EVP_PKEY_free(key) + if C.X_EVP_PKEY_assign_charp(key, C.EVP_PKEY_RSA, (*C.char)(unsafe.Pointer(rsa))) != 1 { + C.X_EVP_PKEY_free(key) return nil, errors.New("failed to assign RSA key") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_0_9.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_0_9.go new file mode 100644 index 00000000000..ed17ef08a40 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_0_9.go @@ -0,0 +1,58 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build openssl_pre_1.0 + +package openssl + +// #include "shim.h" +import "C" +import ( + "errors" + "io/ioutil" +) + +type PublicKey interface { + // Verifies the data signature using PKCS1.15 + VerifyPKCS1v15(method Method, data, sig []byte) error + + // MarshalPKIXPublicKeyPEM converts the public key to PEM-encoded PKIX + // format + MarshalPKIXPublicKeyPEM() (pem_block []byte, err error) + + // MarshalPKIXPublicKeyDER converts the public key to DER-encoded PKIX + // format + MarshalPKIXPublicKeyDER() (der_block []byte, err error) + + evpPKey() *C.EVP_PKEY +} + +func (key *pKey) MarshalPKCS1PrivateKeyPEM() (pem_block []byte, + err error) { + bio := C.BIO_new(C.BIO_s_mem()) + if bio == nil { + return nil, errors.New("failed to allocate memory BIO") + } + defer C.BIO_free(bio) + rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) + if rsa == nil { + return nil, errors.New("failed getting rsa key") + } + defer C.RSA_free(rsa) + if int(C.PEM_write_bio_RSAPrivateKey(bio, rsa, nil, nil, C.int(0), nil, + nil)) != 1 { + return nil, errors.New("failed dumping private key") + } + return ioutil.ReadAll(asAnyBio(bio)) +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0.go new file mode 100644 index 00000000000..6ea2a46e073 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0.go @@ -0,0 +1,132 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +// #include "shim.h" +import "C" + +import ( + "errors" + "io/ioutil" + "runtime" +) + +type PublicKey interface { + // Verifies the data signature using PKCS1.15 + VerifyPKCS1v15(method Method, data, sig []byte) error + + // MarshalPKIXPublicKeyPEM converts the public key to PEM-encoded PKIX + // format + MarshalPKIXPublicKeyPEM() (pem_block []byte, err error) + + // MarshalPKIXPublicKeyDER converts the public key to DER-encoded PKIX + // format + MarshalPKIXPublicKeyDER() (der_block []byte, err error) + + // KeyType returns an identifier for what kind of key is represented by this + // object. + KeyType() NID + + // BaseType returns an identifier for what kind of key is represented + // by this object. + // Keys that share same algorithm but use different legacy formats + // will have the same BaseType. + // + // For example, a key with a `KeyType() == KeyTypeRSA` and a key with a + // `KeyType() == KeyTypeRSA2` would both have `BaseType() == KeyTypeRSA`. + BaseType() NID + + evpPKey() *C.EVP_PKEY +} + +func (key *pKey) MarshalPKCS1PrivateKeyPEM() (pem_block []byte, + err error) { + bio := C.BIO_new(C.BIO_s_mem()) + if bio == nil { + return nil, errors.New("failed to allocate memory BIO") + } + defer C.BIO_free(bio) + + // PEM_write_bio_PrivateKey_traditional will use the key-specific PKCS1 + // format if one is available for that key type, otherwise it will encode + // to a PKCS8 key. + if int(C.X_PEM_write_bio_PrivateKey_traditional(bio, key.key, nil, nil, + C.int(0), nil, nil)) != 1 { + return nil, errors.New("failed dumping private key") + } + + return ioutil.ReadAll(asAnyBio(bio)) +} + +func (key *pKey) KeyType() NID { + return NID(C.EVP_PKEY_id(key.key)) +} + +func (key *pKey) BaseType() NID { + return NID(C.EVP_PKEY_base_id(key.key)) +} + +// GenerateECKey generates a new elliptic curve private key on the speicified +// curve. +func GenerateECKey(curve EllipticCurve) (PrivateKey, error) { + + // Create context for parameter generation + paramCtx := C.EVP_PKEY_CTX_new_id(C.EVP_PKEY_EC, nil) + if paramCtx == nil { + return nil, errors.New("failed creating EC parameter generation context") + } + defer C.EVP_PKEY_CTX_free(paramCtx) + + // Intialize the parameter generation + if int(C.EVP_PKEY_paramgen_init(paramCtx)) != 1 { + return nil, errors.New("failed initializing EC parameter generation context") + } + + // Set curve in EC parameter generation context + if int(C.X_EVP_PKEY_CTX_set_ec_paramgen_curve_nid(paramCtx, C.int(curve))) != 1 { + return nil, errors.New("failed setting curve in EC parameter generation context") + } + + // Create parameter object + var params *C.EVP_PKEY + if int(C.EVP_PKEY_paramgen(paramCtx, ¶ms)) != 1 { + return nil, errors.New("failed creating EC key generation parameters") + } + defer C.EVP_PKEY_free(params) + + // Create context for the key generation + keyCtx := C.EVP_PKEY_CTX_new(params, nil) + if keyCtx == nil { + return nil, errors.New("failed creating EC key generation context") + } + defer C.EVP_PKEY_CTX_free(keyCtx) + + // Generate the key + var privKey *C.EVP_PKEY + if int(C.EVP_PKEY_keygen_init(keyCtx)) != 1 { + return nil, errors.New("failed initializing EC key generation context") + } + if int(C.EVP_PKEY_keygen(keyCtx, &privKey)) != 1 { + return nil, errors.New("failed generating EC private key") + } + + p := &pKey{key: privKey} + runtime.SetFinalizer(p, func(p *pKey) { + C.X_EVP_PKEY_free(p.key) + }) + return p, nil +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0_test.go new file mode 100644 index 00000000000..2a2eda887b7 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0_test.go @@ -0,0 +1,149 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +import ( + "bytes" + "crypto/ecdsa" + "crypto/tls" + "crypto/x509" + "encoding/hex" + pem_pkg "encoding/pem" + "io/ioutil" + "testing" +) + +func TestMarshalEC(t *testing.T) { + if !HasECDH() { + t.Skip("ECDH not available") + } + + key, err := LoadPrivateKeyFromPEM(prime256v1KeyBytes) + if err != nil { + t.Fatal(err) + } + cert, err := LoadCertificateFromPEM(prime256v1CertBytes) + if err != nil { + t.Fatal(err) + } + + privateBlock, _ := pem_pkg.Decode(prime256v1KeyBytes) + key, err = LoadPrivateKeyFromDER(privateBlock.Bytes) + if err != nil { + t.Fatal(err) + } + + pem, err := cert.MarshalPEM() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(pem, prime256v1CertBytes) { + ioutil.WriteFile("generated", pem, 0644) + ioutil.WriteFile("hardcoded", prime256v1CertBytes, 0644) + t.Fatal("invalid cert pem bytes") + } + + pem, err = key.MarshalPKCS1PrivateKeyPEM() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(pem, prime256v1KeyBytes) { + ioutil.WriteFile("generated", pem, 0644) + ioutil.WriteFile("hardcoded", prime256v1KeyBytes, 0644) + t.Fatal("invalid private key pem bytes") + } + tls_cert, err := tls.X509KeyPair(prime256v1CertBytes, prime256v1KeyBytes) + if err != nil { + t.Fatal(err) + } + tls_key, ok := tls_cert.PrivateKey.(*ecdsa.PrivateKey) + if !ok { + t.Fatal("FASDFASDF") + } + _ = tls_key + + der, err := key.MarshalPKCS1PrivateKeyDER() + if err != nil { + t.Fatal(err) + } + tls_der, err := x509.MarshalECPrivateKey(tls_key) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(der, tls_der) { + t.Fatalf("invalid private key der bytes: %s\n v.s. %s\n", + hex.Dump(der), hex.Dump(tls_der)) + } + + der, err = key.MarshalPKIXPublicKeyDER() + if err != nil { + t.Fatal(err) + } + tls_der, err = x509.MarshalPKIXPublicKey(&tls_key.PublicKey) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(der, tls_der) { + ioutil.WriteFile("generated", []byte(hex.Dump(der)), 0644) + ioutil.WriteFile("hardcoded", []byte(hex.Dump(tls_der)), 0644) + t.Fatal("invalid public key der bytes") + } + + pem, err = key.MarshalPKIXPublicKeyPEM() + if err != nil { + t.Fatal(err) + } + tls_pem := pem_pkg.EncodeToMemory(&pem_pkg.Block{ + Type: "PUBLIC KEY", Bytes: tls_der}) + if !bytes.Equal(pem, tls_pem) { + ioutil.WriteFile("generated", pem, 0644) + ioutil.WriteFile("hardcoded", tls_pem, 0644) + t.Fatal("invalid public key pem bytes") + } + + loaded_pubkey_from_pem, err := LoadPublicKeyFromPEM(pem) + if err != nil { + t.Fatal(err) + } + + loaded_pubkey_from_der, err := LoadPublicKeyFromDER(der) + if err != nil { + t.Fatal(err) + } + + new_der_from_pem, err := loaded_pubkey_from_pem.MarshalPKIXPublicKeyDER() + if err != nil { + t.Fatal(err) + } + + new_der_from_der, err := loaded_pubkey_from_der.MarshalPKIXPublicKeyDER() + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(new_der_from_der, tls_der) { + ioutil.WriteFile("generated", []byte(hex.Dump(new_der_from_der)), 0644) + ioutil.WriteFile("hardcoded", []byte(hex.Dump(tls_der)), 0644) + t.Fatal("invalid public key der bytes") + } + + if !bytes.Equal(new_der_from_pem, tls_der) { + ioutil.WriteFile("generated", []byte(hex.Dump(new_der_from_pem)), 0644) + ioutil.WriteFile("hardcoded", []byte(hex.Dump(tls_der)), 0644) + t.Fatal("invalid public key der bytes") + } +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_test.go index 0af90128530..635ef638ec9 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -76,7 +76,7 @@ func TestMarshal(t *testing.T) { } tls_der := x509.MarshalPKCS1PrivateKey(tls_key) if !bytes.Equal(der, tls_der) { - t.Fatal("invalid private key der bytes: %s\n v.s. %s\n", + t.Fatalf("invalid private key der bytes: %s\n v.s. %s\n", hex.Dump(der), hex.Dump(tls_der)) } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/mapping.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/mapping.go index 066aba6b5db..d78cc703472 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/mapping.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/mapping.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl import ( diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/net.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/net.go index 7120d065d15..15c897addd1 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/net.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/net.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/nid.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/nid.go index c80f237b605..6766b849e76 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/nid.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/nid.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Ryan Hileman +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package openssl type NID int const ( + NID_undef NID = 0 NID_rsadsi NID = 1 NID_pkcs NID = 2 NID_md2 NID = 3 @@ -196,4 +197,10 @@ const ( NID_ad_OCSP NID = 178 NID_ad_ca_issuers NID = 179 NID_OCSP_sign NID = 180 + NID_X9_62_id_ecPublicKey NID = 408 + NID_hmac NID = 855 + NID_cmac NID = 894 + NID_dhpublicnumber NID = 920 + NID_tls1_prf NID = 1021 + NID_hkdf NID = 1036 ) diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/oracle_stubs.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/oracle_stubs.go deleted file mode 100644 index 30492f3b9d8..00000000000 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/oracle_stubs.go +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright (C) 2014 Space Monkey, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !cgo - -package openssl - -import ( - "errors" - "net" - "time" -) - -const ( - SSLRecordSize = 16 * 1024 -) - -type Conn struct{} - -func Client(conn net.Conn, ctx *Ctx) (*Conn, error) -func Server(conn net.Conn, ctx *Ctx) (*Conn, error) - -func (c *Conn) Handshake() error -func (c *Conn) PeerCertificate() (*Certificate, error) -func (c *Conn) Close() error -func (c *Conn) Read(b []byte) (n int, err error) -func (c *Conn) Write(b []byte) (written int, err error) - -func (c *Conn) VerifyHostname(host string) error - -func (c *Conn) LocalAddr() net.Addr -func (c *Conn) RemoteAddr() net.Addr -func (c *Conn) SetDeadline(t time.Time) error -func (c *Conn) SetReadDeadline(t time.Time) error -func (c *Conn) SetWriteDeadline(t time.Time) error - -type Ctx struct{} - -type SSLVersion int - -const ( - SSLv3 SSLVersion = 0x02 - TLSv1 SSLVersion = 0x03 - TLSv1_1 SSLVersion = 0x04 - TLSv1_2 SSLVersion = 0x05 - AnyVersion SSLVersion = 0x06 -) - -func NewCtxWithVersion(version SSLVersion) (*Ctx, error) -func NewCtx() (*Ctx, error) -func NewCtxFromFiles(cert_file string, key_file string) (*Ctx, error) -func (c *Ctx) UseCertificate(cert *Certificate) error -func (c *Ctx) UsePrivateKey(key PrivateKey) error - -type CertificateStore struct{} - -func (c *Ctx) GetCertificateStore() *CertificateStore - -func (s *CertificateStore) AddCertificate(cert *Certificate) error - -func (c *Ctx) LoadVerifyLocations(ca_file string, ca_path string) error - -type Options int - -const ( - NoCompression Options = 0 - NoSSLv2 Options = 0 - NoSSLv3 Options = 0 - NoTLSv1 Options = 0 - CipherServerPreference Options = 0 - NoSessionResumptionOrRenegotiation Options = 0 - NoTicket Options = 0 -) - -func (c *Ctx) SetOptions(options Options) Options - -type Modes int - -const ( - ReleaseBuffers Modes = 0 -) - -func (c *Ctx) SetMode(modes Modes) Modes - -type VerifyOptions int - -const ( - VerifyNone VerifyOptions = 0 - VerifyPeer VerifyOptions = 0 - VerifyFailIfNoPeerCert VerifyOptions = 0 - VerifyClientOnce VerifyOptions = 0 -) - -func (c *Ctx) SetVerify(options VerifyOptions) -func (c *Ctx) SetVerifyDepth(depth int) -func (c *Ctx) SetSessionId(session_id []byte) error - -func (c *Ctx) SetCipherList(list string) error - -type SessionCacheModes int - -const ( - SessionCacheOff SessionCacheModes = 0 - SessionCacheClient SessionCacheModes = 0 - SessionCacheServer SessionCacheModes = 0 - SessionCacheBoth SessionCacheModes = 0 - NoAutoClear SessionCacheModes = 0 - NoInternalLookup SessionCacheModes = 0 - NoInternalStore SessionCacheModes = 0 - NoInternal SessionCacheModes = 0 -) - -func (c *Ctx) SetSessionCacheMode(modes SessionCacheModes) SessionCacheModes - -var ( - ValidationError = errors.New("Host validation error") -) - -type CheckFlags int - -const ( - AlwaysCheckSubject CheckFlags = 0 - NoWildcards CheckFlags = 0 -) - -func (c *Certificate) CheckHost(host string, flags CheckFlags) error -func (c *Certificate) CheckEmail(email string, flags CheckFlags) error -func (c *Certificate) CheckIP(ip net.IP, flags CheckFlags) error -func (c *Certificate) VerifyHostname(host string) error - -type PublicKey interface { - MarshalPKIXPublicKeyPEM() (pem_block []byte, err error) - MarshalPKIXPublicKeyDER() (der_block []byte, err error) - evpPKey() struct{} -} - -type PrivateKey interface { - PublicKey - MarshalPKCS1PrivateKeyPEM() (pem_block []byte, err error) - MarshalPKCS1PrivateKeyDER() (der_block []byte, err error) -} - -func LoadPrivateKeyFromPEM(pem_block []byte) (PrivateKey, error) - -type Certificate struct{} - -func LoadCertificateFromPEM(pem_block []byte) (*Certificate, error) - -func (c *Certificate) MarshalPEM() (pem_block []byte, err error) - -func (c *Certificate) PublicKey() (PublicKey, error) diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/pem.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/pem.go index 6dad5972dbd..c8b0c1cf19d 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/pem.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/pem.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Ryan Hileman +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1.go index 2592b6627d1..c227bee8461 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <errno.h> -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> - -#include "openssl/evp.h" -*/ +// #include "shim.h" import "C" import ( @@ -33,7 +24,7 @@ import ( ) type SHA1Hash struct { - ctx C.EVP_MD_CTX + ctx *C.EVP_MD_CTX engine *Engine } @@ -41,7 +32,10 @@ func NewSHA1Hash() (*SHA1Hash, error) { return NewSHA1HashWithEngine(nil) } func NewSHA1HashWithEngine(e *Engine) (*SHA1Hash, error) { hash := &SHA1Hash{engine: e} - C.EVP_MD_CTX_init(&hash.ctx) + hash.ctx = C.X_EVP_MD_CTX_new() + if hash.ctx == nil { + return nil, errors.New("openssl: sha1: unable to allocate ctx") + } runtime.SetFinalizer(hash, func(hash *SHA1Hash) { hash.Close() }) if err := hash.Reset(); err != nil { return nil, err @@ -50,7 +44,10 @@ func NewSHA1HashWithEngine(e *Engine) (*SHA1Hash, error) { } func (s *SHA1Hash) Close() { - C.EVP_MD_CTX_cleanup(&s.ctx) + if s.ctx != nil { + C.X_EVP_MD_CTX_free(s.ctx) + s.ctx = nil + } } func engineRef(e *Engine) *C.ENGINE { @@ -61,7 +58,7 @@ func engineRef(e *Engine) *C.ENGINE { } func (s *SHA1Hash) Reset() error { - if 1 != C.EVP_DigestInit_ex(&s.ctx, C.EVP_sha1(), engineRef(s.engine)) { + if 1 != C.X_EVP_DigestInit_ex(s.ctx, C.X_EVP_sha1(), engineRef(s.engine)) { return errors.New("openssl: sha1: cannot init digest ctx") } return nil @@ -71,7 +68,7 @@ func (s *SHA1Hash) Write(p []byte) (n int, err error) { if len(p) == 0 { return 0, nil } - if 1 != C.EVP_DigestUpdate(&s.ctx, unsafe.Pointer(&p[0]), + if 1 != C.X_EVP_DigestUpdate(s.ctx, unsafe.Pointer(&p[0]), C.size_t(len(p))) { return 0, errors.New("openssl: sha1: cannot update digest") } @@ -79,7 +76,7 @@ func (s *SHA1Hash) Write(p []byte) (n int, err error) { } func (s *SHA1Hash) Sum() (result [20]byte, err error) { - if 1 != C.EVP_DigestFinal_ex(&s.ctx, + if 1 != C.X_EVP_DigestFinal_ex(s.ctx, (*C.uchar)(unsafe.Pointer(&result[0])), nil) { return result, errors.New("openssl: sha1: cannot finalize ctx") } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1_test.go index 37037e4468b..37808b5a53e 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl import ( @@ -37,7 +35,7 @@ func TestSHA1(t *testing.T) { } if expected != got { - t.Fatal("exp:%x got:%x", expected, got) + t.Fatalf("exp:%x got:%x", expected, got) } } } @@ -75,7 +73,7 @@ func TestSHA1Writer(t *testing.T) { } if got != exp { - t.Fatal("exp:%x got:%x", exp, got) + t.Fatalf("exp:%x got:%x", exp, got) } } } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256.go index 6785b32f881..d25c7a959d7 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <errno.h> -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> - -#include "openssl/evp.h" -*/ +// #include "shim.h" import "C" import ( @@ -33,7 +24,7 @@ import ( ) type SHA256Hash struct { - ctx C.EVP_MD_CTX + ctx *C.EVP_MD_CTX engine *Engine } @@ -41,7 +32,10 @@ func NewSHA256Hash() (*SHA256Hash, error) { return NewSHA256HashWithEngine(nil) func NewSHA256HashWithEngine(e *Engine) (*SHA256Hash, error) { hash := &SHA256Hash{engine: e} - C.EVP_MD_CTX_init(&hash.ctx) + hash.ctx = C.X_EVP_MD_CTX_new() + if hash.ctx == nil { + return nil, errors.New("openssl: sha256: unable to allocate ctx") + } runtime.SetFinalizer(hash, func(hash *SHA256Hash) { hash.Close() }) if err := hash.Reset(); err != nil { return nil, err @@ -50,11 +44,14 @@ func NewSHA256HashWithEngine(e *Engine) (*SHA256Hash, error) { } func (s *SHA256Hash) Close() { - C.EVP_MD_CTX_cleanup(&s.ctx) + if s.ctx != nil { + C.X_EVP_MD_CTX_free(s.ctx) + s.ctx = nil + } } func (s *SHA256Hash) Reset() error { - if 1 != C.EVP_DigestInit_ex(&s.ctx, C.EVP_sha256(), engineRef(s.engine)) { + if 1 != C.X_EVP_DigestInit_ex(s.ctx, C.X_EVP_sha256(), engineRef(s.engine)) { return errors.New("openssl: sha256: cannot init digest ctx") } return nil @@ -64,7 +61,7 @@ func (s *SHA256Hash) Write(p []byte) (n int, err error) { if len(p) == 0 { return 0, nil } - if 1 != C.EVP_DigestUpdate(&s.ctx, unsafe.Pointer(&p[0]), + if 1 != C.X_EVP_DigestUpdate(s.ctx, unsafe.Pointer(&p[0]), C.size_t(len(p))) { return 0, errors.New("openssl: sha256: cannot update digest") } @@ -72,7 +69,7 @@ func (s *SHA256Hash) Write(p []byte) (n int, err error) { } func (s *SHA256Hash) Sum() (result [32]byte, err error) { - if 1 != C.EVP_DigestFinal_ex(&s.ctx, + if 1 != C.X_EVP_DigestFinal_ex(s.ctx, (*C.uchar)(unsafe.Pointer(&result[0])), nil) { return result, errors.New("openssl: sha256: cannot finalize ctx") } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256_test.go index 89df88afd44..467e503ab42 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl import ( @@ -37,7 +35,7 @@ func TestSHA256(t *testing.T) { } if expected != got { - t.Fatal("exp:%x got:%x", expected, got) + t.Fatalf("exp:%x got:%x", expected, got) } } } @@ -75,7 +73,7 @@ func TestSHA256Writer(t *testing.T) { } if got != exp { - t.Fatal("exp:%x got:%x", exp, got) + t.Fatalf("exp:%x got:%x", exp, got) } } } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.c new file mode 100644 index 00000000000..bb3239b0571 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.c @@ -0,0 +1,746 @@ +/* + * Copyright (C) 2014 Space Monkey, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <string.h> + +#include "shim.h" + +#include "_cgo_export.h" + +/* + * Functions defined in other .c files + */ +extern int go_init_locks(); +extern unsigned long go_thread_id_callback(); +extern void go_thread_locking_callback(int, int, const char*, int); +static int go_write_bio_puts(BIO *b, const char *str) { + return go_write_bio_write(b, (char*)str, (int)strlen(str)); +} + +/* + * Functions to convey openssl feature defines at runtime + */ +int X_OPENSSL_NO_ECDH() { +#ifdef OPENSSL_NO_ECDH + return 1; +#else + return 0; +#endif +} + +/* + ************************************************ + * v1.1.X and later implementation + ************************************************ + */ +#if OPENSSL_VERSION_NUMBER >= 0x1010000fL + +void X_BIO_set_data(BIO* bio, void* data) { + BIO_set_data(bio, data); +} + +void* X_BIO_get_data(BIO* bio) { + return BIO_get_data(bio); +} + +EVP_MD_CTX* X_EVP_MD_CTX_new() { + return EVP_MD_CTX_new(); +} + +void X_EVP_MD_CTX_free(EVP_MD_CTX* ctx) { + EVP_MD_CTX_free(ctx); +} + +static int x_bio_create(BIO *b) { + BIO_set_shutdown(b, 1); + BIO_set_init(b, 1); + BIO_set_data(b, NULL); + BIO_clear_flags(b, ~0); + return 1; +} + +static int x_bio_free(BIO *b) { + return 1; +} + +static BIO_METHOD *writeBioMethod; +static BIO_METHOD *readBioMethod; + +BIO_METHOD* BIO_s_readBio() { return readBioMethod; } +BIO_METHOD* BIO_s_writeBio() { return writeBioMethod; } + +int x_bio_init_methods() { + writeBioMethod = BIO_meth_new(BIO_TYPE_SOURCE_SINK, "Go Write BIO"); + if (!writeBioMethod) { + return 1; + } + if (1 != BIO_meth_set_write(writeBioMethod, + (int (*)(BIO *, const char *, int))go_write_bio_write)) { + return 2; + } + if (1 != BIO_meth_set_puts(writeBioMethod, go_write_bio_puts)) { + return 3; + } + if (1 != BIO_meth_set_ctrl(writeBioMethod, go_write_bio_ctrl)) { + return 4; + } + if (1 != BIO_meth_set_create(writeBioMethod, x_bio_create)) { + return 5; + } + if (1 != BIO_meth_set_destroy(writeBioMethod, x_bio_free)) { + return 6; + } + + readBioMethod = BIO_meth_new(BIO_TYPE_SOURCE_SINK, "Go Read BIO"); + if (!readBioMethod) { + return 7; + } + if (1 != BIO_meth_set_read(readBioMethod, go_read_bio_read)) { + return 8; + } + if (1 != BIO_meth_set_ctrl(readBioMethod, go_read_bio_ctrl)) { + return 9; + } + if (1 != BIO_meth_set_create(readBioMethod, x_bio_create)) { + return 10; + } + if (1 != BIO_meth_set_destroy(readBioMethod, x_bio_free)) { + return 11; + } + + return 0; +} + +const EVP_MD *X_EVP_dss() { + return NULL; +} + +const EVP_MD *X_EVP_dss1() { + return NULL; +} + +const EVP_MD *X_EVP_sha() { + return NULL; +} + +int X_EVP_CIPHER_CTX_encrypting(const EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_encrypting(ctx); +} + +int X_X509_add_ref(X509* x509) { + return X509_up_ref(x509); +} + +const ASN1_TIME *X_X509_get0_notBefore(const X509 *x) { + return X509_get0_notBefore(x); +} + +const ASN1_TIME *X_X509_get0_notAfter(const X509 *x) { + return X509_get0_notAfter(x); +} + +HMAC_CTX *X_HMAC_CTX_new(void) { + return HMAC_CTX_new(); +} + +void X_HMAC_CTX_free(HMAC_CTX *ctx) { + HMAC_CTX_free(ctx); +} + +int X_PEM_write_bio_PrivateKey_traditional(BIO *bio, EVP_PKEY *key, const EVP_CIPHER *enc, unsigned char *kstr, int klen, pem_password_cb *cb, void *u) { + return PEM_write_bio_PrivateKey_traditional(bio, key, enc, kstr, klen, cb, u); +} + +#endif + + + +/* + ************************************************ + * v1.0.X implementation + ************************************************ + */ +#if OPENSSL_VERSION_NUMBER < 0x1010000fL + +static int x_bio_create(BIO *b) { + b->shutdown = 1; + b->init = 1; + b->num = -1; + b->ptr = NULL; + b->flags = 0; + return 1; +} + +static int x_bio_free(BIO *b) { + return 1; +} + +static BIO_METHOD writeBioMethod = { + BIO_TYPE_SOURCE_SINK, + "Go Write BIO", + (int (*)(BIO *, const char *, int))go_write_bio_write, + NULL, + go_write_bio_puts, + NULL, + go_write_bio_ctrl, + x_bio_create, + x_bio_free, + NULL}; + +static BIO_METHOD* BIO_s_writeBio() { return &writeBioMethod; } + +static BIO_METHOD readBioMethod = { + BIO_TYPE_SOURCE_SINK, + "Go Read BIO", + NULL, + go_read_bio_read, + NULL, + NULL, + go_read_bio_ctrl, + x_bio_create, + x_bio_free, + NULL}; + +static BIO_METHOD* BIO_s_readBio() { return &readBioMethod; } + +int x_bio_init_methods() { + /* statically initialized above */ + return 0; +} + +void X_BIO_set_data(BIO* bio, void* data) { + bio->ptr = data; +} + +void* X_BIO_get_data(BIO* bio) { + return bio->ptr; +} + +EVP_MD_CTX* X_EVP_MD_CTX_new() { + return EVP_MD_CTX_create(); +} + +void X_EVP_MD_CTX_free(EVP_MD_CTX* ctx) { + EVP_MD_CTX_destroy(ctx); +} + +int X_X509_add_ref(X509* x509) { + CRYPTO_add(&x509->references, 1, CRYPTO_LOCK_X509); + return 1; +} + +const ASN1_TIME *X_X509_get0_notBefore(const X509 *x) { + return x->cert_info->validity->notBefore; +} + +const ASN1_TIME *X_X509_get0_notAfter(const X509 *x) { + return x->cert_info->validity->notAfter; +} + +const EVP_MD *X_EVP_dss() { + return EVP_dss(); +} + +const EVP_MD *X_EVP_dss1() { + return EVP_dss1(); +} + +const EVP_MD *X_EVP_sha() { + return EVP_sha(); +} + +int X_EVP_CIPHER_CTX_encrypting(const EVP_CIPHER_CTX *ctx) { + return ctx->encrypt; +} + +HMAC_CTX *X_HMAC_CTX_new(void) { + /* v1.1.0 uses a OPENSSL_zalloc to allocate the memory which does not exist + * in previous versions. malloc+memset to get the same behavior */ + HMAC_CTX *ctx = (HMAC_CTX *)OPENSSL_malloc(sizeof(HMAC_CTX)); + if (ctx) { + memset(ctx, 0, sizeof(HMAC_CTX)); + HMAC_CTX_init(ctx); + } + return ctx; +} + +void X_HMAC_CTX_free(HMAC_CTX *ctx) { + if (ctx) { + HMAC_CTX_cleanup(ctx); + OPENSSL_free(ctx); + } +} + +int X_PEM_write_bio_PrivateKey_traditional(BIO *bio, EVP_PKEY *key, const EVP_CIPHER *enc, unsigned char *kstr, int klen, pem_password_cb *cb, void *u) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + /* PEM_write_bio_PrivateKey always tries to use the PKCS8 format if it + * is available, instead of using the "traditional" format as stated in the + * OpenSSL man page. + * i2d_PrivateKey should give us the correct DER encoding, so we'll just + * use PEM_ASN1_write_bio directly to write the DER encoding with the correct + * type header. */ + + int ppkey_id, pkey_base_id, ppkey_flags; + const char *pinfo, *ppem_str; + char pem_type_str[80]; + + // Lookup the ASN1 method information to get the pem type + if (EVP_PKEY_asn1_get0_info(&ppkey_id, &pkey_base_id, &ppkey_flags, &pinfo, &ppem_str, key->ameth) != 1) { + return 0; + } + // Set up the PEM type string + if (BIO_snprintf(pem_type_str, 80, "%s PRIVATE KEY", ppem_str) <= 0) { + // Failed to write out the pem type string, something is really wrong. + return 0; + } + // Write out everything to the BIO + return PEM_ASN1_write_bio((i2d_of_void *)i2d_PrivateKey, + pem_type_str, bio, key, enc, kstr, klen, cb, u); +#else + return -1; +#endif +} + +#endif + + + +/* + ************************************************ + * common implementation + ************************************************ + */ + +int X_shim_init() { + int rc = 0; + + OPENSSL_config(NULL); + ENGINE_load_builtin_engines(); + SSL_load_error_strings(); + SSL_library_init(); + OpenSSL_add_all_algorithms(); + +#if OPENSSL_VERSION_NUMBER < 0x1010000fL + // Set up OPENSSL thread safety callbacks. + rc = go_init_locks(); + if (rc != 0) { + return rc; + } + CRYPTO_set_locking_callback(go_thread_locking_callback); + CRYPTO_set_id_callback(go_thread_id_callback); +#endif + rc = x_bio_init_methods(); + if (rc != 0) { + return rc; + } + + return 0; +} + +void * X_OPENSSL_malloc(size_t size) { + return OPENSSL_malloc(size); +} + +void X_OPENSSL_free(void *ref) { + OPENSSL_free(ref); +} + +long X_SSL_set_options(SSL* ssl, long options) { + return SSL_set_options(ssl, options); +} + +long X_SSL_get_options(SSL* ssl) { + return SSL_get_options(ssl); +} + +long X_SSL_clear_options(SSL* ssl, long options) { + return SSL_clear_options(ssl, options); +} + +long X_SSL_set_tlsext_host_name(SSL *ssl, const char *name) { + return SSL_set_tlsext_host_name(ssl, name); +} +const char * X_SSL_get_cipher_name(const SSL *ssl) { + return SSL_get_cipher_name(ssl); +} +int X_SSL_session_reused(SSL *ssl) { + return SSL_session_reused(ssl); +} + +int X_SSL_new_index() { + return SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL); +} + +int X_SSL_verify_cb(int ok, X509_STORE_CTX* store) { + SSL* ssl = (SSL *)X509_STORE_CTX_get_ex_data(store, + SSL_get_ex_data_X509_STORE_CTX_idx()); + void* p = SSL_get_ex_data(ssl, get_ssl_idx()); + // get the pointer to the go Ctx object and pass it back into the thunk + return go_ssl_verify_cb_thunk(p, ok, store); +} + +const SSL_METHOD *X_SSLv23_method() { + return SSLv23_method(); +} + +const SSL_METHOD *X_SSLv3_method() { +#ifndef OPENSSL_NO_SSL3_METHOD + return SSLv3_method(); +#else + return NULL; +#endif +} + +const SSL_METHOD *X_TLSv1_method() { + return TLSv1_method(); +} + +const SSL_METHOD *X_TLSv1_1_method() { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return TLSv1_1_method(); +#else + return NULL; +#endif +} + +const SSL_METHOD *X_TLSv1_2_method() { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return TLSv1_2_method(); +#else + return NULL; +#endif +} + +int X_SSL_CTX_new_index() { + return SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL); +} + +long X_SSL_CTX_set_options(SSL_CTX* ctx, long options) { + return SSL_CTX_set_options(ctx, options); +} + +long X_SSL_CTX_clear_options(SSL_CTX* ctx, long options) { + return SSL_CTX_clear_options(ctx, options); +} + +long X_SSL_CTX_get_options(SSL_CTX* ctx) { + return SSL_CTX_get_options(ctx); +} + +long X_SSL_CTX_set_mode(SSL_CTX* ctx, long modes) { + return SSL_CTX_set_mode(ctx, modes); +} + +long X_SSL_CTX_get_mode(SSL_CTX* ctx) { + return SSL_CTX_get_mode(ctx); +} + +long X_SSL_CTX_set_session_cache_mode(SSL_CTX* ctx, long modes) { + return SSL_CTX_set_session_cache_mode(ctx, modes); +} + +long X_SSL_CTX_sess_set_cache_size(SSL_CTX* ctx, long t) { + return SSL_CTX_sess_set_cache_size(ctx, t); +} + +long X_SSL_CTX_sess_get_cache_size(SSL_CTX* ctx) { + return SSL_CTX_sess_get_cache_size(ctx); +} + +long X_SSL_CTX_set_timeout(SSL_CTX* ctx, long t) { + return SSL_CTX_set_timeout(ctx, t); +} + +long X_SSL_CTX_get_timeout(SSL_CTX* ctx) { + return SSL_CTX_get_timeout(ctx); +} + +long X_SSL_CTX_add_extra_chain_cert(SSL_CTX* ctx, X509 *cert) { + return SSL_CTX_add_extra_chain_cert(ctx, cert); +} + +long X_SSL_CTX_set_tlsext_servername_callback( + SSL_CTX* ctx, int (*cb)(SSL *con, int *ad, void *args)) { + return SSL_CTX_set_tlsext_servername_callback(ctx, cb); +} + +int X_SSL_CTX_verify_cb(int ok, X509_STORE_CTX* store) { + SSL* ssl = (SSL *)X509_STORE_CTX_get_ex_data(store, + SSL_get_ex_data_X509_STORE_CTX_idx()); + SSL_CTX* ssl_ctx = SSL_get_SSL_CTX(ssl); + void* p = SSL_CTX_get_ex_data(ssl_ctx, get_ssl_ctx_idx()); + // get the pointer to the go Ctx object and pass it back into the thunk + return go_ssl_ctx_verify_cb_thunk(p, ok, store); +} + +long X_SSL_CTX_set_tmp_dh(SSL_CTX* ctx, DH *dh) { + return SSL_CTX_set_tmp_dh(ctx, dh); +} + +long X_PEM_read_DHparams(SSL_CTX* ctx, DH *dh) { + return SSL_CTX_set_tmp_dh(ctx, dh); +} + +int X_SSL_CTX_set_tlsext_ticket_key_cb(SSL_CTX *sslctx, + int (*cb)(SSL *s, unsigned char key_name[16], + unsigned char iv[EVP_MAX_IV_LENGTH], + EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc)) { + return SSL_CTX_set_tlsext_ticket_key_cb(sslctx, cb); +} + +int X_SSL_CTX_ticket_key_cb(SSL *s, unsigned char key_name[16], + unsigned char iv[EVP_MAX_IV_LENGTH], + EVP_CIPHER_CTX *cctx, HMAC_CTX *hctx, int enc) { + + SSL_CTX* ssl_ctx = SSL_get_SSL_CTX(s); + void* p = SSL_CTX_get_ex_data(ssl_ctx, get_ssl_ctx_idx()); + // get the pointer to the go Ctx object and pass it back into the thunk + return go_ticket_key_cb_thunk(p, s, key_name, iv, cctx, hctx, enc); +} + +int X_BIO_get_flags(BIO *b) { + return BIO_get_flags(b); +} + +void X_BIO_set_flags(BIO *b, int flags) { + return BIO_set_flags(b, flags); +} + +void X_BIO_clear_flags(BIO *b, int flags) { + BIO_clear_flags(b, flags); +} + +int X_BIO_read(BIO *b, void *buf, int len) { + return BIO_read(b, buf, len); +} + +int X_BIO_write(BIO *b, const void *buf, int len) { + return BIO_write(b, buf, len); +} + +BIO *X_BIO_new_write_bio() { + return BIO_new(BIO_s_writeBio()); +} + +BIO *X_BIO_new_read_bio() { + return BIO_new(BIO_s_readBio()); +} + +const EVP_MD *X_EVP_get_digestbyname(const char *name) { + return EVP_get_digestbyname(name); +} + +const EVP_MD *X_EVP_md_null() { + return EVP_md_null(); +} + +const EVP_MD *X_EVP_md5() { + return EVP_md5(); +} + +const EVP_MD *X_EVP_ripemd160() { + return EVP_ripemd160(); +} + +const EVP_MD *X_EVP_sha224() { + return EVP_sha224(); +} + +const EVP_MD *X_EVP_sha1() { + return EVP_sha1(); +} + +const EVP_MD *X_EVP_sha256() { + return EVP_sha256(); +} + +const EVP_MD *X_EVP_sha384() { + return EVP_sha384(); +} + +const EVP_MD *X_EVP_sha512() { + return EVP_sha512(); +} + +int X_EVP_MD_size(const EVP_MD *md) { + return EVP_MD_size(md); +} + +int X_EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl) { + return EVP_DigestInit_ex(ctx, type, impl); +} + +int X_EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *d, size_t cnt) { + return EVP_DigestUpdate(ctx, d, cnt); +} + +int X_EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s) { + return EVP_DigestFinal_ex(ctx, md, s); +} + +int X_EVP_SignInit(EVP_MD_CTX *ctx, const EVP_MD *type) { + return EVP_SignInit(ctx, type); +} + +int X_EVP_SignUpdate(EVP_MD_CTX *ctx, const void *d, unsigned int cnt) { + return EVP_SignUpdate(ctx, d, cnt); +} + +EVP_PKEY *X_EVP_PKEY_new(void) { + return EVP_PKEY_new(); +} + +void X_EVP_PKEY_free(EVP_PKEY *pkey) { + EVP_PKEY_free(pkey); +} + +int X_EVP_PKEY_size(EVP_PKEY *pkey) { + return EVP_PKEY_size(pkey); +} + +struct rsa_st *X_EVP_PKEY_get1_RSA(EVP_PKEY *pkey) { + return EVP_PKEY_get1_RSA(pkey); +} + +int X_EVP_PKEY_set1_RSA(EVP_PKEY *pkey, struct rsa_st *key) { + return EVP_PKEY_set1_RSA(pkey, key); +} + +int X_EVP_PKEY_assign_charp(EVP_PKEY *pkey, int type, char *key) { + return EVP_PKEY_assign(pkey, type, key); +} + + + +int X_EVP_SignFinal(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s, EVP_PKEY *pkey) { + return EVP_SignFinal(ctx, md, s, pkey); +} + +int X_EVP_VerifyInit(EVP_MD_CTX *ctx, const EVP_MD *type) { + return EVP_VerifyInit(ctx, type); +} + +int X_EVP_VerifyUpdate(EVP_MD_CTX *ctx, const void *d, + unsigned int cnt) { + return EVP_VerifyUpdate(ctx, d, cnt); +} + +int X_EVP_VerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sigbuf, unsigned int siglen, EVP_PKEY *pkey) { + return EVP_VerifyFinal(ctx, sigbuf, siglen, pkey); +} + +int X_EVP_CIPHER_block_size(EVP_CIPHER *c) { + return EVP_CIPHER_block_size(c); +} + +int X_EVP_CIPHER_key_length(EVP_CIPHER *c) { + return EVP_CIPHER_key_length(c); +} + +int X_EVP_CIPHER_iv_length(EVP_CIPHER *c) { + return EVP_CIPHER_iv_length(c); +} + +int X_EVP_CIPHER_nid(EVP_CIPHER *c) { + return EVP_CIPHER_nid(c); +} + +int X_EVP_CIPHER_CTX_block_size(EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_block_size(ctx); +} + +int X_EVP_CIPHER_CTX_key_length(EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_key_length(ctx); +} + +int X_EVP_CIPHER_CTX_iv_length(EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_iv_length(ctx); +} + +const EVP_CIPHER *X_EVP_CIPHER_CTX_cipher(EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_cipher(ctx); +} + +#if OPENSSL_VERSION_NUMBER > 0x10000000L +#ifndef OPENSSL_NO_EC +int X_EVP_PKEY_CTX_set_ec_paramgen_curve_nid(EVP_PKEY_CTX *ctx, int nid) { + return EVP_PKEY_CTX_set_ec_paramgen_curve_nid(ctx, nid); +} +#else +int X_EVP_PKEY_CTX_set_ec_paramgen_curve_nid(EVP_PKEY_CTX *ctx, int nid) { + return -2; // not supported +} +#endif +#endif + +// END HERE + +size_t X_HMAC_size(const HMAC_CTX *e) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return HMAC_size(e); +#else + return 0; +#endif +} + +int X_HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len, const EVP_MD *md, ENGINE *impl) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return HMAC_Init_ex(ctx, key, len, md, impl); +#else + return -1; +#endif +} + +int X_HMAC_Update(HMAC_CTX *ctx, const unsigned char *data, size_t len) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return HMAC_Update(ctx, data, len); +#else + return -1; +#endif +} + +int X_HMAC_Final(HMAC_CTX *ctx, unsigned char *md, unsigned int *len) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return HMAC_Final(ctx, md, len); +#else + return -1; +#endif +} + +int X_sk_X509_num(STACK_OF(X509) *sk) { + return sk_X509_num(sk); +} + +X509 *X_sk_X509_value(STACK_OF(X509)* sk, int i) { + return sk_X509_value(sk, i); +} + +#ifdef OPENSSL_FIPS +int X_FIPS_mode(void) { + return FIPS_mode(); +} +int X_FIPS_mode_set(int r) { + return FIPS_mode_set(r); +} +#else +int X_FIPS_mode(void) { + return 0; +} +int X_FIPS_mode_set(int r) { + return 0; +} +#endif diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.h b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.h new file mode 100644 index 00000000000..1e9ddebe8ab --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.h @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2014 Space Monkey, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <stdlib.h> +#include <string.h> + +#include <openssl/opensslconf.h> + +#include <openssl/bio.h> +#include <openssl/conf.h> +#include <openssl/crypto.h> +#include <openssl/dh.h> +#include <openssl/engine.h> +#include <openssl/err.h> +#include <openssl/evp.h> +#include <openssl/hmac.h> +#include <openssl/pem.h> +#include <openssl/ssl.h> +#include <openssl/x509v3.h> + +#ifndef SSL_MODE_RELEASE_BUFFERS +#define SSL_MODE_RELEASE_BUFFERS 0 +#endif + +#ifndef SSL_OP_NO_COMPRESSION +#define SSL_OP_NO_COMPRESSION 0 +#endif + +#ifndef SSL_OP_NO_TLSv1_1 +#define SSL_OP_NO_TLSv1_1 0 +#endif + +#ifndef SSL_OP_NO_TLSv1_2 +#define SSL_OP_NO_TLSv1_2 0 +#endif + +/* shim methods */ +extern int X_shim_init(); + +/* Feature detection methods */ +extern int X_OPENSSL_NO_ECDH(); + +/* Library methods */ +extern void X_OPENSSL_free(void *ref); +extern void *X_OPENSSL_malloc(size_t size); + +/* SSL methods */ +extern long X_SSL_set_options(SSL* ssl, long options); +extern long X_SSL_get_options(SSL* ssl); +extern long X_SSL_clear_options(SSL* ssl, long options); +extern long X_SSL_set_tlsext_host_name(SSL *ssl, const char *name); +extern const char * X_SSL_get_cipher_name(const SSL *ssl); +extern int X_SSL_session_reused(SSL *ssl); +extern int X_SSL_new_index(); + +extern const SSL_METHOD *X_SSLv23_method(); +extern const SSL_METHOD *X_SSLv3_method(); +extern const SSL_METHOD *X_TLSv1_method(); +extern const SSL_METHOD *X_TLSv1_1_method(); +extern const SSL_METHOD *X_TLSv1_2_method(); + +#if defined SSL_CTRL_SET_TLSEXT_HOSTNAME +extern int sni_cb(SSL *ssl_conn, int *ad, void *arg); +#endif +extern int X_SSL_verify_cb(int ok, X509_STORE_CTX* store); + +/* SSL_CTX methods */ +extern int X_SSL_CTX_new_index(); +extern long X_SSL_CTX_set_options(SSL_CTX* ctx, long options); +extern long X_SSL_CTX_clear_options(SSL_CTX* ctx, long options); +extern long X_SSL_CTX_get_options(SSL_CTX* ctx); +extern long X_SSL_CTX_set_mode(SSL_CTX* ctx, long modes); +extern long X_SSL_CTX_get_mode(SSL_CTX* ctx); +extern long X_SSL_CTX_set_session_cache_mode(SSL_CTX* ctx, long modes); +extern long X_SSL_CTX_sess_set_cache_size(SSL_CTX* ctx, long t); +extern long X_SSL_CTX_sess_get_cache_size(SSL_CTX* ctx); +extern long X_SSL_CTX_set_timeout(SSL_CTX* ctx, long t); +extern long X_SSL_CTX_get_timeout(SSL_CTX* ctx); +extern long X_SSL_CTX_add_extra_chain_cert(SSL_CTX* ctx, X509 *cert); +extern long X_SSL_CTX_set_tlsext_servername_callback(SSL_CTX* ctx, int (*cb)(SSL *con, int *ad, void *args)); +extern int X_SSL_CTX_verify_cb(int ok, X509_STORE_CTX* store); +extern long X_SSL_CTX_set_tmp_dh(SSL_CTX* ctx, DH *dh); +extern long X_PEM_read_DHparams(SSL_CTX* ctx, DH *dh); +extern int X_SSL_CTX_set_tlsext_ticket_key_cb(SSL_CTX *sslctx, + int (*cb)(SSL *s, unsigned char key_name[16], + unsigned char iv[EVP_MAX_IV_LENGTH], + EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc)); +extern int X_SSL_CTX_ticket_key_cb(SSL *s, unsigned char key_name[16], + unsigned char iv[EVP_MAX_IV_LENGTH], + EVP_CIPHER_CTX *cctx, HMAC_CTX *hctx, int enc); + +/* BIO methods */ +extern int X_BIO_get_flags(BIO *b); +extern void X_BIO_set_flags(BIO *bio, int flags); +extern void X_BIO_clear_flags(BIO *bio, int flags); +extern void X_BIO_set_data(BIO *bio, void* data); +extern void *X_BIO_get_data(BIO *bio); +extern int X_BIO_read(BIO *b, void *buf, int len); +extern int X_BIO_write(BIO *b, const void *buf, int len); +extern BIO *X_BIO_new_write_bio(); +extern BIO *X_BIO_new_read_bio(); + +/* EVP methods */ +extern const EVP_MD *X_EVP_get_digestbyname(const char *name); +extern EVP_MD_CTX *X_EVP_MD_CTX_new(); +extern void X_EVP_MD_CTX_free(EVP_MD_CTX *ctx); +extern const EVP_MD *X_EVP_md_null(); +extern const EVP_MD *X_EVP_md5(); +extern const EVP_MD *X_EVP_sha(); +extern const EVP_MD *X_EVP_sha1(); +extern const EVP_MD *X_EVP_dss(); +extern const EVP_MD *X_EVP_dss1(); +extern const EVP_MD *X_EVP_ripemd160(); +extern const EVP_MD *X_EVP_sha224(); +extern const EVP_MD *X_EVP_sha256(); +extern const EVP_MD *X_EVP_sha384(); +extern const EVP_MD *X_EVP_sha512(); +extern int X_EVP_MD_size(const EVP_MD *md); +extern int X_EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl); +extern int X_EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *d, size_t cnt); +extern int X_EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s); +extern int X_EVP_SignInit(EVP_MD_CTX *ctx, const EVP_MD *type); +extern int X_EVP_SignUpdate(EVP_MD_CTX *ctx, const void *d, unsigned int cnt); +extern EVP_PKEY *X_EVP_PKEY_new(void); +extern void X_EVP_PKEY_free(EVP_PKEY *pkey); +extern int X_EVP_PKEY_size(EVP_PKEY *pkey); +extern struct rsa_st *X_EVP_PKEY_get1_RSA(EVP_PKEY *pkey); +extern int X_EVP_PKEY_set1_RSA(EVP_PKEY *pkey, struct rsa_st *key); +extern int X_EVP_PKEY_assign_charp(EVP_PKEY *pkey, int type, char *key); +extern int X_EVP_SignFinal(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s, EVP_PKEY *pkey); +extern int X_EVP_VerifyInit(EVP_MD_CTX *ctx, const EVP_MD *type); +extern int X_EVP_VerifyUpdate(EVP_MD_CTX *ctx, const void *d, unsigned int cnt); +extern int X_EVP_VerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sigbuf, unsigned int siglen, EVP_PKEY *pkey); +extern int X_EVP_CIPHER_block_size(EVP_CIPHER *c); +extern int X_EVP_CIPHER_key_length(EVP_CIPHER *c); +extern int X_EVP_CIPHER_iv_length(EVP_CIPHER *c); +extern int X_EVP_CIPHER_nid(EVP_CIPHER *c); +extern int X_EVP_CIPHER_CTX_block_size(EVP_CIPHER_CTX *ctx); +extern int X_EVP_CIPHER_CTX_key_length(EVP_CIPHER_CTX *ctx); +extern int X_EVP_CIPHER_CTX_iv_length(EVP_CIPHER_CTX *ctx); +extern const EVP_CIPHER *X_EVP_CIPHER_CTX_cipher(EVP_CIPHER_CTX *ctx); +extern int X_EVP_CIPHER_CTX_encrypting(const EVP_CIPHER_CTX *ctx); +#if OPENSSL_VERSION_NUMBER > 0x10000000L +extern int X_EVP_PKEY_CTX_set_ec_paramgen_curve_nid(EVP_PKEY_CTX *ctx, int nid); +#endif + +/* HMAC methods */ +extern size_t X_HMAC_size(const HMAC_CTX *e); +extern HMAC_CTX *X_HMAC_CTX_new(void); +extern void X_HMAC_CTX_free(HMAC_CTX *ctx); +extern int X_HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len, const EVP_MD *md, ENGINE *impl); +extern int X_HMAC_Update(HMAC_CTX *ctx, const unsigned char *data, size_t len); +extern int X_HMAC_Final(HMAC_CTX *ctx, unsigned char *md, unsigned int *len); + +/* X509 methods */ +extern int X_X509_add_ref(X509* x509); +extern const ASN1_TIME *X_X509_get0_notBefore(const X509 *x); +extern const ASN1_TIME *X_X509_get0_notAfter(const X509 *x); +extern int X_sk_X509_num(STACK_OF(X509) *sk); +extern X509 *X_sk_X509_value(STACK_OF(X509)* sk, int i); + +/* PEM methods */ +extern int X_PEM_write_bio_PrivateKey_traditional(BIO *bio, EVP_PKEY *key, const EVP_CIPHER *enc, unsigned char *kstr, int klen, pem_password_cb *cb, void *u); + +/* FIPS methods */ +extern int X_FIPS_mode(void); +extern int X_FIPS_mode_set(int r); diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni.c index 5398da869b8..f9e8d16b0e3 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni.c +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni.c @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni_test.go index ee3b1a8bbaf..09e831a45c9 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl.go index 3cc630601d3..117c30c0f99 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,30 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <openssl/crypto.h> -#include <openssl/ssl.h> -#include <openssl/err.h> -#include <openssl/conf.h> - -static long SSL_set_options_not_a_macro(SSL* ssl, long options) { - return SSL_set_options(ssl, options); -} - -static long SSL_get_options_not_a_macro(SSL* ssl) { - return SSL_get_options(ssl); -} - -static long SSL_clear_options_not_a_macro(SSL* ssl, long options) { - return SSL_clear_options(ssl, options); -} - -extern int verify_ssl_cb(int ok, X509_STORE_CTX* store); -*/ +// #include "shim.h" import "C" import ( @@ -53,7 +32,7 @@ const ( ) var ( - ssl_idx = C.SSL_get_ex_new_index(0, nil, nil, nil, nil) + ssl_idx = C.X_SSL_new_index() ) //export get_ssl_idx @@ -66,8 +45,8 @@ type SSL struct { verify_cb VerifyCallback } -//export verify_ssl_cb_thunk -func verify_ssl_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { +//export go_ssl_verify_cb_thunk +func go_ssl_verify_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { defer func() { if err := recover(); err != nil { logger.Critf("openssl: verify callback panic'd: %v", err) @@ -96,19 +75,19 @@ func (s *SSL) GetServername() string { // GetOptions returns SSL options. See // https://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (s *SSL) GetOptions() Options { - return Options(C.SSL_get_options_not_a_macro(s.ssl)) + return Options(C.X_SSL_get_options(s.ssl)) } // SetOptions sets SSL options. See // https://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (s *SSL) SetOptions(options Options) Options { - return Options(C.SSL_set_options_not_a_macro(s.ssl, C.long(options))) + return Options(C.X_SSL_set_options(s.ssl, C.long(options))) } // ClearOptions clear SSL options. See // https://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (s *SSL) ClearOptions(options Options) Options { - return Options(C.SSL_clear_options_not_a_macro(s.ssl, C.long(options))) + return Options(C.X_SSL_clear_options(s.ssl, C.long(options))) } // SetVerify controls peer verification settings. See @@ -116,7 +95,7 @@ func (s *SSL) ClearOptions(options Options) Options { func (s *SSL) SetVerify(options VerifyOptions, verify_cb VerifyCallback) { s.verify_cb = verify_cb if verify_cb != nil { - C.SSL_set_verify(s.ssl, C.int(options), (*[0]byte)(C.verify_ssl_cb)) + C.SSL_set_verify(s.ssl, C.int(options), (*[0]byte)(C.X_SSL_verify_cb)) } else { C.SSL_set_verify(s.ssl, C.int(options), nil) } @@ -131,7 +110,7 @@ func (s *SSL) SetVerifyMode(options VerifyOptions) { // SetVerifyCallback controls peer verification setting. See // http://www.openssl.org/docs/ssl/SSL_CTX_set_verify.html func (s *SSL) SetVerifyCallback(verify_cb VerifyCallback) { - s.SetVerify(s.VerifyMode(), s.verify_cb) + s.SetVerify(s.VerifyMode(), verify_cb) } // GetVerifyCallback returns callback function. See diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl_test.go index 0c088c2eed0..fe2e0de4592 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -81,6 +81,29 @@ ucCCa4lOGgPtXJ0Qf1c8yq5vh4yqkQjrgUTkr+CFDGR6y4CxmNDQxEMYIajaIiSY qmgvgyRayemfO2zR0CPgC6wSoGBth+xW6g+WA8y0z76ZSaWpFi8lVM4= -----END RSA PRIVATE KEY----- `) + prime256v1KeyBytes = []byte(`-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIB/XL0zZSsAu+IQF1AI/nRneabb2S126WFlvvhzmYr1KoAoGCCqGSM49 +AwEHoUQDQgAESSFGWwF6W1hoatKGPPorh4+ipyk0FqpiWdiH+4jIiU39qtOeZGSh +1QgSbzfdHxvoYI0FXM+mqE7wec0kIvrrHw== +-----END EC PRIVATE KEY----- +`) + prime256v1CertBytes = []byte(`-----BEGIN CERTIFICATE----- +MIIChTCCAiqgAwIBAgIJAOQII2LQl4uxMAoGCCqGSM49BAMCMIGcMQswCQYDVQQG +EwJVUzEPMA0GA1UECAwGS2Fuc2FzMRAwDgYDVQQHDAdOb3doZXJlMR8wHQYDVQQK +DBZGYWtlIENlcnRpZmljYXRlcywgSW5jMUkwRwYDVQQDDEBhMWJkZDVmZjg5ZjQy +N2IwZmNiOTdlNDMyZTY5Nzg2NjI2ODJhMWUyNzM4MDhkODE0ZWJiZjY4ODBlYzA3 +NDljMB4XDTE3MTIxNTIwNDU1MVoXDTI3MTIxMzIwNDU1MVowgZwxCzAJBgNVBAYT +AlVTMQ8wDQYDVQQIDAZLYW5zYXMxEDAOBgNVBAcMB05vd2hlcmUxHzAdBgNVBAoM +FkZha2UgQ2VydGlmaWNhdGVzLCBJbmMxSTBHBgNVBAMMQGExYmRkNWZmODlmNDI3 +YjBmY2I5N2U0MzJlNjk3ODY2MjY4MmExZTI3MzgwOGQ4MTRlYmJmNjg4MGVjMDc0 +OWMwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARJIUZbAXpbWGhq0oY8+iuHj6Kn +KTQWqmJZ2If7iMiJTf2q055kZKHVCBJvN90fG+hgjQVcz6aoTvB5zSQi+usfo1Mw +UTAdBgNVHQ4EFgQUfRYAFhlGM1wzvusyGrm26Vrbqm4wHwYDVR0jBBgwFoAUfRYA +FhlGM1wzvusyGrm26Vrbqm4wDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNJ +ADBGAiEA6PWNjm4B6zs3Wcha9qyDdfo1ILhHfk9rZEAGrnfyc2UCIQD1IDVJUkI4 +J/QVoOtP5DOdRPs/3XFy0Bk0qH+Uj5D7LQ== +-----END CERTIFICATE----- +`) ) func NetPipe(t testing.TB) (net.Conn, net.Conn) { diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.go index 23dc3e08305..a064d38592f 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.go @@ -1,4 +1,4 @@ -// Copyright (C) 2015 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,26 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <openssl/ssl.h> -#include <openssl/evp.h> - -static int SSL_CTX_set_tlsext_ticket_key_cb_not_a_macro(SSL_CTX *sslctx, - int (*cb)(SSL *s, unsigned char key_name[16], - unsigned char iv[EVP_MAX_IV_LENGTH], - EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc)) { - - return SSL_CTX_set_tlsext_ticket_key_cb(sslctx, cb); -} - -extern int ticket_key_cb(SSL *s, unsigned char key_name[16], - unsigned char iv[EVP_MAX_IV_LENGTH], - EVP_CIPHER_CTX *cctx, HMAC_CTX *hctx, int enc); -*/ +// #include "shim.h" import "C" import ( @@ -131,8 +114,8 @@ const ( ticket_req_lookupSession = 0 ) -//export ticket_key_cb_thunk -func ticket_key_cb_thunk(p unsafe.Pointer, s *C.SSL, key_name *C.uchar, +//export go_ticket_key_cb_thunk +func go_ticket_key_cb_thunk(p unsafe.Pointer, s *C.SSL, key_name *C.uchar, iv *C.uchar, cctx *C.EVP_CIPHER_CTX, hctx *C.HMAC_CTX, enc C.int) C.int { // no panic's allowed. it's super hard to guarantee any state at this point @@ -231,9 +214,9 @@ func (c *Ctx) SetTicketStore(store *TicketStore) { c.ticket_store = store if store == nil { - C.SSL_CTX_set_tlsext_ticket_key_cb_not_a_macro(c.ctx, nil) + C.X_SSL_CTX_set_tlsext_ticket_key_cb(c.ctx, nil) } else { - C.SSL_CTX_set_tlsext_ticket_key_cb_not_a_macro(c.ctx, - (*[0]byte)(C.ticket_key_cb)) + C.X_SSL_CTX_set_tlsext_ticket_key_cb(c.ctx, + (*[0]byte)(C.X_SSL_CTX_ticket_key_cb)) } } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/verify.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/verify.c deleted file mode 100644 index d55866c4cf0..00000000000 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/verify.c +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (C) 2014 Space Monkey, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include <openssl/ssl.h> -#include "_cgo_export.h" - -int verify_cb(int ok, X509_STORE_CTX* store) { - SSL* ssl = (SSL *)X509_STORE_CTX_get_app_data(store); - SSL_CTX* ssl_ctx = SSL_get_SSL_CTX(ssl); - void* p = SSL_CTX_get_ex_data(ssl_ctx, get_ssl_ctx_idx()); - // get the pointer to the go Ctx object and pass it back into the thunk - return verify_cb_thunk(p, ok, store); -} - -int verify_ssl_cb(int ok, X509_STORE_CTX* store) { - SSL* ssl = (SSL *)X509_STORE_CTX_get_app_data(store); - void* p = SSL_get_ex_data(ssl, get_ssl_idx()); - // get the pointer to the go Ctx object and pass it back into the thunk - return verify_ssl_cb_thunk(p, ok, store); -} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version.go index 8f3d392cde8..86501c696d6 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version.go @@ -17,6 +17,11 @@ package openssl // #include <openssl/opensslv.h> +// #include <openssl/crypto.h> import "C" -const Version string = C.OPENSSL_VERSION_TEXT +const BuildVersion string = C.OPENSSL_VERSION_TEXT + +var Version string = C.GoString(C.SSLeay_version(C.SSLEAY_VERSION)) + +var VersionNumber uint32 = uint32(C.SSLeay()) diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version_test.go new file mode 100644 index 00000000000..9877fb9c7dd --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version_test.go @@ -0,0 +1,29 @@ +// Copyright (C) MongoDB, Inc. 2018-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package openssl + +import ( + "testing" +) + +func TestVersion(t *testing.T) { + v := Version + b := BuildVersion + x := VersionNumber + if len(v) == 0 { + t.Fatal("Version string is empty") + } + if len(b) == 0 { + t.Fatal("BuildVersion string is empty") + } + if x == 0 { + t.Fatal("VersionNumber is zero") + } + t.Logf("Built with headers from: %s", BuildVersion) + t.Logf(" Tests linked against: %s", Version) + t.Logf(" Linked hex version is: %x", VersionNumber) +} diff --git a/src/mongo/rpc/metadata/client_metadata_ismaster.cpp b/src/mongo/rpc/metadata/client_metadata_ismaster.cpp index 234f61c78e8..8a6e84f2eef 100644 --- a/src/mongo/rpc/metadata/client_metadata_ismaster.cpp +++ b/src/mongo/rpc/metadata/client_metadata_ismaster.cpp @@ -65,17 +65,30 @@ const boost::optional<ClientMetadata>& ClientMetadataIsMasterState::getClientMet return _clientMetadata; } -void ClientMetadataIsMasterState::setClientMetadata( - Client* client, boost::optional<ClientMetadata> clientMetadata) { +void ClientMetadataIsMasterState::setClientMetadata(Client* client, + boost::optional<ClientMetadata> clientMetadata, + bool setViaMetadata) { auto& state = get(client); stdx::lock_guard<Client> lk(*client); state._clientMetadata = std::move(clientMetadata); + state._setViaMetadata = setViaMetadata; } Status ClientMetadataIsMasterState::readFromMetadata(OperationContext* txn, BSONElement& element) { + auto& clientMetadataIsMasterState = ClientMetadataIsMasterState::get(txn->getClient()); + + // If client metadata is not present in network requests, reset the in-memory metadata to be + // blank so that the wrong + // app name is not propagated. if (element.eoo()) { + auto client = txn->getClient(); + + if (clientMetadataIsMasterState._setViaMetadata && !client->isInDirectClient()) { + clientMetadataIsMasterState.setClientMetadata(client, boost::none, true); + } + return Status::OK(); } @@ -85,10 +98,8 @@ Status ClientMetadataIsMasterState::readFromMetadata(OperationContext* txn, BSON return swParseClientMetadata.getStatus(); } - auto& clientMetadataIsMasterState = ClientMetadataIsMasterState::get(txn->getClient()); - - clientMetadataIsMasterState.setClientMetadata(txn->getClient(), - std::move(swParseClientMetadata.getValue())); + clientMetadataIsMasterState.setClientMetadata( + txn->getClient(), std::move(swParseClientMetadata.getValue()), true); return Status::OK(); } diff --git a/src/mongo/rpc/metadata/client_metadata_ismaster.h b/src/mongo/rpc/metadata/client_metadata_ismaster.h index 305018343d0..73760abfead 100644 --- a/src/mongo/rpc/metadata/client_metadata_ismaster.h +++ b/src/mongo/rpc/metadata/client_metadata_ismaster.h @@ -59,7 +59,9 @@ public: /** * Set the optional client metadata object. */ - static void setClientMetadata(Client* client, boost::optional<ClientMetadata> clientMetadata); + static void setClientMetadata(Client* client, + boost::optional<ClientMetadata> clientMetadata, + bool setViaMetadata = false); /** * Check a flag to indicate that isMaster has been seen for this Client. @@ -102,6 +104,11 @@ private: // Thread-Safety: // None - must be only be read and written from the thread owning "Client". bool _hasSeenIsMaster{false}; + + // Indicates whether we have set isMaster based on metadata or via isMaster + // Thread-Safety: + // None - must be only be read and written from the thread owning "Client". + bool _setViaMetadata{false}; }; } // namespace mongo diff --git a/src/mongo/s/chunk_version.h b/src/mongo/s/chunk_version.h index 05517ffb609..54c1ca4f576 100644 --- a/src/mongo/s/chunk_version.h +++ b/src/mongo/s/chunk_version.h @@ -59,7 +59,7 @@ public: ChunkVersion() : _combined(0), _epoch(OID()) {} - ChunkVersion(int major, int minor, const OID& epoch) + ChunkVersion(uint32_t major, uint32_t minor, const OID& epoch) : _combined(static_cast<uint64_t>(minor) | (static_cast<uint64_t>(major) << 32)), _epoch(epoch) {} @@ -148,12 +148,12 @@ public: return _combined > 0; } - int majorVersion() const { + uint32_t majorVersion() const { return _combined >> 32; } - int minorVersion() const { - return _combined & 0xFFFF; + uint32_t minorVersion() const { + return _combined & 0xFFFFFFFF; } OID epoch() const { diff --git a/src/mongo/s/chunk_version_test.cpp b/src/mongo/s/chunk_version_test.cpp index 4bea7f466bd..51c7f9d1cf5 100644 --- a/src/mongo/s/chunk_version_test.cpp +++ b/src/mongo/s/chunk_version_test.cpp @@ -28,6 +28,8 @@ #include "mongo/platform/basic.h" +#include <limits> + #include "mongo/db/jsobj.h" #include "mongo/s/chunk_version.h" #include "mongo/unittest/unittest.h" @@ -44,16 +46,16 @@ TEST(Parsing, EpochIsOptional) { ASSERT(canParse); ASSERT(chunkVersionComplete.epoch().isSet()); ASSERT(chunkVersionComplete.epoch() == oid); - ASSERT_EQ(2, chunkVersionComplete.majorVersion()); - ASSERT_EQ(3, chunkVersionComplete.minorVersion()); + ASSERT_EQ(2u, chunkVersionComplete.majorVersion()); + ASSERT_EQ(3u, chunkVersionComplete.minorVersion()); canParse = false; ChunkVersion chunkVersionNoEpoch = ChunkVersion::fromBSON(BSON("lastmod" << Timestamp(Seconds(3), 4)), "lastmod", &canParse); ASSERT(canParse); ASSERT(!chunkVersionNoEpoch.epoch().isSet()); - ASSERT_EQ(3, chunkVersionNoEpoch.majorVersion()); - ASSERT_EQ(4, chunkVersionNoEpoch.minorVersion()); + ASSERT_EQ(3u, chunkVersionNoEpoch.majorVersion()); + ASSERT_EQ(4u, chunkVersionNoEpoch.minorVersion()); } TEST(Comparison, StrictEqual) { @@ -83,5 +85,16 @@ TEST(Comparison, OlderThan) { ASSERT(!ChunkVersion(3, 1, epoch).isOlderThan(ChunkVersion(3, 1, epoch))); } +TEST(ChunkVersionConstruction, CreateWithLargeValues) { + const auto minorVersion = std::numeric_limits<uint32_t>::max(); + const uint32_t majorVersion = 1 << 24; + const auto epoch = OID::gen(); + + ChunkVersion version(majorVersion, minorVersion, epoch); + ASSERT_EQ(majorVersion, version.majorVersion()); + ASSERT_EQ(minorVersion, version.minorVersion()); + ASSERT_EQ(epoch, version.epoch()); +} + } // unnamed namespace } // namespace mongo diff --git a/src/mongo/s/client/shard_remote.cpp b/src/mongo/s/client/shard_remote.cpp index 1e0ea51ed4b..00c9141f2d6 100644 --- a/src/mongo/s/client/shard_remote.cpp +++ b/src/mongo/s/client/shard_remote.cpp @@ -44,6 +44,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/query/query_request.h" #include "mongo/db/repl/read_concern_args.h" +#include "mongo/db/server_parameters.h" #include "mongo/executor/task_executor_pool.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/metadata/repl_set_metadata.h" @@ -72,6 +73,11 @@ const BSONObj kReplMetadata(BSON(rpc::kReplSetMetadataFieldName << 1)); // Allow the command to be executed on a secondary (see ServerSelectionMetadata). const BSONObj kSecondaryOkMetadata{rpc::ServerSelectionMetadata(true, boost::none).toBSON()}; +constexpr bool internalProhibitShardOperationRetryByDefault = false; +MONGO_EXPORT_SERVER_PARAMETER(internalProhibitShardOperationRetry, + bool, + internalProhibitShardOperationRetryByDefault); + /** * Returns a new BSONObj describing the same command and arguments as 'cmdObj', but with maxTimeMS * replaced by maxTimeMSOverride (or removed if maxTimeMSOverride is Milliseconds::max()). @@ -104,6 +110,10 @@ ShardRemote::ShardRemote(const ShardId& id, ShardRemote::~ShardRemote() = default; bool ShardRemote::isRetriableError(ErrorCodes::Error code, RetryPolicy options) { + if (internalProhibitShardOperationRetry.load()) { + return false; + } + if (options == RetryPolicy::kNoRetry) { return false; } diff --git a/src/mongo/shell/dbshell.cpp b/src/mongo/shell/dbshell.cpp index 90c59f0ee55..30b1ff718f2 100644 --- a/src/mongo/shell/dbshell.cpp +++ b/src/mongo/shell/dbshell.cpp @@ -213,6 +213,9 @@ char* shellReadline(const char* prompt, int handlesigint = 0) { } void setupSignals() { +#ifndef _WIN32 + signal(SIGHUP, quitNicely); +#endif signal(SIGINT, quitNicely); } diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index 59eb411cc3b..0e677ee79b1 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -64,9 +64,6 @@ * nodes {Array.<Mongo>} - connection to replica set members */ -/* Global default timeout variable */ -const kReplDefaultTimeoutMS = 10 * 60 * 1000; - var ReplSetTest = function(opts) { 'use strict'; @@ -91,7 +88,9 @@ var ReplSetTest = function(opts) { var _causalConsistency; - this.kDefaultTimeoutMS = kReplDefaultTimeoutMS; + // Some code still references kDefaultTimeoutMS as a (non-static) member variable, so make sure + // it's still accessible that way. + this.kDefaultTimeoutMS = ReplSetTest.kDefaultTimeoutMS; var oplogName = 'oplog.rs'; // Publicly exposed variables @@ -1290,6 +1289,9 @@ var ReplSetTest = function(opts) { // liveNodes must have been populated. var primary = rst.liveNodes.master; var combinedDBs = new Set(primary.getDBNames()); + // replSetConfig will be undefined for master/slave passthrough. + const replSetConfig = + rst.getReplSetConfigFromNode ? rst.getReplSetConfigFromNode() : undefined; rst.getSecondaries().forEach(secondary => { secondary.getDBNames().forEach(dbName => combinedDBs.add(dbName)); @@ -1378,8 +1380,10 @@ var ReplSetTest = function(opts) { // Check that the following collection stats are the same across replica set // members: // capped - // nindexes + // nindexes, except on nodes with buildIndexes: false // ns + const hasSecondaryIndexes = !replSetConfig || + replSetConfig.members[rst.getNodeId(secondary)].buildIndexes !== false; primaryCollections.forEach(collName => { var primaryCollStats = primary.getDB(dbName).runCommand({collStats: collName}); @@ -1389,7 +1393,8 @@ var ReplSetTest = function(opts) { assert.commandWorked(secondaryCollStats); if (primaryCollStats.capped !== secondaryCollStats.capped || - primaryCollStats.nindexes !== secondaryCollStats.nindexes || + (hasSecondaryIndexes && + primaryCollStats.nindexes !== secondaryCollStats.nindexes) || primaryCollStats.ns !== secondaryCollStats.ns) { print(msgPrefix + ', the primary and secondary have different stats for the ' + @@ -1887,17 +1892,21 @@ var ReplSetTest = function(opts) { } if (typeof opts === 'string' || opts instanceof String) { - _constructFromExistingSeedNode(opts); + retryOnNetworkError(function() { + // The primary may unexpectedly step down during startup if under heavy load + // and too slowly processing heartbeats. When it steps down, it closes all of + // its connections. + _constructFromExistingSeedNode(opts); + }, 10); } else { _constructStartNewInstances(opts); } }; /** - * Declare kDefaultTimeoutMS as a static property so we don't have to initialize - * a ReplSetTest object to use it. + * Global default timeout (10 minutes). */ -ReplSetTest.kDefaultTimeoutMS = kReplDefaultTimeoutMS; +ReplSetTest.kDefaultTimeoutMS = 10 * 60 * 1000; /** * Set of states that the replica set can be in. Used for the wait functions. diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js index a1bb04c6bfb..d45beb0d675 100644 --- a/src/mongo/shell/utils.js +++ b/src/mongo/shell/utils.js @@ -37,6 +37,32 @@ function _getErrorWithCode(codeOrObj, message) { return e; } +/** + * Executes the specified function and retries it if it fails due to exception related to network + * error. If it exhausts the number of allowed retries, it simply throws the last exception. + * + * Returns the return value of the input call. + */ +function retryOnNetworkError(func, numRetries, sleepMs) { + numRetries = numRetries || 1; + sleepMs = sleepMs || 1000; + + while (true) { + try { + return func(); + } catch (e) { + if (isNetworkError(e) && numRetries > 0) { + print("Network error occurred and the call will be retried: " + + tojson({error: e.toString(), stack: e.stack})); + numRetries--; + sleep(sleepMs); + } else { + throw e; + } + } + } +} + // Checks if a javascript exception is a network error. function isNetworkError(error) { return error.message.indexOf("error doing query") >= 0 || diff --git a/src/mongo/transport/service_entry_point_test_suite.cpp b/src/mongo/transport/service_entry_point_test_suite.cpp index 5d2945919e3..8cbd86bae06 100644 --- a/src/mongo/transport/service_entry_point_test_suite.cpp +++ b/src/mongo/transport/service_entry_point_test_suite.cpp @@ -133,7 +133,9 @@ void ServiceEntryPointTestSuite::MockTLHarness::asyncWait(Ticket&& ticket, SSLPeerInfo ServiceEntryPointTestSuite::MockTLHarness::getX509PeerInfo( const ConstSessionHandle& session) const { - return SSLPeerInfo("mock", stdx::unordered_set<RoleName>{}); + auto name = SSLX509Name(std::vector<std::vector<SSLX509Name::Entry>>( + {{{kOID_CommonName.toString(), 19 /* Printable String */, "mock"}}})); + return SSLPeerInfo(name, stdx::unordered_set<RoleName>{}); } TransportLayer::Stats ServiceEntryPointTestSuite::MockTLHarness::sessionStats() { diff --git a/src/mongo/transport/transport_layer_legacy.cpp b/src/mongo/transport/transport_layer_legacy.cpp index 680853ffe04..cc998286aae 100644 --- a/src/mongo/transport/transport_layer_legacy.cpp +++ b/src/mongo/transport/transport_layer_legacy.cpp @@ -324,7 +324,7 @@ Status TransportLayerLegacy::_runTicket(Ticket ticket) { // If we didn't have an X509 subject name, see if we have one now if (!conn->sslPeerInfo) { auto info = conn->amp->getX509PeerInfo(); - if (info.subjectName != "") { + if (!info.subjectName.empty()) { conn->sslPeerInfo = info; } } diff --git a/src/mongo/util/concurrency/notification.h b/src/mongo/util/concurrency/notification.h index d24fc84e5f9..25f0c65f187 100644 --- a/src/mongo/util/concurrency/notification.h +++ b/src/mongo/util/concurrency/notification.h @@ -102,12 +102,10 @@ public: * set (in which case a subsequent call to get is guaranteed to not block) or false otherwise. * If the wait is interrupted, throws an exception. */ - bool waitFor(OperationContext* txn, Microseconds waitTimeout) { - const auto waitDeadline = Date_t::now() + waitTimeout; - + bool waitFor(OperationContext* txn, Milliseconds waitTimeout) { stdx::unique_lock<stdx::mutex> lock(_mutex); - return _condVar.wait_until( - lock, waitDeadline.toSystemTimePoint(), [&]() { return !!_value; }); + return txn->waitForConditionOrInterruptFor( + _condVar, lock, waitTimeout, [&]() { return !!_value; }); } private: @@ -137,7 +135,7 @@ public: _notification.set(true); } - bool waitFor(OperationContext* txn, Microseconds waitTimeout) { + bool waitFor(OperationContext* txn, Milliseconds waitTimeout) { return _notification.waitFor(txn, waitTimeout); } diff --git a/src/mongo/util/exception_filter_win32.cpp b/src/mongo/util/exception_filter_win32.cpp index db0e3e9bb56..30d904a88cf 100644 --- a/src/mongo/util/exception_filter_win32.cpp +++ b/src/mongo/util/exception_filter_win32.cpp @@ -129,8 +129,8 @@ LONG WINAPI exceptionFilter(struct _EXCEPTION_POINTERS* excPointers) { sizeof(addressString), "0x%p", excPointers->ExceptionRecord->ExceptionAddress); - log() << "*** unhandled exception " << exceptionString << " at " << addressString - << ", terminating"; + severe() << "*** unhandled exception " << exceptionString << " at " << addressString + << ", terminating"; if (excPointers->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { ULONG acType = excPointers->ExceptionRecord->ExceptionInformation[0]; const char* acTypeString; @@ -152,10 +152,10 @@ LONG WINAPI exceptionFilter(struct _EXCEPTION_POINTERS* excPointers) { sizeof(addressString), " 0x%p", excPointers->ExceptionRecord->ExceptionInformation[1]); - log() << "*** access violation was a " << acTypeString << addressString; + severe() << "*** access violation was a " << acTypeString << addressString; } - log() << "*** stack trace for unhandled exception:"; + severe() << "*** stack trace for unhandled exception:"; // Create a copy of context record because printWindowsStackTrace will mutate it. CONTEXT contextCopy(*(excPointers->ContextRecord)); @@ -166,7 +166,7 @@ LONG WINAPI exceptionFilter(struct _EXCEPTION_POINTERS* excPointers) { // Don't go through normal shutdown procedure. It may make things worse. // Do not go through _exit or ExitProcess(), terminate immediately - log() << "*** immediate exit due to unhandled exception"; + severe() << "*** immediate exit due to unhandled exception"; TerminateProcess(GetCurrentProcess(), EXIT_ABRUPT); // We won't reach here diff --git a/src/mongo/util/hex.cpp b/src/mongo/util/hex.cpp index d589d751b45..4ee2967ddff 100644 --- a/src/mongo/util/hex.cpp +++ b/src/mongo/util/hex.cpp @@ -62,6 +62,10 @@ std::string integerToHexDef(T inInt) { } template <> +std::string integerToHex<char>(char val) { + return integerToHexDef(val); +} +template <> std::string integerToHex<int>(int val) { return integerToHexDef(val); } diff --git a/src/mongo/util/net/SConscript b/src/mongo/util/net/SConscript index 59b546fad65..d2647622471 100644 --- a/src/mongo/util/net/SConscript +++ b/src/mongo/util/net/SConscript @@ -64,6 +64,17 @@ networkEnv.Library( ) env.Library( + target='ssl_manager_status', + source=[ + "ssl_manager_status.cpp", + ], + LIBDEPS=[ + 'network', + '$BUILD_DIR/mongo/db/commands/core', + ], +) + +env.Library( target='message_port_mock', source=[ "message_port_mock.cpp", diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp index 1b120a3be83..dd6e73c3e08 100644 --- a/src/mongo/util/net/ssl_manager.cpp +++ b/src/mongo/util/net/ssl_manager.cpp @@ -51,6 +51,7 @@ #include "mongo/util/concurrency/threadlocal.h" #include "mongo/util/debug_util.h" #include "mongo/util/exit.h" +#include "mongo/util/hex.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/net/message.h" @@ -82,6 +83,8 @@ const SSLParams& getSSLGlobalParams() { return sslGlobalParams; } +namespace { + /** * Configurable via --setParameter disableNonSSLConnectionLogging=true. If false (default) * if the sslMode is set to preferSSL, we will log connections that are not using SSL. @@ -92,6 +95,18 @@ ExportedServerParameter<bool, ServerParameterType::kStartupOnly> "disableNonSSLConnectionLogging", &sslGlobalParams.disableNonSSLConnectionLogging); +ExportedServerParameter<bool, ServerParameterType::kStartupOnly> + suppressNoTLSPeerCertificateWarning(ServerParameterSet::getGlobal(), + "suppressNoTLSPeerCertificateWarning", + &sslGlobalParams.suppressNoTLSPeerCertificateWarning); + +ExportedServerParameter<bool, ServerParameterType::kStartupOnly> sslWithholdClientCertificate( + ServerParameterSet::getGlobal(), + "sslWithholdClientCertificate", + &sslGlobalParams.tlsWithholdClientCertificate); + +} // namespace + class OpenSSLCipherConfigParameter : public ExportedServerParameter<std::string, ServerParameterType::kStartupOnly> { public: @@ -159,6 +174,9 @@ IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(ASN1_SEQUENCE_ANY, ASN1_SET_ANY, ASN const STACK_OF(X509_EXTENSION) * X509_get0_extensions(const X509* peerCert) { return peerCert->cert_info->extensions; } +inline int X509_NAME_ENTRY_set(const X509_NAME_ENTRY* ne) { + return ne->set; +} #endif /** @@ -316,6 +334,7 @@ private: bool _weakValidation; bool _allowInvalidCertificates; bool _allowInvalidHostnames; + bool _suppressNoCertificateWarning; SSLConfiguration _sslConfiguration; /** @@ -356,7 +375,7 @@ private: */ bool _parseAndValidateCertificate(const std::string& keyFile, const std::string& keyPassword, - std::string* subjectName, + SSLX509Name* subjectName, Date_t* serverNotAfter); @@ -418,12 +437,18 @@ void setupFIPS() { fassertFailedNoTrace(17089); #endif } + +TLSVersionCounts tlsVersionCounts; + } // namespace +TLSVersionCounts& TLSVersionCounts::get() { + return tlsVersionCounts; +} + // Global variable indicating if this is a server or a client instance bool isSSLServer = false; - MONGO_INITIALIZER(SetupOpenSSL)(InitializerContext*) { SSL_library_init(); SSL_load_error_strings(); @@ -463,23 +488,42 @@ SSLManagerInterface* getSSLManager() { return NULL; } -std::string getCertificateSubjectName(X509* cert) { - std::string result; +SSLX509Name getCertificateSubjectX509Name(X509* cert) { + std::vector<std::vector<SSLX509Name::Entry>> entries; + + auto name = X509_get_subject_name(cert); + int count = X509_NAME_entry_count(name); + int prevSet = -1; + std::vector<SSLX509Name::Entry> rdn; + for (int i = count - 1; i >= 0; --i) { + auto* entry = X509_NAME_get_entry(name, i); + + const auto currentSet = X509_NAME_ENTRY_set(entry); + if (currentSet != prevSet) { + if (!rdn.empty()) { + entries.push_back(std::move(rdn)); + rdn = std::vector<SSLX509Name::Entry>(); + } + prevSet = currentSet; + } - BIO* out = BIO_new(BIO_s_mem()); - uassert(16884, "unable to allocate BIO memory", NULL != out); - ON_BLOCK_EXIT(BIO_free, out); + char buffer[128]; + // OBJ_obj2txt can only fail if we pass a nullptr from get_object, + // or if OpenSSL's BN library falls over. + // In either case, just panic. + uassert(ErrorCodes::InvalidSSLConfiguration, + "Unable to parse certiciate subject name", + OBJ_obj2txt(buffer, sizeof(buffer), X509_NAME_ENTRY_get_object(entry), 1) > 0); - if (X509_NAME_print_ex(out, X509_get_subject_name(cert), 0, XN_FLAG_RFC2253) >= 0) { - if (BIO_number_written(out) > 0) { - result.resize(BIO_number_written(out)); - BIO_read(out, &result[0], result.size()); - } - } else { - log() << "failed to convert subject name to RFC2253 format"; + const auto* str = X509_NAME_ENTRY_get_data(entry); + rdn.emplace_back( + buffer, str->type, std::string(reinterpret_cast<const char*>(str->data), str->length)); + } + if (!rdn.empty()) { + entries.push_back(std::move(rdn)); } - return result; + return SSLX509Name(std::move(entries)); } SSLConnection::SSLConnection(SSL_CTX* context, Socket* sock, const char* initialBytes, int len) @@ -512,6 +556,96 @@ SSLConnection::~SSLConnection() { } namespace { +std::string x509OidToShortName(const std::string& name) { + const auto nid = OBJ_txt2nid(name.c_str()); + if (nid == 0) { + return name; + } + const auto* sn = OBJ_nid2sn(nid); + if (!sn) { + return name; + } + return sn; +} + +// Characters that need to be escaped in RFC 2253 +const std::array<char, 7> rfc2253EscapeChars = {',', '+', '"', '\\', '<', '>', ';'}; + +// See section "2.4 Converting an AttributeValue from ASN.1 to a String" in RFC 2243 +std::string escapeRfc2253(StringData str) { + std::string ret; + + if (str.size() > 0) { + size_t pos = 0; + + // a space or "#" character occurring at the beginning of the string + if (str[0] == ' ') { + ret = "\\ "; + pos = 1; + } else if (str[0] == '#') { + ret = "\\#"; + pos = 1; + } + + while (pos < str.size()) { + if (static_cast<signed char>(str[pos]) < 0) { + ret += '\\'; + ret += integerToHex(str[pos]); + } else { + if (std::find(rfc2253EscapeChars.cbegin(), rfc2253EscapeChars.cend(), str[pos]) != + rfc2253EscapeChars.cend()) { + ret += '\\'; + } + + ret += str[pos]; + } + ++pos; + } + + // a space character occurring at the end of the string + if (ret.size() > 2 && ret[ret.size() - 1] == ' ') { + ret[ret.size() - 1] = '\\'; + ret += ' '; + } + } + + return ret; +} + +} // namespace + +StatusWith<std::string> SSLX509Name::getOID(StringData oid) const { + for (const auto& rdn : _entries) { + for (const auto& entry : rdn) { + if (entry.oid == oid) { + return entry.value; + } + } + } + return {ErrorCodes::KeyNotFound, "OID does not exist"}; +} + +StringBuilder& operator<<(StringBuilder& os, const SSLX509Name& name) { + std::string comma; + for (const auto& rdn : name._entries) { + std::string plus; + os << comma; + for (const auto& entry : rdn) { + os << plus << x509OidToShortName(entry.oid) << "=" << escapeRfc2253(entry.value); + plus = "+"; + } + comma = ","; + } + return os; +} + +std::string SSLX509Name::toString() const { + StringBuilder os; + os << *this; + return os.str(); +} + +namespace { void canonicalizeClusterDN(std::vector<std::string>* dn) { // remove all RDNs we don't care about for (size_t i = 0; i < dn->size(); i++) { @@ -526,30 +660,62 @@ void canonicalizeClusterDN(std::vector<std::string>* dn) { } std::stable_sort(dn->begin(), dn->end()); } + +constexpr StringData kOID_DC = "0.9.2342.19200300.100.1.25"_sd; +constexpr StringData kOID_O = "2.5.4.10"_sd; +constexpr StringData kOID_OU = "2.5.4.11"_sd; + +std::vector<SSLX509Name::Entry> canonicalizeClusterDN( + const std::vector<std::vector<SSLX509Name::Entry>>& entries) { + std::vector<SSLX509Name::Entry> ret; + + for (const auto& rdn : entries) { + for (const auto& entry : rdn) { + if ((entry.oid != kOID_DC) && (entry.oid != kOID_O) && (entry.oid != kOID_OU)) { + continue; + } + ret.push_back(entry); + } + } + std::stable_sort(ret.begin(), ret.end()); + return ret; +} +} // namespace + +/** + * The behavior of isClusterMember() is subtly different when passed + * an SSLX509Name versus a StringData. + * + * The SSLX509Name version (immediately below) compares distinguished + * names in their raw, unescaped forms and provides a more reliable match. + * + * The StringData version attempts to do a simplified string compare + * with the serialized version of the server subject name. + * + * Because escaping is not checked in the StringData version, + * some not-strictly matching RDNs will appear to share O/OU/DC with the + * server subject name. Therefore, that variant should be called with care. + */ +bool SSLConfiguration::isClusterMember(const SSLX509Name& subject) const { + auto client = canonicalizeClusterDN(subject._entries); + auto server = canonicalizeClusterDN(serverSubjectName._entries); + + return !client.empty() && (client == server); } bool SSLConfiguration::isClusterMember(StringData subjectName) const { std::vector<std::string> clientRDN = StringSplitter::split(subjectName.toString(), ","); - std::vector<std::string> serverRDN = StringSplitter::split(serverSubjectName, ","); + std::vector<std::string> serverRDN = StringSplitter::split(serverSubjectName.toString(), ","); canonicalizeClusterDN(&clientRDN); canonicalizeClusterDN(&serverRDN); - if (clientRDN.size() == 0 || clientRDN.size() != serverRDN.size()) { - return false; - } - - for (size_t i = 0; i < serverRDN.size(); i++) { - if (clientRDN[i] != serverRDN[i]) { - return false; - } - } - return true; + return !clientRDN.empty() && (clientRDN == serverRDN); } BSONObj SSLConfiguration::getServerStatusBSON() const { BSONObjBuilder security; - security.append("SSLServerSubjectName", serverSubjectName); + security.append("SSLServerSubjectName", serverSubjectName.toString()); security.appendBool("SSLServerHasCertificateAuthority", hasCA); security.appendDate("SSLServerCertificateExpirationDate", serverCertificateExpirationDate); return security.obj(); @@ -562,7 +728,8 @@ SSLManager::SSLManager(const SSLParams& params, bool isServer) _clientContext(nullptr, _free_ssl_context), _weakValidation(params.sslWeakCertificateValidation), _allowInvalidCertificates(params.sslAllowInvalidCertificates), - _allowInvalidHostnames(params.sslAllowInvalidHostnames) { + _allowInvalidHostnames(params.sslAllowInvalidHostnames), + _suppressNoCertificateWarning(params.suppressNoTLSPeerCertificateWarning) { if (!_initSynchronousSSLContext(&_clientContext, params, ConnectionDirection::kOutgoing)) { uasserted(16768, "ssl initialization problem"); } @@ -716,23 +883,33 @@ Status SSLManager::initSSLContext(SSL_CTX* context, << getSSLErrorMessage(ERR_get_error())); } - if (direction == ConnectionDirection::kOutgoing && !params.sslClusterFile.empty()) { + if (direction == ConnectionDirection::kOutgoing && params.tlsWithholdClientCertificate) { + // Do not send a client certificate if they have been suppressed. + + } else if (direction == ConnectionDirection::kOutgoing && !params.sslClusterFile.empty()) { + // Use the configured clusterFile as our client certificate. ::EVP_set_pw_prompt("Enter cluster certificate passphrase"); if (!_setupPEM(context, params.sslClusterFile, params.sslClusterPassword)) { return Status(ErrorCodes::InvalidSSLConfiguration, "Can not set up ssl clusterFile."); } + } else if (!params.sslPEMKeyFile.empty()) { - // Use the pemfile for everything else + // Use the base pemKeyFile for any other outgoing connections, + // as well as all incoming connections. ::EVP_set_pw_prompt("Enter PEM passphrase"); if (!_setupPEM(context, params.sslPEMKeyFile, params.sslPEMKeyPassword)) { return Status(ErrorCodes::InvalidSSLConfiguration, "Can not set up PEM key file."); } } - const auto status = - params.sslCAFile.empty() ? _setupSystemCA(context) : _setupCA(context, params.sslCAFile); - if (!status.isOK()) + std::string cafile = params.sslCAFile; + if (direction == ConnectionDirection::kIncoming && !params.sslClusterCAFile.empty()) { + cafile = params.sslClusterCAFile; + } + const auto status = cafile.empty() ? _setupSystemCA(context) : _setupCA(context, cafile); + if (!status.isOK()) { return status; + } if (!params.sslCRLFile.empty()) { if (!_setupCRL(context, params.sslCRLFile)) { @@ -795,7 +972,7 @@ unsigned long long SSLManager::_convertASN1ToMillis(ASN1_TIME* asn1time) { bool SSLManager::_parseAndValidateCertificate(const std::string& keyFile, const std::string& keyPassword, - std::string* subjectName, + SSLX509Name* subjectName, Date_t* serverCertificateExpirationDate) { BIO* inBIO = BIO_new(BIO_s_file()); if (inBIO == NULL) { @@ -822,7 +999,7 @@ bool SSLManager::_parseAndValidateCertificate(const std::string& keyFile, } ON_BLOCK_EXIT(X509_free, x509); - *subjectName = getCertificateSubjectName(x509); + *subjectName = getCertificateSubjectX509Name(x509); if (serverCertificateExpirationDate != NULL) { unsigned long long notBeforeMillis = _convertASN1ToMillis(X509_get_notBefore(x509)); if (notBeforeMillis == 0) { @@ -1210,8 +1387,36 @@ bool SSLManager::_hostNameMatch(const char* nameToMatch, const char* certHostNam } } +void recordTLSVersion(const SSL* conn) { + int protocol = SSL_version(conn); + + auto& counts = mongo::TLSVersionCounts::get(); + switch (protocol) { + case TLS1_VERSION: + counts.tls10.addAndFetch(1); + break; + case TLS1_1_VERSION: + counts.tls11.addAndFetch(1); + break; + case TLS1_2_VERSION: + counts.tls12.addAndFetch(1); + break; +#ifdef TLS1_3_VERSION + case TLS1_3_VERSION: + counts.tls13.addAndFetch(1); + break; +#endif + default: + // Do nothing + break; + } +} + StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertificate( SSL* conn, const std::string& remoteHost) { + + recordTLSVersion(conn); + if (!_sslConfiguration.hasCA && isSSLServer) return {boost::none}; @@ -1219,7 +1424,11 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi if (NULL == peerCert) { // no certificate presented by peer if (_weakValidation) { - warning() << "no SSL certificate provided by peer"; + // do not give warning if certificate warnings are suppressed + if (!_suppressNoCertificateWarning) { + warning() << "no SSL certificate provided by peer"; + } + return {boost::none}; } else { auto msg = "no SSL certificate provided by peer; connection rejected"; error() << msg; @@ -1246,8 +1455,8 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi } // TODO: check optional cipher restriction, using cert. - std::string peerSubjectName = getCertificateSubjectName(peerCert); - LOG(2) << "Accepted TLS connection from peer: " << peerSubjectName; + auto peerSubject = getCertificateSubjectX509Name(peerCert); + LOG(2) << "Accepted TLS connection from peer: " << peerSubject; StatusWith<stdx::unordered_set<RoleName>> swPeerCertificateRoles = _parsePeerRoles(peerCert); if (!swPeerCertificateRoles.isOK()) { @@ -1258,7 +1467,7 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi // perform hostname validation of the remote server if (remoteHost.empty()) { return boost::make_optional( - SSLPeerInfo(peerSubjectName, std::move(swPeerCertificateRoles.getValue()))); + SSLPeerInfo(peerSubject, std::move(swPeerCertificateRoles.getValue()))); } // Try to match using the Subject Alternate Name, if it exists. @@ -1288,19 +1497,19 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi } } sk_GENERAL_NAME_pop_free(sanNames, GENERAL_NAME_free); - } else if (peerSubjectName.find("CN=") != std::string::npos) { + } else { // If Subject Alternate Name (SAN) doesn't exist and Common Name (CN) does, // check Common Name. - int cnBegin = peerSubjectName.find("CN=") + 3; - int cnEnd = peerSubjectName.find(",", cnBegin); - std::string commonName = peerSubjectName.substr(cnBegin, cnEnd - cnBegin); - - if (_hostNameMatch(remoteHost.c_str(), commonName.c_str())) { - cnMatch = true; + auto swCN = peerSubject.getOID(kOID_CommonName); + if (swCN.isOK()) { + auto commonName = std::move(swCN.getValue()); + if (_hostNameMatch(remoteHost.c_str(), commonName.c_str())) { + cnMatch = true; + } + certificateNames << "CN: " << commonName; + } else { + certificateNames << "No Common Name (CN) or Subject Alternate Names (SAN) found"; } - certificateNames << "CN: " << commonName; - } else { - certificateNames << "No Common Name (CN) or Subject Alternate Names (SAN) found"; } if (!sanMatch && !cnMatch) { @@ -1316,7 +1525,7 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi } } - return boost::make_optional(SSLPeerInfo(peerSubjectName, stdx::unordered_set<RoleName>())); + return boost::make_optional(SSLPeerInfo(peerSubject, stdx::unordered_set<RoleName>())); } diff --git a/src/mongo/util/net/ssl_manager.h b/src/mongo/util/net/ssl_manager.h index bbdafdabff1..0058f7223b2 100644 --- a/src/mongo/util/net/ssl_manager.h +++ b/src/mongo/util/net/ssl_manager.h @@ -38,6 +38,8 @@ #include "mongo/base/disallow_copying.h" #include "mongo/base/string_data.h" #include "mongo/bson/bsonobj.h" +#include "mongo/db/service_context.h" +#include "mongo/platform/atomic_word.h" #include "mongo/util/decorable.h" #include "mongo/util/net/sock.h" #include "mongo/util/net/ssl_types.h" @@ -72,18 +74,11 @@ public: }; struct SSLConfiguration { - SSLConfiguration() : serverSubjectName(""), clientSubjectName("") {} - SSLConfiguration(const std::string& serverSubjectName, - const std::string& clientSubjectName, - const Date_t& serverCertificateExpirationDate) - : serverSubjectName(serverSubjectName), - clientSubjectName(clientSubjectName), - serverCertificateExpirationDate(serverCertificateExpirationDate) {} - bool isClusterMember(StringData subjectName) const; + bool isClusterMember(const SSLX509Name& subjectName) const; BSONObj getServerStatusBSON() const; - std::string serverSubjectName; - std::string clientSubjectName; + SSLX509Name serverSubjectName; + SSLX509Name clientSubjectName; Date_t serverCertificateExpirationDate; bool hasCA = false; }; @@ -106,6 +101,17 @@ const ASN1OID mongodbRolesOID("1.3.6.1.4.1.34601.2.1.1", "MongoRoles", "Sequence of MongoDB Database Roles"); +/** + * Counts of negogtiated version used by TLS connections. + */ +struct TLSVersionCounts { + AtomicInt64 tls10; + AtomicInt64 tls11; + AtomicInt64 tls12; + + static TLSVersionCounts& get(); +}; + class SSLManagerInterface : public Decorable<SSLManagerInterface> { public: static std::unique_ptr<SSLManagerInterface> create(const SSLParams& params, bool isServer); diff --git a/src/mongo/util/net/ssl_manager_status.cpp b/src/mongo/util/net/ssl_manager_status.cpp new file mode 100644 index 00000000000..559d06d4dd2 --- /dev/null +++ b/src/mongo/util/net/ssl_manager_status.cpp @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/util/net/ssl_manager.h" + +#include "mongo/config.h" +#include "mongo/db/commands/server_status.h" + + +#ifdef MONGO_CONFIG_SSL + +namespace mongo { +namespace { + +/** + * Status section of which tls versions connected to MongoDB and completed an SSL handshake. + * Note: Clients are only not counted if they try to connect to the server with a unsupported TLS + * version. They are still counted if the server rejects them for certificate issues in + * parseAndValidatePeerCertificate. + */ +class TLSVersionSatus : public ServerStatusSection { +public: + TLSVersionSatus() : ServerStatusSection("transportSecurity") {} + + bool includeByDefault() const final { + return true; + } + + BSONObj generateSection(OperationContext* txn, const BSONElement& configElement) const final { + auto& counts = TLSVersionCounts::get(); + + BSONObjBuilder builder; + builder.append("1.0", counts.tls10.load()); + builder.append("1.1", counts.tls11.load()); + builder.append("1.2", counts.tls12.load()); + return builder.obj(); + } +} tlsVersionStatus; + +} // namespace +} // namespace mongo + +#endif diff --git a/src/mongo/util/net/ssl_options.cpp b/src/mongo/util/net/ssl_options.cpp index b9785d1b83f..0d1d3504ae7 100644 --- a/src/mongo/util/net/ssl_options.cpp +++ b/src/mongo/util/net/ssl_options.cpp @@ -81,6 +81,11 @@ Status addSSLServerOptions(moe::OptionSection* options) { options->addOptionChaining( "net.ssl.CAFile", "sslCAFile", moe::String, "Certificate Authority file for SSL"); + options->addOptionChaining("net.ssl.clusterCAFile", + "sslClusterCAFile", + moe::String, + "CA used for verifying remotes during outbound connections"); + options->addOptionChaining( "net.ssl.CRLFile", "sslCRLFile", moe::String, "Certificate Revocation List file for SSL"); @@ -327,6 +332,12 @@ Status storeSSLServerOptions(const moe::Environment& params) { .generic_string(); } + if (params.count("net.ssl.clusterCAFile")) { + sslGlobalParams.sslClusterCAFile = + boost::filesystem::absolute(params["net.ssl.clusterCAFile"].as<std::string>()) + .generic_string(); + } + if (params.count("net.ssl.CRLFile")) { sslGlobalParams.sslCRLFile = boost::filesystem::absolute(params["net.ssl.CRLFile"].as<std::string>()) diff --git a/src/mongo/util/net/ssl_options.h b/src/mongo/util/net/ssl_options.h index aef2860093b..02a82108529 100644 --- a/src/mongo/util/net/ssl_options.h +++ b/src/mongo/util/net/ssl_options.h @@ -50,6 +50,7 @@ struct SSLParams { std::string sslClusterFile; // --sslInternalKeyFile std::string sslClusterPassword; // --sslInternalKeyPassword std::string sslCAFile; // --sslCAFile + std::string sslClusterCAFile; // --sslClusterCAFile std::string sslCRLFile; // --sslCRLFile std::string sslCipherConfig; // --sslCipherConfig std::vector<Protocols> sslDisabledProtocols; // --sslDisabledProtocols @@ -59,6 +60,9 @@ struct SSLParams { bool sslAllowInvalidHostnames = false; // --sslAllowInvalidHostnames bool disableNonSSLConnectionLogging = false; // --setParameter disableNonSSLConnectionLogging=true + bool suppressNoTLSPeerCertificateWarning = + false; // --setParameter suppressNoTLSPeerCertificateWarning + bool tlsWithholdClientCertificate = false; // --setParameter tlsWithholdClientCertificate SSLParams() { sslMode.store(SSLMode_disabled); diff --git a/src/mongo/util/net/ssl_types.h b/src/mongo/util/net/ssl_types.h index fc8f600625c..f7c2fa33050 100644 --- a/src/mongo/util/net/ssl_types.h +++ b/src/mongo/util/net/ssl_types.h @@ -29,21 +29,84 @@ #include <string> +#include "mongo/bson/util/builder.h" #include "mongo/db/auth/role_name.h" #include "mongo/stdx/unordered_set.h" namespace mongo { +constexpr StringData kOID_CommonName = "2.5.4.3"_sd; + +/** + * Represents a structed X509 certificate subject name. + * For example: C=US,O=MongoDB,OU=KernelTeam,CN=server + * would be held as a four element vector of Entries. + * The first entry of which yould be broken down something like: + * {{"2.5.4.6", 19, "US"}}. + * Note that _entries is a vector of vectors to accomodate + * multi-value RDNs. + */ +class SSLX509Name { +public: + struct Entry { + Entry(std::string oid, int type, std::string value) + : oid(std::move(oid)), type(type), value(std::move(value)) {} + std::string oid; // e.g. "2.5.4.8" (ST) + int type; // e.g. 19 (PRINTABLESTRING) + std::string value; + std::tuple<const std::string&, const int&, const std::string&> equalityLens() const { + return std::tie(oid, type, value); + } + }; + + SSLX509Name() = default; + explicit SSLX509Name(std::vector<std::vector<Entry>> entries) : _entries(std::move(entries)) {} + + /** + * Retreive the first instance of the value for a given OID in this name. + * Returns ErrorCodes::KeyNotFound if the OID does not exist. + */ + StatusWith<std::string> getOID(StringData oid) const; + + bool empty() const { + return std::all_of(_entries.cbegin(), _entries.cend(), [](const std::vector<Entry>& e) { + return e.empty(); + }); + } + + friend StringBuilder& operator<<(StringBuilder&, const SSLX509Name&); + std::string toString() const; + + friend bool operator==(const SSLX509Name& lhs, const SSLX509Name& rhs) { + return lhs._entries == rhs._entries; + } + friend bool operator!=(const SSLX509Name& lhs, const SSLX509Name& rhs) { + return !(lhs._entries == rhs._entries); + } + +private: + friend struct SSLConfiguration; + std::vector<std::vector<Entry>> _entries; +}; + +std::ostream& operator<<(std::ostream&, const SSLX509Name&); +inline bool operator==(const SSLX509Name::Entry& lhs, const SSLX509Name::Entry& rhs) { + return lhs.equalityLens() == rhs.equalityLens(); +} +inline bool operator<(const SSLX509Name::Entry& lhs, const SSLX509Name::Entry& rhs) { + return lhs.equalityLens() < rhs.equalityLens(); +} + /** * Contains information extracted from the peer certificate which is consumed by subsystems * outside of the networking stack. */ struct SSLPeerInfo { - SSLPeerInfo(std::string subjectName, stdx::unordered_set<RoleName> roles) + SSLPeerInfo(SSLX509Name subjectName, stdx::unordered_set<RoleName> roles) : subjectName(std::move(subjectName)), roles(std::move(roles)) {} SSLPeerInfo() = default; - std::string subjectName; + SSLX509Name subjectName; stdx::unordered_set<RoleName> roles; }; diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index d7df48cee9a..9db71335d7b 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "65d96ccb972b239c8af5aa24a03d215eb143b0e4", + "commit": "7a6598ca9b54c358803aa6290dce618f0abed63f", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.4" diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 688efa10398..b76192c0cf9 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -391,6 +391,18 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, */ WT_PAGE_LOCK(session, page); + /* + * Now that the page is locked, if attempting to evict it, check again + * whether eviction is permitted. The page's state could have changed + * while we were waiting to acquire the lock (e.g., the page could have + * split). + */ + if (LF_ISSET(WT_EVICTING) && + !__wt_page_can_evict(session, ref, NULL)) { + WT_PAGE_UNLOCK(session, page); + return (EBUSY); + } + oldest_id = __wt_txn_oldest_id(session); if (LF_ISSET(WT_EVICTING)) mod->last_eviction_id = oldest_id; diff --git a/version.json b/version.json index efa53697d06..bfd81f44317 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { - "githash": "0d6a9242c11b99ddadcfb6e86a850b6ba487530a", - "version": "3.4.16" + "githash": "4410706bef6463369ea2f42399e9843903b31923", + "version": "3.4.18" }
\ No newline at end of file |
