New upstream version 6.0.20upstream

9 files changed, 139 insertions, 19 deletions

diff --git a/src/mongo/shell/SConscript b/src/mongo/shell/SConscript
index d8f54958a73..3b742237bc6 100644
--- a/src/mongo/shell/SConscript
+++ b/src/mongo/shell/SConscript
@@ -203,6 +203,7 @@ if get_option('ssl') == 'on':
                 "fle_shell_options.idl",
             ],
             LIBDEPS_PRIVATE=[
+                '$BUILD_DIR/mongo/bson/bson_validate',
                 '$BUILD_DIR/mongo/client/clientdriver_minimal',
                 '$BUILD_DIR/mongo/crypto/aead_encryption',
                 '$BUILD_DIR/mongo/crypto/encrypted_field_config',
diff --git a/src/mongo/shell/check_log.js b/src/mongo/shell/check_log.js
index 1a7c4312c0b..e31dd863519 100644
--- a/src/mongo/shell/check_log.js
+++ b/src/mongo/shell/check_log.js
@@ -30,6 +30,32 @@ checkLog = (function() {
      * is found in the logs. Note: this function does not throw an exception, so the return
      * value should not be ignored.
      */
+    const getLogMessage = function(conn, msg) {
+        const logMessages = getGlobalLog(conn);
+        if (logMessages === null) {
+            return null;
+        }
+        if (msg instanceof RegExp) {
+            for (let logMsg of logMessages) {
+                if (logMsg.search(msg) != -1) {
+                    return logMsg;
+                }
+            }
+        } else {
+            for (let logMsg of logMessages) {
+                if (logMsg.includes(msg)) {
+                    return logMsg;
+                }
+            }
+        }
+        return null;
+    };
+
+    /*
+     * Calls the 'getLog' function on the provided connection 'conn' to see if the provided msg
+     * is found in the logs. Note: this function does not throw an exception, so the return
+     * value should not be ignored.
+     */
     const checkContainsOnce = function(conn, msg) {
         const logMessages = getGlobalLog(conn);
         if (logMessages === null) {
@@ -184,6 +210,29 @@ checkLog = (function() {
             {runHangAnalyzer: false});
     };
 
+    /*
+     * Calls the 'getLog' function at regular intervals on the provided connection 'conn' until
+     * the provided 'msg' is found in the logs and returned, or it times out. Throws an exception on
+     * timeout.
+     */
+    let containsLog = function(conn, msg, timeoutMillis = 5 * 60 * 1000, retryIntervalMS = 300) {
+        // Don't run the hang analyzer because we don't expect contains() to always succeed.
+        let logMsg = null;
+        assert.soon(
+            function() {
+                logMsg = getLogMessage(conn, msg);
+                if (logMsg) {
+                    return true;
+                }
+                return false;
+            },
+            'Could not find log entries containing the following message: ' + msg,
+            timeoutMillis,
+            retryIntervalMS,
+            {runHangAnalyzer: false});
+        return logMsg;
+    };
+
     let containsJson = function(conn, id, attrsDict, timeoutMillis = 5 * 60 * 1000) {
         // Don't run the hang analyzer because we don't expect contains() to always succeed.
         assert.soon(
@@ -423,12 +472,14 @@ checkLog = (function() {
 
     return {
         getGlobalLog: getGlobalLog,
+        getLogMessage: getLogMessage,
         checkContainsOnce: checkContainsOnce,
         checkContainsOnceJson: checkContainsOnceJson,
         checkContainsWithCountJson: checkContainsWithCountJson,
         checkContainsWithAtLeastCountJson: checkContainsWithAtLeastCountJson,
         checkContainsOnceJsonStringMatch: checkContainsOnceJsonStringMatch,
         contains: contains,
+        containsLog: containsLog,
         containsJson: containsJson,
         containsRelaxedJson: containsRelaxedJson,
         containsWithCount: containsWithCount,
diff --git a/src/mongo/shell/data_consistency_checker.js b/src/mongo/shell/data_consistency_checker.js
index 3a1621b11de..b3303c76857 100644
--- a/src/mongo/shell/data_consistency_checker.js
+++ b/src/mongo/shell/data_consistency_checker.js
@@ -370,6 +370,28 @@ var {DataConsistencyChecker} = (function() {
                             delete syncingInfo.idIndex.ns;
                         }
 
+                        // If the servers are using encryption and they specify an encryption option
+                        // in versions <7.2 this is stored on the primary but not the secondary.
+                        // This is not an actual failure since the data is correct on all nodes. We
+                        // can safely ignore this element in the configString.
+                        const encryptionRegex = /encryption=\(?[^)]*\),?/;
+
+                        if (sourceInfo.options && sourceInfo.options.storageEngine &&
+                            sourceInfo.options.storageEngine.wiredTiger &&
+                            sourceInfo.options.storageEngine.wiredTiger.configString) {
+                            sourceInfo.options.storageEngine.wiredTiger.configString =
+                                sourceInfo.options.storageEngine.wiredTiger.configString.replace(
+                                    encryptionRegex, "");
+                        }
+
+                        if (syncingInfo.options && syncingInfo.options.storageEngine &&
+                            syncingInfo.options.storageEngine.wiredTiger &&
+                            syncingInfo.options.storageEngine.wiredTiger.configString) {
+                            syncingInfo.options.storageEngine.wiredTiger.configString =
+                                syncingInfo.options.storageEngine.wiredTiger.configString.replace(
+                                    encryptionRegex, "");
+                        }
+
                         if (!bsonBinaryEqual(syncingInfo, sourceInfo)) {
                             prettyPrint(
                                 `the two nodes have different attributes for the collection or view ${
diff --git a/src/mongo/shell/encrypted_dbclient_base.cpp b/src/mongo/shell/encrypted_dbclient_base.cpp
index b9151685a93..391ef009fd3 100644
--- a/src/mongo/shell/encrypted_dbclient_base.cpp
+++ b/src/mongo/shell/encrypted_dbclient_base.cpp
@@ -301,7 +301,7 @@ BSONObj EncryptedDBClientBase::validateBSONElement(ConstDataRange out, uint8_t b
 
         builder.appendNum(static_cast<uint32_t>(docLength));
         builder.appendChar(static_cast<uint8_t>(bsonType));
-        builder.appendStr(valueString, true);
+        builder.appendCStr(valueString);
         builder.appendBuf(out.data(), out.length());
         builder.appendChar('\0');
 
@@ -471,7 +471,7 @@ void EncryptedDBClientBase::encrypt(mozjs::MozJSImplScope* scope,
         }
 
         plaintextBuilder.appendNum(static_cast<uint32_t>(valueStr.size() + 1));
-        plaintextBuilder.appendStr(valueStr, true);
+        plaintextBuilder.appendStrBytesAndNul(valueStr);
         bsonType = BSONType::String;
 
     } else if (args.get(1).isNumber()) {
diff --git a/src/mongo/shell/query.js b/src/mongo/shell/query.js
index a5d920cb109..b43e13c23e8 100644
--- a/src/mongo/shell/query.js
+++ b/src/mongo/shell/query.js
@@ -98,6 +98,10 @@ DBQuery.prototype._canUseCommandCursor = function() {
         (this._options & DBQuery.Option.exhaust) === 0;
 };
 
+DBQuery.prototype._isTailableCursor = function() {
+    return (this._options & DBQuery.Option.tailable) !== 0;
+};
+
 /**
  * This method is exposed only for the purpose of testing and should not be used in most contexts.
  *
@@ -301,7 +305,9 @@ DBQuery.prototype.skip = function(skip) {
 DBQuery.prototype.hasNext = function() {
     this._exec();
 
-    if (this._limit > 0 && this._cursorSeen >= this._limit) {
+    // Return when limit is reached for tailable cursors. For other cursor options, like an exhaust
+    // cursor, the server manages closing.
+    if (this._isTailableCursor() && this._limit > 0 && this._cursorSeen >= this._limit) {
         this._cursor.close();
         return false;
     }
diff --git a/src/mongo/shell/servers.js b/src/mongo/shell/servers.js
index 1040b3c991d..ece24b66a6b 100644
--- a/src/mongo/shell/servers.js
+++ b/src/mongo/shell/servers.js
@@ -25,17 +25,6 @@ var _parsePath = function() {
     return dbpath;
 };
 
-var _parsePort = function() {
-    var port = "";
-    for (var i = 0; i < arguments.length; ++i)
-        if (arguments[i] == "--port")
-            port = arguments[i + 1];
-
-    if (port == "")
-        throw Error("No port specified");
-    return port;
-};
-
 var createMongoArgs = function(binaryName, args) {
     if (!Array.isArray(args)) {
         throw new Error("The second argument to createMongoArgs must be an array");
@@ -82,6 +71,17 @@ MongoRunner.mongosPath = "mongos";
 MongoRunner.mongoqPath = "mongoqd";
 MongoRunner.mongoShellPath = "mongo";
 
+MongoRunner.parsePort = function() {
+    var port = "";
+    for (var i = 0; i < arguments.length; ++i)
+        if (arguments[i] == "--port")
+            port = arguments[i + 1];
+
+    if (port == "")
+        throw Error("No port specified");
+    return port;
+};
+
 MongoRunner.VersionSub = function(pattern, version) {
     this.pattern = pattern;
     this.version = version;
@@ -1640,7 +1640,7 @@ MongoRunner._startWithArgs = function(argArray, env, waitForConnect) {
     // TODO: Make there only be one codepath for starting mongo processes
 
     argArray = appendSetParameterArgs(argArray);
-    var port = _parsePort.apply(null, argArray);
+    var port = MongoRunner.parsePort.apply(null, argArray);
     var pid = -1;
     if (env === undefined) {
         pid = _startMongoProgram.apply(null, argArray);
@@ -1675,7 +1675,7 @@ MongoRunner._startWithArgs = function(argArray, env, waitForConnect) {
  * command line arguments to the program.
  */
 startMongoProgram = function() {
-    var port = _parsePort.apply(null, arguments);
+    var port = MongoRunner.parsePort.apply(null, arguments);
 
     // Enable test commands.
     // TODO: Make this work better with multi-version testing so that we can support
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index 9136fa49333..1319746ea1a 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -1034,8 +1034,7 @@ var ShardingTest = function(params) {
     assert(isObject(params), 'ShardingTest configuration must be a JSON object');
 
     var testName = params.name || jsTest.name();
-    var otherParams = Object.merge(params, params.other || {});
-
+    var otherParams = Object.deepMerge(params, params.other || {});
     var numShards = otherParams.hasOwnProperty('shards') ? otherParams.shards : 2;
     var mongosVerboseLevel = otherParams.hasOwnProperty('verbose') ? otherParams.verbose : 1;
     var numMongos = otherParams.hasOwnProperty('mongos') ? otherParams.mongos : 1;
diff --git a/src/mongo/shell/types.js b/src/mongo/shell/types.js
index c06601f9161..ce8fd02b172 100644
--- a/src/mongo/shell/types.js
+++ b/src/mongo/shell/types.js
@@ -292,6 +292,36 @@ Object.merge = function(dst, src, deep) {
     return Object.extend(clone, src, deep);
 };
 
+// If there is a conflict in values of a key for two objects being merged, the second value will
+// override the first one in the merged object
+Object.deepMerge = function(...objects) {
+    const isObject = obj => obj && typeof obj === 'object';
+
+    // Create new object prev to hold combination of all object fields.
+    return objects.reduce((prev, obj) => {
+        if (obj === undefined) {
+            obj = {};
+        }
+        Object.keys(obj).forEach(key => {
+            const pVal = prev[key];  // Get the values for key from the two objects being merged.
+            const oVal = obj[key];
+
+            if (Array.isArray(pVal) &&
+                Array.isArray(oVal)) {  // If both are arrays then concatenate them into a new
+                                        // array and add it to prev.
+                prev[key] = pVal.concat(...oVal);
+            } else if (isObject(pVal) &&
+                       isObject(oVal)) {  // If both are objects then recursively merge again.
+                prev[key] = Object.deepMerge(pVal, oVal);
+            } else {  // In all other cases set prev[key] to obj[key].
+                prev[key] = oVal;
+            }
+        });
+
+        return prev;
+    }, {});
+};
+
 Object.keySet = function(o) {
     var ret = new Array();
     for (var i in o) {
diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js
index da2c29fa493..e1e09a83a7b 100644
--- a/src/mongo/shell/utils.js
+++ b/src/mongo/shell/utils.js
@@ -29,6 +29,13 @@ function _getErrorWithCode(codeOrObj, message) {
 
         if (codeOrObj.hasOwnProperty("writeErrors")) {
             e.writeErrors = codeOrObj.writeErrors;
+        } else if ((codeOrObj instanceof BulkWriteResult || codeOrObj instanceof BulkWriteError) &&
+                   codeOrObj.hasWriteErrors()) {
+            e.writeErrors = codeOrObj.getWriteErrors();
+        }
+
+        if (codeOrObj instanceof WriteResult && codeOrObj.hasWriteError()) {
+            e.writeErrors = [codeOrObj.getWriteError()];
         }
 
         if (codeOrObj.hasOwnProperty("errorLabels")) {
@@ -85,7 +92,11 @@ function isNetworkError(errorOrResponse) {
         "error doing query",
         "socket exception",
         "SocketException",
-        "HostNotFound"
+        "HostNotFound",
+        "HostUnreachable",
+        "NetworkTimeout",
+        "ConnectionPoolExpired",
+        "ConnectionError"
     ];
 
     // Then check if it's an Error, if so see if any of the known network error strings appear