New upstream version 3.4.14upstream/3.4.14

21 files changed, 428 insertions, 163 deletions

diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript
index d65046ed6fa..e887882df87 100644
--- a/src/mongo/db/storage/SConscript
+++ b/src/mongo/db/storage/SConscript
@@ -11,7 +11,6 @@ env.SConscript(
     ],
 )
 
-
 env.Library(
     target='journal_listener',
     source=[
@@ -41,7 +40,6 @@ env.Library(
         ],
     )
 
-
 env.Library(
     target='bson_collection_catalog_entry',
     source=[
@@ -75,6 +73,19 @@ env.Library(
 )
 
 env.Library(
+    target='encryption_hooks',
+    source= [
+        'encryption_hooks.cpp',
+        ],
+    LIBDEPS= ['$BUILD_DIR/mongo/base',
+              '$BUILD_DIR/mongo/db/service_context'],
+    PROGDEPS_DEPENDENTS=[
+        '$BUILD_DIR/mongo/mongod',
+        '$BUILD_DIR/mongo/mongos',
+        ],
+    )
+
+env.Library(
     target='storage_options',
     source=[
         'storage_options.cpp',
diff --git a/src/mongo/db/storage/encryption_hooks.cpp b/src/mongo/db/storage/encryption_hooks.cpp
new file mode 100644
index 00000000000..9ee9a317dbe
--- /dev/null
+++ b/src/mongo/db/storage/encryption_hooks.cpp
@@ -0,0 +1,96 @@
+/**
+ *    Copyright (C) 2017 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/storage/encryption_hooks.h"
+
+#include <boost/filesystem/path.hpp>
+
+#include "mongo/base/init.h"
+#include "mongo/db/service_context.h"
+#include "mongo/db/storage/data_protector.h"
+#include "mongo/stdx/memory.h"
+
+namespace mongo {
+
+/* Make a EncryptionHooks pointer a decoration on the global ServiceContext */
+MONGO_INITIALIZER_WITH_PREREQUISITES(SetEncryptionHooks, ("SetGlobalEnvironment"))
+(InitializerContext* context) {
+    auto encryptionHooks = stdx::make_unique<EncryptionHooks>();
+    EncryptionHooks::set(getGlobalServiceContext(), std::move(encryptionHooks));
+
+    return Status::OK();
+}
+
+namespace {
+const auto getEncryptionHooks =
+    ServiceContext::declareDecoration<std::unique_ptr<EncryptionHooks>>();
+}  // namespace
+
+void EncryptionHooks::set(ServiceContext* service, std::unique_ptr<EncryptionHooks> custHooks) {
+    auto& hooks = getEncryptionHooks(service);
+    invariant(custHooks);
+    hooks = std::move(custHooks);
+}
+
+EncryptionHooks* EncryptionHooks::get(ServiceContext* service) {
+    return getEncryptionHooks(service).get();
+}
+
+EncryptionHooks::~EncryptionHooks() {}
+
+bool EncryptionHooks::enabled() const {
+    return false;
+}
+
+bool EncryptionHooks::restartRequired() {
+    return false;
+}
+
+std::unique_ptr<DataProtector> EncryptionHooks::getDataProtector() {
+    return std::unique_ptr<DataProtector>();
+}
+
+boost::filesystem::path EncryptionHooks::getProtectedPathSuffix() {
+    return "";
+}
+
+Status EncryptionHooks::protectTmpData(
+    const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) {
+    return Status(ErrorCodes::InternalError,
+                  "Encryption hooks must be enabled to use preprocessTmpData.");
+}
+
+Status EncryptionHooks::unprotectTmpData(
+    const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) {
+    return Status(ErrorCodes::InternalError,
+                  "Encryption hooks must be enabled to use postprocessTmpData.");
+}
+}  // namespace mongo
diff --git a/src/mongo/db/storage/encryption_hooks.h b/src/mongo/db/storage/encryption_hooks.h
new file mode 100644
index 00000000000..e1c9d553a10
--- /dev/null
+++ b/src/mongo/db/storage/encryption_hooks.h
@@ -0,0 +1,99 @@
+/**
+ *    Copyright (C) 2017 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "mongo/base/disallow_copying.h"
+#include "mongo/db/jsobj.h"
+
+namespace boost {
+namespace filesystem {
+class path;
+}  // namespace filesystem
+}  // namespace boost
+
+namespace mongo {
+class DataProtector;
+class ServiceContext;
+
+class EncryptionHooks {
+public:
+    static void set(ServiceContext* service, std::unique_ptr<EncryptionHooks> custHooks);
+
+    static EncryptionHooks* get(ServiceContext* service);
+
+    virtual ~EncryptionHooks();
+
+    /**
+     * Returns true if the encryption hooks are enabled.
+     */
+    virtual bool enabled() const;
+
+    /**
+     * Perform any encryption engine initialization/sanity checking that needs to happen after
+     * storage engine initialization but before the server starts accepting incoming connections.
+     *
+     * Returns true if the server needs to be rebooted because of configuration changes.
+     */
+    virtual bool restartRequired();
+
+    /**
+     * Returns the maximum size addition when doing transforming temp data.
+     */
+    size_t additionalBytesForProtectedBuffer() {
+        return 33;
+    }
+
+    /**
+     * Get the data protector object
+     */
+    virtual std::unique_ptr<DataProtector> getDataProtector();
+
+    /**
+     * Get an implementation specific path suffix to tag files with
+     */
+    virtual boost::filesystem::path getProtectedPathSuffix();
+
+    /**
+     * Transform temp data to non-readable form before writing it to disk.
+     */
+    virtual Status protectTmpData(
+        const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen);
+
+    /**
+     * Tranforms temp data back to readable form, after reading from disk.
+     */
+    virtual Status unprotectTmpData(
+        const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen);
+};
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp
index a537a9c63c6..8c816d39f31 100644
--- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp
+++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp
@@ -556,8 +556,15 @@ void EphemeralForTestRecordStore::temp_cappedTruncateAfter(OperationContext* txn
     Records::iterator it =
         inclusive ? _data->records.lower_bound(end) : _data->records.upper_bound(end);
     while (it != _data->records.end()) {
-        txn->recoveryUnit()->registerChange(new RemoveChange(txn, _data, it->first, it->second));
-        _data->dataSize -= it->second.size;
+        RecordId id = it->first;
+        EphemeralForTestRecord record = it->second;
+
+        if (_cappedCallback) {
+            uassertStatusOK(_cappedCallback->aboutToDeleteCapped(txn, id, record.toRecordData()));
+        }
+
+        txn->recoveryUnit()->registerChange(new RemoveChange(txn, _data, id, record));
+        _data->dataSize -= record.size;
         _data->records.erase(it++);
     }
 }
diff --git a/src/mongo/db/storage/key_string.cpp b/src/mongo/db/storage/key_string.cpp
index 624c87a2404..fc580ca111d 100644
--- a/src/mongo/db/storage/key_string.cpp
+++ b/src/mongo/db/storage/key_string.cpp
@@ -1366,7 +1366,8 @@ void toBsonValue(uint8_t ctype,
                 Decimal128 dec(Decimal128::Value{lowbits, highbits});
                 if (isNegative)
                     dec = dec.negate();
-                dec = adjustDecimalExponent(typeBits, dec);
+                if (dec.isFinite())
+                    dec = adjustDecimalExponent(typeBits, dec);
                 *stream << dec;
                 break;
             }
@@ -1385,7 +1386,8 @@ void toBsonValue(uint8_t ctype,
                 Decimal128 dec(bin, Decimal128::kRoundTo34Digits, roundAwayFromZero);
                 if (hasDecimalContinuation)
                     dec = readDecimalContinuation(reader, inverted, dec);
-                dec = adjustDecimalExponent(typeBits, dec);
+                if (dec.isFinite())
+                    dec = adjustDecimalExponent(typeBits, dec);
                 *stream << dec;
             }
             break;
diff --git a/src/mongo/db/storage/key_string_test.cpp b/src/mongo/db/storage/key_string_test.cpp
index b29d1b01da8..46a11c2592a 100644
--- a/src/mongo/db/storage/key_string_test.cpp
+++ b/src/mongo/db/storage/key_string_test.cpp
@@ -596,6 +596,15 @@ const std::vector<BSONObj>& getInterestingElements(KeyString::Version version) {
 
     // Something that needs multiple bytes of typeBits
     elements.push_back(BSON("" << BSON_ARRAY("" << BSONSymbol("") << 0 << 0ll << 0.0 << -0.0)));
+    if (version != KeyString::Version::V0) {
+        // Something with exceptional typeBits for Decimal
+        elements.push_back(
+            BSON("" << BSON_ARRAY("" << BSONSymbol("") << Decimal128::kNegativeInfinity
+                                     << Decimal128::kPositiveInfinity
+                                     << Decimal128::kPositiveNaN
+                                     << Decimal128("0.0000000")
+                                     << Decimal128("-0E1000"))));
+    }
 
     //
     // Interesting numeric cases
@@ -605,6 +614,11 @@ const std::vector<BSONObj>& getInterestingElements(KeyString::Version version) {
     elements.push_back(BSON("" << 0ll));
     elements.push_back(BSON("" << 0.0));
     elements.push_back(BSON("" << -0.0));
+    if (version != KeyString::Version::V0) {
+        Decimal128("0.0.0000000");
+        Decimal128("-0E1000");
+    }
+
     elements.push_back(BSON("" << std::numeric_limits<double>::quiet_NaN()));
     elements.push_back(BSON("" << std::numeric_limits<double>::infinity()));
     elements.push_back(BSON("" << -std::numeric_limits<double>::infinity()));
@@ -740,6 +754,11 @@ const std::vector<BSONObj>& getInterestingElements(KeyString::Version version) {
         elements.push_back(BSON("" << Decimal128("4.940656458412465441765687928682214E-324")));
         elements.push_back(BSON("" << Decimal128("-4.940656458412465441765687928682214E-324")));
         elements.push_back(BSON("" << Decimal128("-4.940656458412465441765687928682213E-324")));
+
+        // Non-finite values. Note: can't roundtrip negative NaNs, so not testing here.
+        elements.push_back(BSON("" << Decimal128::kPositiveNaN));
+        elements.push_back(BSON("" << Decimal128::kNegativeInfinity));
+        elements.push_back(BSON("" << Decimal128::kPositiveInfinity));
     }
 
     // Tricky double precision number for binary/decimal conversion: very close to a decimal
@@ -823,8 +842,36 @@ void testPermutation(KeyString::Version version,
     }
 }
 
+namespace {
+std::random_device rd;
+std::mt19937_64 seedGen(rd());
+
+// To be used by perf test for seeding, so that the entire test is repeatable in case of error.
+unsigned newSeed() {
+    unsigned int seed = seedGen();  // Replace by the reported number to repeat test execution.
+    log() << "Initializing random number generator using seed " << seed;
+    return seed;
+};
+
+std::vector<BSONObj> thinElements(std::vector<BSONObj> elements,
+                                  unsigned seed,
+                                  size_t maxElements) {
+    std::mt19937_64 gen(seed);
+
+    if (elements.size() <= maxElements)
+        return elements;
+
+    log() << "only keeping " << maxElements << " of " << elements.size()
+          << " elements using random selection";
+    std::shuffle(elements.begin(), elements.end(), gen);
+    elements.resize(maxElements);
+    return elements;
+}
+}  // namespace
+
+
 TEST_F(KeyStringTest, AllPermCompare) {
-    const std::vector<BSONObj>& elements = getInterestingElements(version);
+    std::vector<BSONObj> elements = getInterestingElements(version);
 
     for (size_t i = 0; i < elements.size(); i++) {
         const BSONObj& o = elements[i];
@@ -839,19 +886,23 @@ TEST_F(KeyStringTest, AllPermCompare) {
 }
 
 TEST_F(KeyStringTest, AllPerm2Compare) {
-#if !defined(MONGO_CONFIG_OPTIMIZED_BUILD)
-    log() << "\t\t\tskipping permutation testing on non-optimized build";
-    return;
-#endif
+    std::vector<BSONObj> baseElements = getInterestingElements(version);
+    auto seed = newSeed();
 
-    const std::vector<BSONObj>& baseElements = getInterestingElements(version);
+    // Select only a small subset of elements, as the combination is quadratic.
+    // We want to select two subsets independently, so all combinations will get tested eventually.
+    // kMaxPermElements is the desired number of elements to pass to testPermutation.
+    const size_t kMaxPermElements = kDebugBuild ? 100000 : 500000;
+    size_t maxElements = sqrt(kMaxPermElements);
+    auto firstElements = thinElements(baseElements, seed, maxElements);
+    auto secondElements = thinElements(baseElements, seed + 1, maxElements);
 
     std::vector<BSONObj> elements;
-    for (size_t i = 0; i < baseElements.size(); i++) {
-        for (size_t j = 0; j < baseElements.size(); j++) {
+    for (size_t i = 0; i < firstElements.size(); i++) {
+        for (size_t j = 0; j < secondElements.size(); j++) {
             BSONObjBuilder b;
-            b.appendElements(baseElements[i]);
-            b.appendElements(baseElements[j]);
+            b.appendElements(firstElements[i]);
+            b.appendElements(secondElements[j]);
             BSONObj o = b.obj();
             elements.push_back(o);
         }
@@ -927,6 +978,27 @@ TEST_F(KeyStringTest, NaNs) {
     ASSERT(std::isnan(toBson(ks2a, ONE_ASCENDING)[""].Double()));
     ASSERT(std::isnan(toBson(ks1d, ONE_DESCENDING)[""].Double()));
     ASSERT(std::isnan(toBson(ks2d, ONE_DESCENDING)[""].Double()));
+
+    if (version == KeyString::Version::V0)
+        return;
+
+    const auto nan3 = Decimal128::kPositiveNaN;
+    const auto nan4 = Decimal128::kNegativeNaN;
+    // Since we only output a single NaN, we can only do ROUNDTRIP testing for nan1.
+    ROUNDTRIP(version, BSON("" << nan3));
+    const KeyString ks3a(version, BSON("" << nan3), ONE_ASCENDING);
+    const KeyString ks3d(version, BSON("" << nan3), ONE_DESCENDING);
+
+    const KeyString ks4a(version, BSON("" << nan4), ONE_ASCENDING);
+    const KeyString ks4d(version, BSON("" << nan4), ONE_DESCENDING);
+
+    ASSERT_EQ(ks1a, ks4a);
+    ASSERT_EQ(ks1d, ks4d);
+
+    ASSERT(toBson(ks3a, ONE_ASCENDING)[""].Decimal().isNaN());
+    ASSERT(toBson(ks4a, ONE_ASCENDING)[""].Decimal().isNaN());
+    ASSERT(toBson(ks3d, ONE_DESCENDING)[""].Decimal().isNaN());
+    ASSERT(toBson(ks4d, ONE_DESCENDING)[""].Decimal().isNaN());
 }
 TEST_F(KeyStringTest, NumberOrderLots) {
     std::vector<BSONObj> numbers;
@@ -1074,16 +1146,6 @@ const uint64_t kMinPerfMicros = 20 * 1000;
 const uint64_t kMinPerfSamples = 50 * 1000;
 typedef std::vector<BSONObj> Numbers;
 
-std::random_device rd;
-std::mt19937 seedGen(rd());
-
-// To be used by perf test for seeding, so that the entire test is repeatable in case of error.
-unsigned newSeed() {
-    unsigned int seed = seedGen();  // Replace by the reported number to repeat test execution.
-    log() << "Initializing random number generator using seed " << seed;
-    return seed;
-};
-
 /**
  * Evaluates ROUNDTRIP on all items in Numbers a sufficient number of times to take at least
  * kMinPerfMicros microseconds. Logs the elapsed time per ROUNDTRIP evaluation.
diff --git a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp
index e185afe03c7..71b3ada3ed4 100644
--- a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp
+++ b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp
@@ -36,6 +36,10 @@
 #include <boost/filesystem/path.hpp>
 #include <fstream>
 
+#ifdef __linux__
+#include <sys/sysmacros.h>
+#endif
+
 #include "mongo/db/mongod_options.h"
 #include "mongo/db/storage/mmap_v1/data_file_sync.h"
 #include "mongo/db/storage/mmap_v1/dur.h"
diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript
index 9d2a8fc8035..42eb3b4d41d 100644
--- a/src/mongo/db/storage/wiredtiger/SConscript
+++ b/src/mongo/db/storage/wiredtiger/SConscript
@@ -82,6 +82,7 @@ if wiredtiger:
             ],
         LIBDEPS=['storage_wiredtiger_core',
                  'storage_wiredtiger_customization_hooks',
+                 '$BUILD_DIR/mongo/db/concurrency/lock_manager',
                  '$BUILD_DIR/mongo/db/storage/kv/kv_engine',
                  '$BUILD_DIR/mongo/db/storage/storage_engine_lock_file',
                  '$BUILD_DIR/mongo/db/storage/storage_engine_metadata',
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp
index d14b33de5f7..e40d3a58edb 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp
@@ -31,12 +31,9 @@
 
 #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h"
 
-#include <boost/filesystem/path.hpp>
-
 #include "mongo/base/init.h"
 #include "mongo/base/string_data.h"
 #include "mongo/db/service_context.h"
-#include "mongo/db/storage/data_protector.h"
 #include "mongo/stdx/memory.h"
 
 namespace mongo {
@@ -44,7 +41,7 @@ namespace mongo {
 /* Make a WiredTigerCustomizationHooks pointer a decoration on the global ServiceContext */
 MONGO_INITIALIZER_WITH_PREREQUISITES(SetWiredTigerCustomizationHooks, ("SetGlobalEnvironment"))
 (InitializerContext* context) {
-    auto customizationHooks = stdx::make_unique<EmptyWiredTigerCustomizationHooks>();
+    auto customizationHooks = stdx::make_unique<WiredTigerCustomizationHooks>();
     WiredTigerCustomizationHooks::set(getGlobalServiceContext(), std::move(customizationHooks));
 
     return Status::OK();
@@ -66,37 +63,14 @@ WiredTigerCustomizationHooks* WiredTigerCustomizationHooks::get(ServiceContext*
     return getCustomizationHooks(service).get();
 }
 
-EmptyWiredTigerCustomizationHooks::~EmptyWiredTigerCustomizationHooks() {}
-
-bool EmptyWiredTigerCustomizationHooks::enabled() const {
-    return false;
-}
+WiredTigerCustomizationHooks::~WiredTigerCustomizationHooks() {}
 
-bool EmptyWiredTigerCustomizationHooks::restartRequired() {
+bool WiredTigerCustomizationHooks::enabled() const {
     return false;
 }
 
-std::string EmptyWiredTigerCustomizationHooks::getTableCreateConfig(StringData tableName) {
-    return "";
-}
-
-std::unique_ptr<DataProtector> EmptyWiredTigerCustomizationHooks::getDataProtector() {
-    return std::unique_ptr<DataProtector>();
-}
-
-boost::filesystem::path EmptyWiredTigerCustomizationHooks::getProtectedPathSuffix() {
+std::string WiredTigerCustomizationHooks::getTableCreateConfig(StringData tableName) {
     return "";
 }
 
-Status EmptyWiredTigerCustomizationHooks::protectTmpData(
-    const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) {
-    return Status(ErrorCodes::InternalError,
-                  "Customization hooks must be enabled to use preprocessTmpData.");
-}
-
-Status EmptyWiredTigerCustomizationHooks::unprotectTmpData(
-    const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) {
-    return Status(ErrorCodes::InternalError,
-                  "Customization hooks must be enabled to use postprocessTmpData.");
-}
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h
index e78995481bd..1ff86a8799e 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h
@@ -32,20 +32,11 @@
 #include <memory>
 #include <string>
 
-#include "mongo/base/disallow_copying.h"
-#include "mongo/db/jsobj.h"
-
-namespace boost {
-namespace filesystem {
-class path;
-}  // namespace filesystem
-}  // namespace boost
-
 namespace mongo {
-class DataProtector;
 class StringData;
 class ServiceContext;
 
+// Interface and default implementation for WiredTiger customization hooks
 class WiredTigerCustomizationHooks {
 public:
     static void set(ServiceContext* service,
@@ -53,76 +44,18 @@ public:
 
     static WiredTigerCustomizationHooks* get(ServiceContext* service);
 
-    virtual ~WiredTigerCustomizationHooks() = default;
+    virtual ~WiredTigerCustomizationHooks();
 
     /**
      * Returns true if the customization hooks are enabled.
      */
-    virtual bool enabled() const = 0;
-
-    /**
-     * Perform any encryption engine initialization/sanity checking that needs to happen after
-     * storage engine initialization but before the server starts accepting incoming connections.
-     *
-     * Returns true if the server needs to be rebooted because of configuration changes.
-     */
-    virtual bool restartRequired() = 0;
+    virtual bool enabled() const;
 
     /**
      *  Gets an additional configuration string for the provided table name on a
      *  `WT_SESSION::create` call.
      */
-    virtual std::string getTableCreateConfig(StringData tableName) = 0;
-
-    /**
-     * Returns the maximum size addition when doing transforming temp data.
-     */
-    size_t additionalBytesForProtectedBuffer() {
-        return 33;
-    }
-
-    /**
-     * Get the data protector object
-     */
-    virtual std::unique_ptr<DataProtector> getDataProtector() = 0;
-
-    /**
-     * Get an implementation specific path suffix to tag files with
-     */
-    virtual boost::filesystem::path getProtectedPathSuffix() = 0;
-
-    /**
-     * Transform temp data to non-readable form before writing it to disk.
-     */
-    virtual Status protectTmpData(
-        const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) = 0;
-
-    /**
-     * Tranforms temp data back to readable form, after reading from disk.
-     */
-    virtual Status unprotectTmpData(
-        const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) = 0;
+    virtual std::string getTableCreateConfig(StringData tableName);
 };
 
-// Empty default implementation of the abstract class WiredTigerCustomizationHooks
-class EmptyWiredTigerCustomizationHooks : public WiredTigerCustomizationHooks {
-public:
-    ~EmptyWiredTigerCustomizationHooks() override;
-
-    bool enabled() const override;
-
-    bool restartRequired() override;
-
-    std::string getTableCreateConfig(StringData tableName) override;
-
-    std::unique_ptr<DataProtector> getDataProtector() override;
-
-    boost::filesystem::path getProtectedPathSuffix() override;
-
-    Status protectTmpData(
-        const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) override;
-
-    Status unprotectTmpData(
-        const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) override;
-};
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
index 6331594a57c..ecf3c183996 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
@@ -1102,6 +1102,7 @@ void WiredTigerIndexUnique::_unindex(WT_CURSOR* c,
                 triggerWriteConflictAtPoint(c);
                 return;
             }
+            invariantWTOK(ret);
             WT_ITEM value;
             invariantWTOK(c->get_value(c, &value));
             BufReader br(value.data, value.size);
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index f48fff37b0b..aafb52b30c0 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -228,6 +228,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
         ss << "checkpoint=(wait=" << wiredTigerGlobalOptions.checkpointDelaySecs;
         ss << ",log_size=2GB),";
         ss << "statistics_log=(wait=" << wiredTigerGlobalOptions.statisticsLogDelaySecs << "),";
+        ss << "verbose=(recovery_progress),";
     }
     ss << WiredTigerCustomizationHooks::get(getGlobalServiceContext())
               ->getTableCreateConfig("system");
@@ -253,6 +254,13 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
                 msgassertedNoTrace(28718, s.reason());
             }
             invariantWTOK(_conn->close(_conn, NULL));
+            // After successful recovery, remove the journal directory.
+            try {
+                boost::filesystem::remove_all(journalPath);
+            } catch (std::exception& e) {
+                error() << "error removing journal dir " << journalPath.string() << ' ' << e.what();
+                throw;
+            }
         }
         // This setting overrides the earlier setting because it is later in the config string.
         ss << ",log=(enabled=false),";
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
index 0d3cf93a307..7761857bf90 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
@@ -909,7 +909,7 @@ int64_t WiredTigerRecordStore::storageSize(OperationContext* txn,
     if (_isEphemeral) {
         return dataSize(txn);
     }
-    WiredTigerSession* session = WiredTigerRecoveryUnit::get(txn)->getSession(txn);
+    WiredTigerSession* session = WiredTigerRecoveryUnit::get(txn)->getSessionNoTxn(txn);
     StatusWith<int64_t> result =
         WiredTigerUtil::getStatisticsValueAs<int64_t>(session->getSession(),
                                                       "statistics:" + getURI(),
@@ -1194,7 +1194,9 @@ bool WiredTigerRecordStore::yieldAndAwaitOplogDeletionRequest(OperationContext*
 
     // The top-level locks were freed, so also release any potential low-level (storage engine)
     // locks that might be held.
-    txn->recoveryUnit()->abandonSnapshot();
+    WiredTigerRecoveryUnit* recoveryUnit = (WiredTigerRecoveryUnit*)txn->recoveryUnit();
+    recoveryUnit->abandonSnapshot();
+    recoveryUnit->beginIdle();
 
     // Wait for an oplog deletion request, or for this record store to have been destroyed.
     oplogStones->awaitHasExcessStonesOrDead();
@@ -1219,15 +1221,22 @@ void WiredTigerRecordStore::reclaimOplog(OperationContext* txn) {
         try {
             WriteUnitOfWork wuow(txn);
 
-            WiredTigerCursor startwrap(_uri, _tableId, true, txn);
-            WT_CURSOR* start = startwrap.get();
-            start->set_key(start, _makeKey(_oplogStones->firstRecord));
+            WiredTigerCursor cwrap(_uri, _tableId, true, txn);
+            WT_CURSOR* cursor = cwrap.get();
 
-            WiredTigerCursor endwrap(_uri, _tableId, true, txn);
-            WT_CURSOR* end = endwrap.get();
-            end->set_key(end, _makeKey(stone->lastRecord));
+            // The first record in the oplog should be within the truncate range.
+            int ret = WT_READ_CHECK(cursor->next(cursor));
+            invariantWTOK(ret);
+            int64_t key;
+            invariantWTOK(cursor->get_key(cursor, &key));
+            RecordId firstRecord = _fromKey(key);
+            if (firstRecord < _oplogStones->firstRecord || firstRecord > stone->lastRecord) {
+                warning() << "First oplog record " << firstRecord << " is not in truncation range ("
+                          << _oplogStones->firstRecord << ", " << stone->lastRecord << ")";
+            }
 
-            invariantWTOK(session->truncate(session, nullptr, start, end, nullptr));
+            cursor->set_key(cursor, _makeKey(stone->lastRecord));
+            invariantWTOK(session->truncate(session, nullptr, nullptr, cursor, nullptr));
             _changeNumRecords(txn, -stone->records);
             _increaseDataSize(txn, -stone->bytes);
 
@@ -1619,7 +1628,7 @@ void WiredTigerRecordStore::appendCustomStats(OperationContext* txn,
         result->appendIntOrLL("sleepCount", _cappedSleep.load());
         result->appendIntOrLL("sleepMS", _cappedSleepMS.load());
     }
-    WiredTigerSession* session = WiredTigerRecoveryUnit::get(txn)->getSession(txn);
+    WiredTigerSession* session = WiredTigerRecoveryUnit::get(txn)->getSessionNoTxn(txn);
     WT_SESSION* s = session->getSession();
     BSONObjBuilder bob(result->subobjStart(_engineName));
     {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp
index 66fd5e0ddfa..fe35bd07651 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp
@@ -43,6 +43,7 @@
 #include "mongo/util/concurrency/ticketholder.h"
 #include "mongo/util/log.h"
 #include "mongo/util/mongoutils/str.h"
+#include "mongo/util/scopeguard.h"
 #include "mongo/util/stacktrace.h"
 
 namespace mongo {
@@ -172,7 +173,12 @@ WiredTigerSession* WiredTigerRecoveryUnit::getSession(OperationContext* opCtx) {
 
 WiredTigerSession* WiredTigerRecoveryUnit::getSessionNoTxn(OperationContext* opCtx) {
     _ensureSession();
-    return _session.get();
+    WiredTigerSession* session = _session.get();
+
+    // Dropping the queued idents might block session, which is not desired for fastpath workflow
+    // like FTDC thread. Disable dropping of queued idents for such sessions.
+    session->dropQueuedIdentsAtSessionEndAllowed(false);
+    return session;
 }
 
 void WiredTigerRecoveryUnit::abandonSnapshot() {
@@ -249,6 +255,13 @@ void WiredTigerRecoveryUnit::_txnOpen(OperationContext* opCtx) {
     _active = true;
 }
 
+void WiredTigerRecoveryUnit::beginIdle() {
+    // Close all cursors, we don't want to keep any old cached cursors around.
+    if (_session) {
+        _session->closeAllCursors("");
+    }
+}
+
 // ---------------------
 
 WiredTigerCursor::WiredTigerCursor(const std::string& uri,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h
index 86d9eece13d..695ad331240 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h
@@ -84,6 +84,12 @@ public:
     WiredTigerSession* getSession(OperationContext* opCtx);
 
     /**
+     * Enter a period of wait or computation during which there are no WT calls.
+     * Any non-relevant cached handles can be closed.
+     */
+    void beginIdle();
+
+    /**
      * Returns a session without starting a new WT txn on the session. Will not close any already
      * running session.
      */
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp
index 4967dfc2f86..3cb87349fdb 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp
@@ -35,6 +35,7 @@
 
 #include "mongo/base/checked_cast.h"
 #include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/concurrency/d_concurrency.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h"
@@ -56,6 +57,8 @@ bool WiredTigerServerStatusSection::includeByDefault() const {
 
 BSONObj WiredTigerServerStatusSection::generateSection(OperationContext* txn,
                                                        const BSONElement& configElement) const {
+    Lock::GlobalLock lk(txn->lockState(), LockMode::MODE_IS, UINT_MAX);
+
     // The session does not open a transaction here as one is not needed and opening one would
     // mean that execution could become blocked when a new transaction cannot be allocated
     // immediately.
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
index 4fccd06ed6d..9df6ba94651 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
@@ -36,6 +36,7 @@
 #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h"
 
 #include "mongo/base/error_codes.h"
+#include "mongo/db/server_parameters.h"
 #include "mongo/db/storage/journal_listener.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
@@ -46,13 +47,32 @@
 
 namespace mongo {
 
+std::atomic<std::int32_t> kWiredTigerCursorCacheSize(10000);  // NOLINT
+
+class WiredTigerCursorCacheSize
+    : public ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime> {
+public:
+    WiredTigerCursorCacheSize()
+        : ExportedServerParameter<std::int32_t, ServerParameterType::kStartupAndRuntime>(
+              ServerParameterSet::getGlobal(),
+              "wiredTigerCursorCacheSize",
+              &kWiredTigerCursorCacheSize) {}
+
+    virtual Status validate(const std::int32_t& potentialNewValue) {
+        if (potentialNewValue < 0) {
+            return Status(ErrorCodes::BadValue,
+                          str::stream()
+                              << "wiredTigerCursorCacheSize must be greater than or equal "
+                              << "to 0, but attempted to set to: "
+                              << potentialNewValue);
+        }
+
+        return Status::OK();
+    }
+} WiredTigerCursorCacheSizeSetting;
+
 WiredTigerSession::WiredTigerSession(WT_CONNECTION* conn, uint64_t epoch, uint64_t cursorEpoch)
-    : _epoch(epoch),
-      _cursorEpoch(cursorEpoch),
-      _session(NULL),
-      _cursorGen(0),
-      _cursorsCached(0),
-      _cursorsOut(0) {
+    : _epoch(epoch), _cursorEpoch(cursorEpoch), _session(NULL), _cursorGen(0), _cursorsOut(0) {
     invariantWTOK(conn->open_session(conn, NULL, "isolation=snapshot", &_session));
 }
 
@@ -65,7 +85,6 @@ WiredTigerSession::WiredTigerSession(WT_CONNECTION* conn,
       _cache(cache),
       _session(NULL),
       _cursorGen(0),
-      _cursorsCached(0),
       _cursorsOut(0) {
     invariantWTOK(conn->open_session(conn, NULL, "isolation=snapshot", &_session));
 }
@@ -83,7 +102,6 @@ WT_CURSOR* WiredTigerSession::getCursor(const std::string& uri, uint64_t id, boo
             WT_CURSOR* c = i->_cursor;
             _cursors.erase(i);
             _cursorsOut++;
-            _cursorsCached--;
             return c;
         }
     }
@@ -107,17 +125,11 @@ void WiredTigerSession::releaseCursor(uint64_t id, WT_CURSOR* cursor) {
 
     // Cursors are pushed to the front of the list and removed from the back
     _cursors.push_front(WiredTigerCachedCursor(id, _cursorGen++, cursor));
-    _cursorsCached++;
-
-    // "Old" is defined as not used in the last N**2 operations, if we have N cursors cached.
-    // The reasoning here is to imagine a workload with N tables performing operations randomly
-    // across all of them (i.e., each cursor has 1/N chance of used for each operation).  We
-    // would like to cache N cursors in that case, so any given cursor could go N**2 operations
-    // in between use.
-    while (_cursorGen - _cursors.back()._gen > 10000) {
+
+    std::uint64_t cursorCacheSize = static_cast<std::uint64_t>(kWiredTigerCursorCacheSize.load());
+    while (!_cursors.empty() && _cursorGen - _cursors.back()._gen > cursorCacheSize) {
         cursor = _cursors.back()._cursor;
         _cursors.pop_back();
-        _cursorsCached--;
         invariantWTOK(cursor->close(cursor));
     }
 }
@@ -125,9 +137,10 @@ void WiredTigerSession::releaseCursor(uint64_t id, WT_CURSOR* cursor) {
 void WiredTigerSession::closeAllCursors(const std::string& uri) {
     invariant(_session);
 
+    bool all = (uri == "");
     for (auto i = _cursors.begin(); i != _cursors.end();) {
         WT_CURSOR* cursor = i->_cursor;
-        if (cursor && uri == cursor->uri) {
+        if (cursor && (all || uri == cursor->uri)) {
             invariantWTOK(cursor->close(cursor));
             i = _cursors.erase(i);
         } else
@@ -344,6 +357,11 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) {
 
     bool returnedToCache = false;
     uint64_t currentEpoch = _epoch.load();
+    bool dropQueuedIdentsAtSessionEnd = session->isDropQueuedIdentsAtSessionEndAllowed();
+
+    // Reset this session's flag for dropping queued idents to default, before returning it to
+    // session cache.
+    session->dropQueuedIdentsAtSessionEndAllowed(true);
 
     if (session->_getEpoch() == currentEpoch) {  // check outside of lock to reduce contention
         stdx::lock_guard<stdx::mutex> lock(_cacheLock);
@@ -357,7 +375,7 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) {
     if (!returnedToCache)
         delete session;
 
-    if (_engine && _engine->haveDropsQueued())
+    if (dropQueuedIdentsAtSessionEnd && _engine && _engine->haveDropsQueued())
         _engine->dropSomeQueuedIdents();
 }
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
index e8fa9811796..34c05a3304b 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
@@ -99,12 +99,24 @@ public:
 
     void closeCursorsForQueuedDrops(WiredTigerKVEngine* engine);
 
+    /**
+     * Closes all cached cursors matching the uri.  If the uri is empty,
+     * all cached cursors are closed.
+     */
     void closeAllCursors(const std::string& uri);
 
     int cursorsOut() const {
         return _cursorsOut;
     }
 
+    bool isDropQueuedIdentsAtSessionEndAllowed() const {
+        return _dropQueuedIdentsAtSessionEnd;
+    }
+
+    void dropQueuedIdentsAtSessionEndAllowed(bool dropQueuedIdentsAtSessionEnd) {
+        _dropQueuedIdentsAtSessionEnd = dropQueuedIdentsAtSessionEnd;
+    }
+
     static uint64_t genTableId();
 
     /**
@@ -134,7 +146,8 @@ private:
     WT_SESSION* _session;            // owned
     CursorCache _cursors;            // owned
     uint64_t _cursorGen;
-    int _cursorsCached, _cursorsOut;
+    int _cursorsOut;
+    bool _dropQueuedIdentsAtSessionEnd = true;
 };
 
 /**
@@ -174,8 +187,8 @@ public:
     void closeCursorsForQueuedDrops();
 
     /**
-     * Closes all cached cursors and ensures that previously opened cursors will be closed on
-     * release.
+     * Closes all cached cursors matching the uri.  If the uri is empty,
+     * all cached cursors are closed.
      */
     void closeAllCursors(const std::string& uri);
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp
index 7783cea4cbf..bab41079a9b 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp
@@ -202,7 +202,7 @@ void WiredTigerSizeStorer::syncCache(bool syncToDisk) {
 
     WT_SESSION* session = _session.getSession();
     invariantWTOK(session->begin_transaction(session, syncToDisk ? "sync=true" : ""));
-    ScopeGuard rollbacker = MakeGuard(session->rollback_transaction, session, "");
+    auto rollbacker = MakeGuard(session->rollback_transaction, session, "");
 
     for (Map::iterator it = myMap.begin(); it != myMap.end(); ++it) {
         string uriKey = it->first;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp
index da0f618f1bd..e6000e62604 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp
@@ -38,6 +38,7 @@
 #include "mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.h"
 #include "mongo/util/log.h"
 #include "mongo/util/mongoutils/str.h"
+#include "mongo/util/scopeguard.h"
 
 namespace mongo {
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index f00b4a33804..c0f0907f8da 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -103,9 +103,13 @@ void WiredTigerUtil::fetchTypeAndSourceURI(OperationContext* opCtx,
 
 StatusWith<std::string> WiredTigerUtil::getMetadata(OperationContext* opCtx, StringData uri) {
     invariant(opCtx);
-    WiredTigerCursor curwrap("metadata:create", WiredTigerSession::kMetadataTableId, false, opCtx);
-    WT_CURSOR* cursor = curwrap.get();
+
+    auto session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx);
+    WT_CURSOR* cursor =
+        session->getCursor("metadata:create", WiredTigerSession::kMetadataTableId, false);
     invariant(cursor);
+    ON_BLOCK_EXIT([&] { session->releaseCursor(WiredTigerSession::kMetadataTableId, cursor); });
+
     std::string strUri = uri.toString();
     cursor->set_key(cursor, strUri.c_str());
     int ret = cursor->search(cursor);