Update upstream source from tag 'upstream/6.0.0'master

Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322

1 files changed, 23 insertions, 84 deletions

diff --git a/src/mongo/db/sorter/sorter.cpp b/src/mongo/db/sorter/sorter.cpp
index ce460b0fc89..db53ed53a1a 100644
--- a/src/mongo/db/sorter/sorter.cpp
+++ b/src/mongo/db/sorter/sorter.cpp
@@ -58,13 +58,11 @@
 #include "mongo/db/service_context.h"
 #include "mongo/db/storage/encryption_hooks.h"
 #include "mongo/db/storage/storage_options.h"
-#include "mongo/db/storage/storage_parameters_gen.h"
 #include "mongo/platform/atomic_word.h"
 #include "mongo/platform/overflow_arithmetic.h"
 #include "mongo/s/is_mongos.h"
 #include "mongo/util/assert_util.h"
 #include "mongo/util/destructor_guard.h"
-#include "mongo/util/file.h"
 #include "mongo/util/str.h"
 
 // As this file is included in various places we need to handle the case of having the log header
@@ -606,6 +604,7 @@ protected:
      * following:
      *
      * {1, 2, 3, 4, 5}
+     * {12, 3, 4, 5}
      * {12, 34, 5}
      * {1234, 5}
      */
@@ -655,7 +654,7 @@ protected:
                 auto iteratorPtr = std::shared_ptr<Iterator>(writer.done());
                 mergeIterator->closeSource();
                 mergedIterators.push_back(std::move(iteratorPtr));
-                this->_stats.incrementSpilledRanges();
+                this->_numSpills++;
             }
 
             LOGV2_DEBUG(6033101,
@@ -716,7 +715,7 @@ public:
                                this->_opts.dbName,
                                range.getChecksum());
                        });
-        this->_stats.setSpilledRanges(this->_iters.size());
+        this->_numSpills = this->_iters.size();
     }
 
     void add(const Key& key, const Value& val) {
@@ -724,24 +723,12 @@ public:
 
         _data.emplace_back(key.getOwned(), val.getOwned());
 
-        auto& memPool = this->_memPool;
-        if (memPool) {
-            auto memUsedInsideSorter = (sizeof(Key) + sizeof(Value)) * (_data.size() + 1);
-            _memUsed = memPool->memUsage() + memUsedInsideSorter;
-            this->_totalDataSizeSorted = _memUsed;
-        } else {
-            auto memUsage = key.memUsageForSorter() + val.memUsageForSorter();
-            _memUsed += memUsage;
-            this->_totalDataSizeSorted += memUsage;
-        }
+        auto memUsage = key.memUsageForSorter() + val.memUsageForSorter();
+        _memUsed += memUsage;
+        this->_totalDataSizeSorted += memUsage;
 
-        if (_memUsed > this->_opts.maxMemoryUsageBytes) {
+        if (_memUsed > this->_opts.maxMemoryUsageBytes)
             spill();
-            if (memPool) {
-                // We expect that all buffers are unused at this point.
-                memPool->freeUnused();
-            }
-        }
     }
 
     void emplace(Key&& key, Value&& val) override {
@@ -820,7 +807,7 @@ private:
 
         _memUsed = 0;
 
-        this->_stats.incrementSpilledRanges();
+        this->_numSpills++;
     }
 
     bool _done = false;
@@ -837,7 +824,7 @@ public:
     typedef SortIteratorInterface<Key, Value> Iterator;
 
     LimitOneSorter(const SortOptions& opts, const Comparator& comp)
-        : Sorter<Key, Value>(opts), _comp(comp), _haveData(false) {
+        : _comp(comp), _haveData(false) {
         verify(opts.limit == 1);
     }
 
@@ -1107,7 +1094,7 @@ private:
 
         _memUsed = 0;
 
-        this->_stats.incrementSpilledRanges();
+        this->_numSpills++;
     }
 
     bool _done = false;
@@ -1127,39 +1114,21 @@ private:
 
 }  // namespace sorter
 
-namespace {
-SharedBufferFragmentBuilder makeMemPool() {
-    return SharedBufferFragmentBuilder(
-        gOperationMemoryPoolBlockInitialSizeKB.loadRelaxed() * static_cast<size_t>(1024),
-        SharedBufferFragmentBuilder::DoubleGrowStrategy(
-            gOperationMemoryPoolBlockMaxSizeKB.loadRelaxed() * static_cast<size_t>(1024)));
-}
-}  // namespace
-
 template <typename Key, typename Value>
 Sorter<Key, Value>::Sorter(const SortOptions& opts)
-    : SorterBase(opts.sorterTracker),
-      _opts(opts),
+    : _opts(opts),
       _file(opts.extSortAllowed ? std::make_shared<Sorter<Key, Value>::File>(
                                       opts.tempDir + "/" + nextFileName(), opts.sorterFileStats)
-                                : nullptr) {
-    if (opts.useMemPool) {
-        _memPool.emplace(makeMemPool());
-    }
-}
+                                : nullptr) {}
 
 template <typename Key, typename Value>
 Sorter<Key, Value>::Sorter(const SortOptions& opts, const std::string& fileName)
-    : SorterBase(opts.sorterTracker),
-      _opts(opts),
+    : _opts(opts),
       _file(std::make_shared<Sorter<Key, Value>::File>(opts.tempDir + "/" + fileName,
                                                        opts.sorterFileStats)) {
     invariant(opts.extSortAllowed);
     invariant(!opts.tempDir.empty());
     invariant(!fileName.empty());
-    if (opts.useMemPool) {
-        _memPool.emplace(makeMemPool());
-    }
 }
 
 template <typename Key, typename Value>
@@ -1177,32 +1146,12 @@ typename Sorter<Key, Value>::PersistedState Sorter<Key, Value>::persistDataForSh
 }
 
 template <typename Key, typename Value>
-Sorter<Key, Value>::File::File(std::string path, SorterFileStats* stats)
-    : _path(std::move(path)), _stats(stats) {
-    invariant(!_path.empty());
-    if (_stats && boost::filesystem::exists(_path) && boost::filesystem::is_regular_file(_path)) {
-        _stats->addSpilledDataSize(boost::filesystem::file_size(_path));
-    }
-}
-
-template <typename Key, typename Value>
 Sorter<Key, Value>::File::~File() {
     if (_stats && _file.is_open()) {
         _stats->closed.addAndFetch(1);
     }
 
     if (_keep) {
-        if (!_file.is_open()) {
-            return;
-        }
-        DESTRUCTOR_GUARD(_file.flush());
-
-        mongo::File fileForFsync;
-        fileForFsync.open(_path.string().c_str());
-        if (fileForFsync.is_open()) {
-            fileForFsync.fsync();
-        }
-
         return;
     }
 
@@ -1257,9 +1206,6 @@ void Sorter<Key, Value>::File::write(const char* data, std::streamsize size) {
     try {
         _file.write(data, size);
         _offset += size;
-        if (_stats) {
-            this->_stats->addSpilledDataSize(size);
-        };
     } catch (const std::system_error& ex) {
         if (ex.code() == std::errc::no_space_on_device) {
             uasserted(ErrorCodes::OutOfDiskSpace,
@@ -1329,7 +1275,7 @@ SortedFileWriter<Key, Value>::SortedFileWriter(
     : _settings(settings),
       _file(std::move(file)),
       _fileStartOffset(_file->currentOffset()),
-      _opts(opts) {
+      _dbName(opts.dbName) {
     // This should be checked by consumers, but if we get here don't allow writes.
     uassert(
         16946, "Attempting to use external sort from mongos. This is not allowed.", !isMongos());
@@ -1366,10 +1312,6 @@ void SortedFileWriter<Key, Value>::spill() {
     if (size == 0)
         return;
 
-    if (_opts.sorterFileStats) {
-        _opts.sorterFileStats->addSpilledDataSizeUncompressed(size);
-    }
-
     std::string compressed;
     snappy::Compress(outBuffer, size, &compressed);
     verify(compressed.size() <= size_t(std::numeric_limits<int32_t>::max()));
@@ -1390,7 +1332,7 @@ void SortedFileWriter<Key, Value>::spill() {
                                                         reinterpret_cast<uint8_t*>(out.get()),
                                                         protectedSizeMax,
                                                         &resultLen,
-                                                        _opts.dbName);
+                                                        _dbName);
         uassert(28842,
                 str::stream() << "Failed to compress data: " << status.toString(),
                 status.isOK());
@@ -1411,7 +1353,7 @@ SortIteratorInterface<Key, Value>* SortedFileWriter<Key, Value>::done() {
     spill();
 
     return new sorter::FileIterator<Key, Value>(
-        _file, _fileStartOffset, _file->currentOffset(), _settings, _opts.dbName, _checksum);
+        _file, _fileStartOffset, _file->currentOffset(), _settings, _dbName, _checksum);
 }
 
 template <typename Key, typename Value, typename Comparator, typename BoundMaker>
@@ -1419,8 +1361,7 @@ BoundedSorter<Key, Value, Comparator, BoundMaker>::BoundedSorter(const SortOptio
                                                                  Comparator comp,
                                                                  BoundMaker makeBound,
                                                                  bool checkInput)
-    : BoundedSorterInterface<Key, Value>(opts),
-      compare(comp),
+    : compare(comp),
       makeBound(makeBound),
       _comparePairs{compare},
       _checkInput(checkInput),
@@ -1435,8 +1376,7 @@ void BoundedSorter<Key, Value, Comparator, BoundMaker>::add(Key key, Value value
     invariant(!_done);
     // If a new value violates what we thought was our min bound, something has gone wrong.
     uassert(6369910,
-            str::stream() << "BoundedSorter input is too out-of-order: with bound "
-                          << _min->toString() << ", did not expect input " << key.toString(),
+            "BoundedSorter input is too out-of-order.",
             !_checkInput || !_min || compare(*_min, key) <= 0);
 
     // Each new item can potentially give us a tighter bound (a higher min).
@@ -1521,11 +1461,10 @@ std::pair<Key, Value> BoundedSorter<Key, Value, Comparator, BoundMaker>::next()
         _heap.pop();
 
         auto memUsage = result.first.memUsageForSorter() + result.second.memUsageForSorter();
-        if (static_cast<int64_t>(memUsage) > static_cast<int64_t>(_memUsed)) {
-            _memUsed = 0;
-        } else {
-            _memUsed -= memUsage;
-        }
+        tassert(6409301,
+                "Memory usage for BoundedSorter is invalid",
+                memUsage >= 0 && static_cast<size_t>(memUsage) <= _memUsed);
+        _memUsed -= memUsage;
     };
 
     auto pullFromSpilled = [this, &result]() {
@@ -1580,7 +1519,7 @@ void BoundedSorter<Key, Value, Comparator, BoundMaker>::_spill() {
                           << " bytes, but did not opt in to external sorting.",
             _opts.extSortAllowed);
 
-    this->_stats.incrementSpilledRanges();
+    ++_numSpills;
 
     // Write out all the values from the heap in sorted order.
     SortedFileWriter<Key, Value> writer(_opts, _file, {});