summaryrefslogtreecommitdiff
path: root/src/mongo/db/sorter/sorter.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/mongo/db/sorter/sorter.cpp')
-rw-r--r--src/mongo/db/sorter/sorter.cpp107
1 files changed, 23 insertions, 84 deletions
diff --git a/src/mongo/db/sorter/sorter.cpp b/src/mongo/db/sorter/sorter.cpp
index ce460b0fc89..db53ed53a1a 100644
--- a/src/mongo/db/sorter/sorter.cpp
+++ b/src/mongo/db/sorter/sorter.cpp
@@ -58,13 +58,11 @@
#include "mongo/db/service_context.h"
#include "mongo/db/storage/encryption_hooks.h"
#include "mongo/db/storage/storage_options.h"
-#include "mongo/db/storage/storage_parameters_gen.h"
#include "mongo/platform/atomic_word.h"
#include "mongo/platform/overflow_arithmetic.h"
#include "mongo/s/is_mongos.h"
#include "mongo/util/assert_util.h"
#include "mongo/util/destructor_guard.h"
-#include "mongo/util/file.h"
#include "mongo/util/str.h"
// As this file is included in various places we need to handle the case of having the log header
@@ -606,6 +604,7 @@ protected:
* following:
*
* {1, 2, 3, 4, 5}
+ * {12, 3, 4, 5}
* {12, 34, 5}
* {1234, 5}
*/
@@ -655,7 +654,7 @@ protected:
auto iteratorPtr = std::shared_ptr<Iterator>(writer.done());
mergeIterator->closeSource();
mergedIterators.push_back(std::move(iteratorPtr));
- this->_stats.incrementSpilledRanges();
+ this->_numSpills++;
}
LOGV2_DEBUG(6033101,
@@ -716,7 +715,7 @@ public:
this->_opts.dbName,
range.getChecksum());
});
- this->_stats.setSpilledRanges(this->_iters.size());
+ this->_numSpills = this->_iters.size();
}
void add(const Key& key, const Value& val) {
@@ -724,24 +723,12 @@ public:
_data.emplace_back(key.getOwned(), val.getOwned());
- auto& memPool = this->_memPool;
- if (memPool) {
- auto memUsedInsideSorter = (sizeof(Key) + sizeof(Value)) * (_data.size() + 1);
- _memUsed = memPool->memUsage() + memUsedInsideSorter;
- this->_totalDataSizeSorted = _memUsed;
- } else {
- auto memUsage = key.memUsageForSorter() + val.memUsageForSorter();
- _memUsed += memUsage;
- this->_totalDataSizeSorted += memUsage;
- }
+ auto memUsage = key.memUsageForSorter() + val.memUsageForSorter();
+ _memUsed += memUsage;
+ this->_totalDataSizeSorted += memUsage;
- if (_memUsed > this->_opts.maxMemoryUsageBytes) {
+ if (_memUsed > this->_opts.maxMemoryUsageBytes)
spill();
- if (memPool) {
- // We expect that all buffers are unused at this point.
- memPool->freeUnused();
- }
- }
}
void emplace(Key&& key, Value&& val) override {
@@ -820,7 +807,7 @@ private:
_memUsed = 0;
- this->_stats.incrementSpilledRanges();
+ this->_numSpills++;
}
bool _done = false;
@@ -837,7 +824,7 @@ public:
typedef SortIteratorInterface<Key, Value> Iterator;
LimitOneSorter(const SortOptions& opts, const Comparator& comp)
- : Sorter<Key, Value>(opts), _comp(comp), _haveData(false) {
+ : _comp(comp), _haveData(false) {
verify(opts.limit == 1);
}
@@ -1107,7 +1094,7 @@ private:
_memUsed = 0;
- this->_stats.incrementSpilledRanges();
+ this->_numSpills++;
}
bool _done = false;
@@ -1127,39 +1114,21 @@ private:
} // namespace sorter
-namespace {
-SharedBufferFragmentBuilder makeMemPool() {
- return SharedBufferFragmentBuilder(
- gOperationMemoryPoolBlockInitialSizeKB.loadRelaxed() * static_cast<size_t>(1024),
- SharedBufferFragmentBuilder::DoubleGrowStrategy(
- gOperationMemoryPoolBlockMaxSizeKB.loadRelaxed() * static_cast<size_t>(1024)));
-}
-} // namespace
-
template <typename Key, typename Value>
Sorter<Key, Value>::Sorter(const SortOptions& opts)
- : SorterBase(opts.sorterTracker),
- _opts(opts),
+ : _opts(opts),
_file(opts.extSortAllowed ? std::make_shared<Sorter<Key, Value>::File>(
opts.tempDir + "/" + nextFileName(), opts.sorterFileStats)
- : nullptr) {
- if (opts.useMemPool) {
- _memPool.emplace(makeMemPool());
- }
-}
+ : nullptr) {}
template <typename Key, typename Value>
Sorter<Key, Value>::Sorter(const SortOptions& opts, const std::string& fileName)
- : SorterBase(opts.sorterTracker),
- _opts(opts),
+ : _opts(opts),
_file(std::make_shared<Sorter<Key, Value>::File>(opts.tempDir + "/" + fileName,
opts.sorterFileStats)) {
invariant(opts.extSortAllowed);
invariant(!opts.tempDir.empty());
invariant(!fileName.empty());
- if (opts.useMemPool) {
- _memPool.emplace(makeMemPool());
- }
}
template <typename Key, typename Value>
@@ -1177,32 +1146,12 @@ typename Sorter<Key, Value>::PersistedState Sorter<Key, Value>::persistDataForSh
}
template <typename Key, typename Value>
-Sorter<Key, Value>::File::File(std::string path, SorterFileStats* stats)
- : _path(std::move(path)), _stats(stats) {
- invariant(!_path.empty());
- if (_stats && boost::filesystem::exists(_path) && boost::filesystem::is_regular_file(_path)) {
- _stats->addSpilledDataSize(boost::filesystem::file_size(_path));
- }
-}
-
-template <typename Key, typename Value>
Sorter<Key, Value>::File::~File() {
if (_stats && _file.is_open()) {
_stats->closed.addAndFetch(1);
}
if (_keep) {
- if (!_file.is_open()) {
- return;
- }
- DESTRUCTOR_GUARD(_file.flush());
-
- mongo::File fileForFsync;
- fileForFsync.open(_path.string().c_str());
- if (fileForFsync.is_open()) {
- fileForFsync.fsync();
- }
-
return;
}
@@ -1257,9 +1206,6 @@ void Sorter<Key, Value>::File::write(const char* data, std::streamsize size) {
try {
_file.write(data, size);
_offset += size;
- if (_stats) {
- this->_stats->addSpilledDataSize(size);
- };
} catch (const std::system_error& ex) {
if (ex.code() == std::errc::no_space_on_device) {
uasserted(ErrorCodes::OutOfDiskSpace,
@@ -1329,7 +1275,7 @@ SortedFileWriter<Key, Value>::SortedFileWriter(
: _settings(settings),
_file(std::move(file)),
_fileStartOffset(_file->currentOffset()),
- _opts(opts) {
+ _dbName(opts.dbName) {
// This should be checked by consumers, but if we get here don't allow writes.
uassert(
16946, "Attempting to use external sort from mongos. This is not allowed.", !isMongos());
@@ -1366,10 +1312,6 @@ void SortedFileWriter<Key, Value>::spill() {
if (size == 0)
return;
- if (_opts.sorterFileStats) {
- _opts.sorterFileStats->addSpilledDataSizeUncompressed(size);
- }
-
std::string compressed;
snappy::Compress(outBuffer, size, &compressed);
verify(compressed.size() <= size_t(std::numeric_limits<int32_t>::max()));
@@ -1390,7 +1332,7 @@ void SortedFileWriter<Key, Value>::spill() {
reinterpret_cast<uint8_t*>(out.get()),
protectedSizeMax,
&resultLen,
- _opts.dbName);
+ _dbName);
uassert(28842,
str::stream() << "Failed to compress data: " << status.toString(),
status.isOK());
@@ -1411,7 +1353,7 @@ SortIteratorInterface<Key, Value>* SortedFileWriter<Key, Value>::done() {
spill();
return new sorter::FileIterator<Key, Value>(
- _file, _fileStartOffset, _file->currentOffset(), _settings, _opts.dbName, _checksum);
+ _file, _fileStartOffset, _file->currentOffset(), _settings, _dbName, _checksum);
}
template <typename Key, typename Value, typename Comparator, typename BoundMaker>
@@ -1419,8 +1361,7 @@ BoundedSorter<Key, Value, Comparator, BoundMaker>::BoundedSorter(const SortOptio
Comparator comp,
BoundMaker makeBound,
bool checkInput)
- : BoundedSorterInterface<Key, Value>(opts),
- compare(comp),
+ : compare(comp),
makeBound(makeBound),
_comparePairs{compare},
_checkInput(checkInput),
@@ -1435,8 +1376,7 @@ void BoundedSorter<Key, Value, Comparator, BoundMaker>::add(Key key, Value value
invariant(!_done);
// If a new value violates what we thought was our min bound, something has gone wrong.
uassert(6369910,
- str::stream() << "BoundedSorter input is too out-of-order: with bound "
- << _min->toString() << ", did not expect input " << key.toString(),
+ "BoundedSorter input is too out-of-order.",
!_checkInput || !_min || compare(*_min, key) <= 0);
// Each new item can potentially give us a tighter bound (a higher min).
@@ -1521,11 +1461,10 @@ std::pair<Key, Value> BoundedSorter<Key, Value, Comparator, BoundMaker>::next()
_heap.pop();
auto memUsage = result.first.memUsageForSorter() + result.second.memUsageForSorter();
- if (static_cast<int64_t>(memUsage) > static_cast<int64_t>(_memUsed)) {
- _memUsed = 0;
- } else {
- _memUsed -= memUsage;
- }
+ tassert(6409301,
+ "Memory usage for BoundedSorter is invalid",
+ memUsage >= 0 && static_cast<size_t>(memUsage) <= _memUsed);
+ _memUsed -= memUsage;
};
auto pullFromSpilled = [this, &result]() {
@@ -1580,7 +1519,7 @@ void BoundedSorter<Key, Value, Comparator, BoundMaker>::_spill() {
<< " bytes, but did not opt in to external sorting.",
_opts.extSortAllowed);
- this->_stats.incrementSpilledRanges();
+ ++_numSpills;
// Write out all the values from the heap in sorted order.
SortedFileWriter<Key, Value> writer(_opts, _file, {});
generated by cgit v1.2.3 (git 2.39.1) at 2026-09-11 13:24:58 +0000