diff options
Diffstat (limited to 'src/mongo/db/sorter/sorter_test.cpp')
| -rw-r--r-- | src/mongo/db/sorter/sorter_test.cpp | 158 |
1 files changed, 58 insertions, 100 deletions
diff --git a/src/mongo/db/sorter/sorter_test.cpp b/src/mongo/db/sorter/sorter_test.cpp index 328dcd256c1..3388d35b8f5 100644 --- a/src/mongo/db/sorter/sorter_test.cpp +++ b/src/mongo/db/sorter/sorter_test.cpp @@ -329,55 +329,41 @@ class SortedFileWriterAndFileIteratorTests { public: void run() { unittest::TempDir tempDir("sortedFileWriterTests"); - SorterTracker sorterTracker; - SorterFileStats sorterFileStats(&sorterTracker); + SorterFileStats sorterFileStats; const SortOptions opts = SortOptions().TempDir(tempDir.path()).FileStats(&sorterFileStats); + auto makeFile = [&] { + return std::make_shared<Sorter<IntWrapper, IntWrapper>::File>( + opts.tempDir + "/" + nextFileName(), opts.sorterFileStats); + }; - int currentFileSize = 0; - - // small - currentFileSize = _appendToFile(&opts, currentFileSize, 5); + { // small + SortedFileWriter<IntWrapper, IntWrapper> sorter(opts, makeFile()); + sorter.addAlreadySorted(0, 0); + sorter.addAlreadySorted(1, -1); + sorter.addAlreadySorted(2, -2); + sorter.addAlreadySorted(3, -3); + sorter.addAlreadySorted(4, -4); + ASSERT_ITERATORS_EQUIVALENT(std::shared_ptr<IWIterator>(sorter.done()), + std::make_shared<IntIterator>(0, 5)); + } ASSERT_EQ(sorterFileStats.opened.load(), 1); ASSERT_EQ(sorterFileStats.closed.load(), 1); - ASSERT_LTE(sorterTracker.bytesSpilled.load(), currentFileSize); - // big - currentFileSize = _appendToFile(&opts, currentFileSize, 10 * 1000 * 1000); + { // big + SortedFileWriter<IntWrapper, IntWrapper> sorter(opts, makeFile()); + for (int i = 0; i < 10 * 1000 * 1000; i++) + sorter.addAlreadySorted(i, -i); + + ASSERT_ITERATORS_EQUIVALENT(std::shared_ptr<IWIterator>(sorter.done()), + std::make_shared<IntIterator>(0, 10 * 1000 * 1000)); + } ASSERT_EQ(sorterFileStats.opened.load(), 2); ASSERT_EQ(sorterFileStats.closed.load(), 2); - ASSERT_LTE(sorterTracker.bytesSpilled.load(), currentFileSize); ASSERT(boost::filesystem::is_empty(tempDir.path())); } - -private: - int _appendToFile(const SortOptions* opts, int currentFileSize, int range) { - auto makeFile = [&] { - return std::make_shared<Sorter<IntWrapper, IntWrapper>::File>( - opts->tempDir + "/" + nextFileName(), opts->sorterFileStats); - }; - - int currentBufSize = 0; - SortedFileWriter<IntWrapper, IntWrapper> sorter(*opts, makeFile()); - for (int i = 0; i < range; ++i) { - sorter.addAlreadySorted(i, -i); - currentBufSize += sizeof(i) + sizeof(-i); - - if (currentBufSize > static_cast<int>(kSortedFileBufferSize)) { - // File size only increases if buffer size exceeds limit and spills. Each spill - // includes the buffer and the size of the spill. - currentFileSize += currentBufSize + sizeof(uint32_t); - currentBufSize = 0; - } - } - ASSERT_ITERATORS_EQUIVALENT(std::shared_ptr<IWIterator>(sorter.done()), - std::make_shared<IntIterator>(0, range)); - // Anything left in-memory is spilled to disk when sorter.done(). - currentFileSize += currentBufSize + sizeof(uint32_t); - return currentFileSize; - } }; @@ -460,9 +446,7 @@ public: void run() { unittest::TempDir tempDir("sorterTests"); - SorterTracker sorterTracker; - const SortOptions opts = - SortOptions().TempDir(tempDir.path()).ExtSortAllowed().Tracker(&sorterTracker); + const SortOptions opts = SortOptions().TempDir(tempDir.path()).ExtSortAllowed(); { // test empty (no limit) ASSERT_ITERATORS_EQUIVALENT(done(makeSorter(opts).get()), @@ -596,7 +580,7 @@ public: return 0; } - virtual size_t correctSpilledRanges() const { + virtual size_t correctNumSpills() const { return 0; } @@ -620,13 +604,13 @@ private: if (numRanges == 0) return; - auto numSpilledRangesOccurred = correctSpilledRanges(); + auto numSpillsOccurred = correctNumSpills(); auto state = sorter->persistDataForShutdown(); if (opts.extSortAllowed) { ASSERT_NE(state.fileName, ""); } ASSERT_EQ(state.ranges.size(), numRanges); - ASSERT_EQ(sorter->stats().spilledRanges(), numSpilledRangesOccurred); + ASSERT_EQ(sorter->numSpills(), numSpillsOccurred); } }; @@ -726,7 +710,7 @@ public: static_cast<std::size_t>(2)); } - size_t correctSpilledRanges() const override { + size_t correctNumSpills() const override { // We add 1 to the calculation since the call to persistDataForShutdown() spills the // remaining in-memory Sorter data to disk, adding one extra range. std::size_t spillsToMerge = NUM_ITEMS * sizeof(IWPair) / MEM_LIMIT + 1; @@ -871,12 +855,11 @@ DEATH_TEST_F(SorterMakeFromExistingRangesTest, EmptyFileName, "!fileName.empty() TEST_F(SorterMakeFromExistingRangesTest, SkipFileCheckingOnEmptyRanges) { auto fileName = "unused_sorter_file"; - SorterTracker sorterTracker; - auto opts = SortOptions().ExtSortAllowed().TempDir("unused_temp_dir").Tracker(&sorterTracker); + auto opts = SortOptions().ExtSortAllowed().TempDir("unused_temp_dir"); auto sorter = std::unique_ptr<IWSorter>( IWSorter::makeFromExistingRanges(fileName, {}, opts, IWComparator(ASC))); - ASSERT_EQ(0, sorter->stats().spilledRanges()); + ASSERT_EQ(0, sorter->numSpills()); auto iter = std::unique_ptr<IWIterator>(sorter->done()); ASSERT_EQ(0, sorter->numSorted()); @@ -922,13 +905,12 @@ TEST_F(SorterMakeFromExistingRangesTest, CorruptedFile) { ofs << "invalid sorter data"; } auto fileName = tempFilePath.filename().string(); - SorterTracker sorterTracker; - auto opts = SortOptions().ExtSortAllowed().TempDir(tempDir.path()).Tracker(&sorterTracker); + auto opts = SortOptions().ExtSortAllowed().TempDir(tempDir.path()); auto sorter = std::unique_ptr<IWSorter>( IWSorter::makeFromExistingRanges(fileName, makeSampleRanges(), opts, IWComparator(ASC))); // The number of spills is set when NoLimitSorter is constructed from existing ranges. - ASSERT_EQ(makeSampleRanges().size(), sorter->stats().spilledRanges()); + ASSERT_EQ(makeSampleRanges().size(), sorter->numSpills()); ASSERT_EQ(0, sorter->numSorted()); // 16817 - error reading file. @@ -937,13 +919,11 @@ TEST_F(SorterMakeFromExistingRangesTest, CorruptedFile) { TEST_F(SorterMakeFromExistingRangesTest, RoundTrip) { unittest::TempDir tempDir(_agent.getSuiteName() + "_" + _agent.getTestName()); - SorterTracker sorterTracker; auto opts = SortOptions() .ExtSortAllowed() .TempDir(tempDir.path()) - .MaxMemoryUsageBytes(sizeof(IWSorter::Data)) - .Tracker(&sorterTracker); + .MaxMemoryUsageBytes(sizeof(IWSorter::Data)); IWPair pairInsertedBeforeShutdown(1, 100); @@ -967,7 +947,7 @@ TEST_F(SorterMakeFromExistingRangesTest, RoundTrip) { IWSorter::makeFromExistingRanges(state.fileName, state.ranges, opts, IWComparator(ASC))); // The number of spills is set when NoLimitSorter is constructed from existing ranges. - ASSERT_EQ(state.ranges.size(), sorter->stats().spilledRanges()); + ASSERT_EQ(state.ranges.size(), sorter->numSpills()); // Ensure that the restored sorter can accept additional data. IWPair pairInsertedAfterStartup(2, 200); @@ -1037,7 +1017,7 @@ public: Key operator()(Key k, const Doc&) const { return k - 10; } - Document serialize(const SerializationOptions& opts = {}) const { + Document serialize() const { MONGO_UNREACHABLE; } }; @@ -1045,7 +1025,7 @@ public: Key operator()(Key k, const Doc&) const { return k + 10; } - Document serialize(const SerializationOptions& opts = {}) const { + Document serialize() const { MONGO_UNREACHABLE; } }; @@ -1206,12 +1186,8 @@ TEST_F(BoundedSorterTest, MemoryLimitsNoExtSortAllowed) { } TEST_F(BoundedSorterTest, SpillSorted) { - SorterTracker sorterTracker; - auto options = SortOptions() - .ExtSortAllowed() - .TempDir("unused_temp_dir") - .MaxMemoryUsageBytes(16) - .Tracker(&sorterTracker); + auto options = + SortOptions().ExtSortAllowed().TempDir("unused_temp_dir").MaxMemoryUsageBytes(16); sorter = makeAsc(options); auto output = sort({ @@ -1227,7 +1203,7 @@ TEST_F(BoundedSorterTest, SpillSorted) { }); assertSorted(output); - ASSERT_EQ(sorter->stats().spilledRanges(), 3); + ASSERT_EQ(sorter->numSpills(), 3); } TEST_F(BoundedSorterTest, SpillSortedExceptOne) { @@ -1249,16 +1225,12 @@ TEST_F(BoundedSorterTest, SpillSortedExceptOne) { }); assertSorted(output); - ASSERT_EQ(sorter->stats().spilledRanges(), 3); + ASSERT_EQ(sorter->numSpills(), 3); } TEST_F(BoundedSorterTest, SpillAlmostSorted) { - SorterTracker sorterTracker; - auto options = SortOptions() - .ExtSortAllowed() - .TempDir("unused_temp_dir") - .MaxMemoryUsageBytes(16) - .Tracker(&sorterTracker); + auto options = + SortOptions().ExtSortAllowed().TempDir("unused_temp_dir").MaxMemoryUsageBytes(16); sorter = makeAsc(options); auto output = sort({ @@ -1276,7 +1248,7 @@ TEST_F(BoundedSorterTest, SpillAlmostSorted) { }); assertSorted(output); - ASSERT_EQ(sorter->stats().spilledRanges(), 2); + ASSERT_EQ(sorter->numSpills(), 2); } TEST_F(BoundedSorterTest, SpillWrongInput) { @@ -1310,7 +1282,7 @@ TEST_F(BoundedSorterTest, SpillWrongInput) { ASSERT_EQ(output[5].time, 15); ASSERT_EQ(output[6].time, 16); - ASSERT_EQ(sorter->stats().spilledRanges(), 2); + ASSERT_EQ(sorter->numSpills(), 2); // Test that by default, bad input like this would be detected. sorter = makeAsc(options); @@ -1319,13 +1291,8 @@ TEST_F(BoundedSorterTest, SpillWrongInput) { } TEST_F(BoundedSorterTest, LimitNoSpill) { - SorterTracker sorterTracker; - auto options = SortOptions() - .ExtSortAllowed() - .TempDir("unused_temp_dir") - .MaxMemoryUsageBytes(40) - .Tracker(&sorterTracker) - .Limit(2); + auto options = + SortOptions().ExtSortAllowed().TempDir("unused_temp_dir").MaxMemoryUsageBytes(40).Limit(2); sorter = makeAsc(options); auto output = sort( @@ -1348,17 +1315,12 @@ TEST_F(BoundedSorterTest, LimitNoSpill) { ASSERT_EQ(output[0].time, 0); ASSERT_EQ(output[1].time, 3); - ASSERT_EQ(sorter->stats().spilledRanges(), 0); + ASSERT_EQ(sorter->numSpills(), 0); } TEST_F(BoundedSorterTest, LimitSpill) { - SorterTracker sorterTracker; - auto options = SortOptions() - .ExtSortAllowed() - .TempDir("unused_temp_dir") - .MaxMemoryUsageBytes(40) - .Tracker(&sorterTracker) - .Limit(3); + auto options = + SortOptions().ExtSortAllowed().TempDir("unused_temp_dir").MaxMemoryUsageBytes(40).Limit(3); sorter = makeAsc(options); auto output = sort( @@ -1382,7 +1344,7 @@ TEST_F(BoundedSorterTest, LimitSpill) { ASSERT_EQ(output[1].time, 3); ASSERT_EQ(output[2].time, 10); - ASSERT_EQ(sorter->stats().spilledRanges(), 1); + ASSERT_EQ(sorter->numSpills(), 1); } TEST_F(BoundedSorterTest, DescSorted) { @@ -1627,26 +1589,22 @@ TEST_F(BoundedSorterTest, CompoundLimit) { } TEST_F(BoundedSorterTest, CompoundSpill) { - SorterTracker sorterTracker; - auto options = SortOptions() - .ExtSortAllowed() - .TempDir("unused_temp_dir") - .Tracker(&sorterTracker) - .MaxMemoryUsageBytes(40); + auto options = + SortOptions().ExtSortAllowed().TempDir("unused_temp_dir").MaxMemoryUsageBytes(40); sorter = makeAsc(options); // When each partition is small enough, we don't spill. - ASSERT_EQ(sorter->stats().spilledRanges(), 0); + ASSERT_EQ(sorter->numSpills(), 0); auto output = sort({ {1001}, {1007}, }); assertSorted(output); - ASSERT_EQ(sorter->stats().spilledRanges(), 0); + ASSERT_EQ(sorter->numSpills(), 0); // If any individual partition is large enough, we do spill. sorter->restart(); - ASSERT_EQ(sorter->stats().spilledRanges(), 0); + ASSERT_EQ(sorter->numSpills(), 0); output = sort({ {1}, {5}, @@ -1661,17 +1619,17 @@ TEST_F(BoundedSorterTest, CompoundSpill) { {7}, }); assertSorted(output); - ASSERT_EQ(sorter->stats().spilledRanges(), 1); + ASSERT_EQ(sorter->numSpills(), 1); // If later partitions are small again, they don't spill. sorter->restart(); - ASSERT_EQ(sorter->stats().spilledRanges(), 1); + ASSERT_EQ(sorter->numSpills(), 1); output = sort({ {11}, {17}, }); assertSorted(output); - ASSERT_EQ(sorter->stats().spilledRanges(), 1); + ASSERT_EQ(sorter->numSpills(), 1); } } // namespace |