diff options
Diffstat (limited to 'src/mongo')
898 files changed, 32648 insertions, 6703 deletions
diff --git a/src/mongo/SConscript b/src/mongo/SConscript index fa319073e68..48bee2cfd56 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -148,7 +148,6 @@ baseEnv.Library( 'base/validate_locale.cpp', 'bson/bson_comparator_interface_base.cpp', 'bson/bson_depth.cpp', - 'bson/bson_validate.cpp', 'bson/bsonelement.cpp', 'bson/bsonmisc.cpp', 'bson/bsonobj.cpp', diff --git a/src/mongo/base/dependency_graph.cpp b/src/mongo/base/dependency_graph.cpp index 8d9107b5a10..faa8b8cdae3 100644 --- a/src/mongo/base/dependency_graph.cpp +++ b/src/mongo/base/dependency_graph.cpp @@ -29,6 +29,7 @@ #include "mongo/base/dependency_graph.h" +#include "mongo/util/stacktrace.h" #include <algorithm> #include <fmt/format.h> #include <fmt/ranges.h> diff --git a/src/mongo/base/error_codes.yml b/src/mongo/base/error_codes.yml index 199ecbefc35..6b43f74536f 100644 --- a/src/mongo/base/error_codes.yml +++ b/src/mongo/base/error_codes.yml @@ -492,6 +492,8 @@ error_codes: - {code: 379, name: DatabaseMetadataRefreshCanceled, categories: [InternalOnly]} + - {code: 387, name: EncounteredFLEPayloadWhileApplyingHmac} + - {code: 388, name: TransactionTooLargeForCache} # This error code is not generated internally in mongod/s servers, but must be parsed and @@ -502,6 +504,8 @@ error_codes: - {code: 384, name: ConnectionError, categories: [NetworkError,RetriableError,InternalOnly]} + - {code: 408, name: CannotInsertTimeseriesBucketsWithMixedSchema} + # Error codes 4000-8999 are reserved. # Non-sequential error codes for compatibility only) diff --git a/src/mongo/base/string_data.h b/src/mongo/base/string_data.h index ad26243afc0..be2c6592ea9 100644 --- a/src/mongo/base/string_data.h +++ b/src/mongo/base/string_data.h @@ -128,7 +128,7 @@ public: */ bool equalCaseInsensitive(StringData other) const; - void copyTo(char* dest, bool includeEndingNull) const; + size_t copy(char* dest, size_t count, size_t pos = 0) const; constexpr StringData substr(size_t pos, size_t n = std::numeric_limits<size_t>::max()) const; @@ -237,11 +237,13 @@ inline bool StringData::equalCaseInsensitive(StringData other) const { }); } -inline void StringData::copyTo(char* dest, bool includeEndingNull) const { - if (_data) - memcpy(dest, _data, size()); - if (includeEndingNull) - dest[size()] = 0; +inline size_t StringData::copy(char* dest, size_t count, size_t pos) const { + if (MONGO_unlikely(pos > size())) + throw std::out_of_range("pos > size()"); + + const auto rcount = std::min(count, size() - pos); + std::char_traits<char>::copy(dest, rawData() + pos, rcount); + return rcount; } inline size_t StringData::find(char c, size_t fromPos) const { diff --git a/src/mongo/bson/SConscript b/src/mongo/bson/SConscript index 31eb607823c..472aed01ad3 100644 --- a/src/mongo/bson/SConscript +++ b/src/mongo/bson/SConscript @@ -28,6 +28,7 @@ env.CppUnitTest( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/bson/util/bson_column', ], ) @@ -39,6 +40,7 @@ env.Benchmark( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', ], ) @@ -50,6 +52,21 @@ env.CppLibfuzzerTest( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', + ], +) + +env.Library( + target='bson_validate', + source=[ + 'bson_validate.cpp', + 'bson_validate.idl', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/util/bson_column', + '$BUILD_DIR/mongo/idl/idl_parser', + '$BUILD_DIR/mongo/idl/server_parameter', ], ) diff --git a/src/mongo/bson/bson_validate.cpp b/src/mongo/bson/bson_validate.cpp index 726a12b818c..da4b54ef194 100644 --- a/src/mongo/bson/bson_validate.cpp +++ b/src/mongo/bson/bson_validate.cpp @@ -83,9 +83,15 @@ constexpr ErrorCodes::Error InvalidBSON = ErrorCodes::InvalidBSON; constexpr ErrorCodes::Error NonConformantBSON = ErrorCodes::NonConformantBSON; template <bool precise> +Status _doValidateColumn(const char* originalBuffer, + uint64_t maxLength, + ValidationVersion validationVersion); + +template <bool precise> class ValidateBuffer { public: - ValidateBuffer(const char* data, uint64_t maxLength) : _data(data), _maxLength(maxLength) { + ValidateBuffer(const char* data, uint64_t maxLength, ValidationVersion validationVersion) + : _data(data), _maxLength(maxLength), _validationVersion(validationVersion) { if constexpr (precise) _frames.resize(BSONDepth::getMaxAllowableDepth() + 1); } @@ -127,30 +133,21 @@ public: // Handle one element without using iterative loop, and without expecting // multiple instances or an EOO. Only resume with the iterative loop if - // the frame stack has been incremented, meaning we have nested objects - - // Save pointer to currFrame->end so we can fill it in once we know the size - const char** preEnd = &(_currFrame->end); - const char* ptr = _validateElem(Cursor{_data + 2, _data + _maxLength}, *_data); + // we have nested objects + _currElem = _data; + const char* ptr = _validateElem<false>(Cursor{_data + 2, _data + _maxLength}, *_data); - if (_currFrame != _frames.begin()) { - // We know that type was kObject or kArray, so size is fieldname, type, - // and a stored int + if (_firstFrameUpdated) { + // We know that type was kObject/kArray/kCodeWScope + // Size is fieldname, type, and a stored int int64_t size = static_cast<int64_t>(ConstDataView(_data + 2).read<LittleEndian<int32_t>>()) + 2; uassert(InvalidBSON, "BSON literal content exceeds buffer size", (size_t)size <= _maxLength); - *preEnd = _data + size; - const char* internalEnd = _currFrame->end; - _popFrame(); - uassert(InvalidBSON, - "BSON literal nested content does not end at external end", - _currFrame->end == internalEnd); _validateIterative(Cursor{ptr, _data + size}); return size; } else { - *preEnd = ptr; return ptr - _data; } } @@ -216,7 +213,10 @@ private: uassert(ErrorCodes::Overflow, "BSONObj exceeds maximum nested object depth", ++_currFrame != _frames.end()); + return _updateFrame(cursor); + } + const char* _updateFrame(Cursor cursor) { auto obj = cursor.ptr; auto len = cursor.template read<int32_t>(); uassert(ErrorCodes::InvalidBSON, "Nested BSON object has to be at least 5 bytes", len >= 5); @@ -236,18 +236,19 @@ private: return true; } - static const char* _validateSpecial(Cursor cursor, uint8_t type) { + const char* _validateSpecial(Cursor cursor, uint8_t type) { switch (type) { case BSONType::BinData: { auto count = cursor.template read<uint32_t>(); auto subtype = cursor.template read<uint8_t>(); const char* columnStart = cursor.ptr; cursor.skip(count); - if (subtype == BinDataType::Column) { + if (subtype == BinDataType::Column && _validationVersion >= V2_Column) { /* do not pass down cursor; we want to reset the nesting depth */ - uassert(NonConformantBSON, - "Invalid BSON column", - validateBSONColumn(columnStart, count).isOK()); + uassert( + NonConformantBSON, + "Invalid BSON column", + _doValidateColumn<precise>(columnStart, count, _validationVersion).isOK()); } break; } @@ -274,10 +275,15 @@ private: return cursor.ptr; } + template <bool nestedFrame> const char* _pushCodeWithScope(Cursor cursor) { - cursor.ptr = _pushFrame(cursor); // Push a dummy frame to check the CodeWScope size. - cursor.skipString(); // Now skip the BSON UTF8 string containing the code. - _currElem = cursor.ptr - 1; // Use the terminating NUL as adummy scope element. + // Push a dummy frame to check the CodeWScope size. + if constexpr (nestedFrame) + cursor.ptr = _pushFrame(cursor); + else + cursor.ptr = _updateFrame(cursor); + cursor.skipString(); // Now skip the BSON UTF8 string containing the code. + _currElem = cursor.ptr - 1; // Use the terminating NUL as a dummy scope element. return _pushFrame(cursor); } @@ -291,21 +297,30 @@ private: } } + template <bool nestedFrame> const char* _validateElem(Cursor cursor, uint8_t type) { if (MONGO_unlikely(type > JSTypeMax)) return _validateSpecial(cursor, type); auto style = kTypeInfoTable[type]; - if (MONGO_likely(style <= kSkip16)) + if (MONGO_likely(style <= kSkip16)) { cursor.skip(style * 4); - else if (MONGO_likely(style == kString)) + } else if (MONGO_likely(style == kString)) { cursor.skipString(); - else if (MONGO_likely(style == kObjectOrArray)) - cursor.ptr = _pushFrame(cursor); - else if (MONGO_unlikely(precise && type == CodeWScope)) - cursor.ptr = _pushCodeWithScope(cursor); - else + } else if (MONGO_likely(style == kObjectOrArray)) { + if constexpr (nestedFrame) { + cursor.ptr = _pushFrame(cursor); + } else { + cursor.ptr = _updateFrame(cursor); + _firstFrameUpdated = true; + } + } else if (MONGO_unlikely(precise && type == CodeWScope)) { + cursor.ptr = _pushCodeWithScope<nestedFrame>(cursor); + if constexpr (!nestedFrame) + _firstFrameUpdated = true; + } else { cursor.ptr = _validateSpecial(cursor, type); + } return cursor.ptr; } @@ -319,7 +334,7 @@ private: uint8_t type = *cursor.ptr; _currElem = cursor.ptr; cursor.ptr += len + 1; - cursor.ptr = _validateElem(cursor, type); + cursor.ptr = _validateElem<true>(cursor, type); if constexpr (precise) { // See if the _id field was just validated. If so, set the global scope element. @@ -361,11 +376,16 @@ private: const char* _currElem = nullptr; // Element to validate: only the name is known to be good. typename Frames::iterator _currFrame; // Frame currently being validated. Frames _frames; // Has end pointers to check and the containing element for precise mode. + bool _firstFrameUpdated = false; // Has the first frame received nested while measuring an elem + ValidationVersion _validationVersion; }; +template <bool precise> class ColumnValidator { public: - static Status doValidateBSONColumn(const char* originalBuffer, int maxLength) noexcept { + static Status doValidateBSONColumn(const char* originalBuffer, + int maxLength, + ValidationVersion validationVersion) noexcept { // run control pointer through to end of buffer // run over literal data as directed by lengths from control // check formatting of Simple8B blocks @@ -398,7 +418,8 @@ public: return Status::OK(); } } else if (bsoncolumn::isUncompressedLiteralControlByte(control)) { - ptr += ValidateBuffer<false>(ptr, end - ptr).validateAndMeasureElem(); + ptr += ValidateBuffer<precise>(ptr, end - ptr, validationVersion) + .validateAndMeasureElem(); } else if (bsoncolumn::isInterleavedStartControlByte(control)) { // interleaved objects begin with a reference object, and then a series // of diff blocks for followup objects, ending with an EOO. Nesting interleaved @@ -430,20 +451,50 @@ public: return Status(NonConformantBSON, "Missing terminating EOO"); } }; + +template <bool precise> +Status _doValidateColumn(const char* originalBuffer, + uint64_t maxLength, + ValidationVersion validationVersion) { + if constexpr (precise) { + // First try validating using the fast but less precise version. That version will return + // a not-OK status for objects with CodeWScope or nesting exceeding 32 levels. These cases + // and actual failures will rerun the precise version that gives a detailed error context. + if (MONGO_likely(ColumnValidator<false>::doValidateBSONColumn( + originalBuffer, maxLength, validationVersion) + .isOK())) + return Status::OK(); + + return ColumnValidator<true>::doValidateBSONColumn( + originalBuffer, maxLength, validationVersion); + } else { + return ColumnValidator<false>::doValidateBSONColumn( + originalBuffer, maxLength, validationVersion); + } +} } // namespace -Status validateBSON(const char* originalBuffer, uint64_t maxLength) noexcept { +Status validateBSON(const char* originalBuffer, + uint64_t maxLength, + ValidationVersion validationVersion) noexcept { // First try validating using the fast but less precise version. That version will return // a not-OK status for objects with CodeWScope or nesting exceeding 32 levels. These cases and // actual failures will rerun the precise version that gives a detailed error context. - if (MONGO_likely(ValidateBuffer<false>(originalBuffer, maxLength).validate().isOK())) + if (MONGO_likely( + ValidateBuffer<false>(originalBuffer, maxLength, validationVersion).validate().isOK())) return Status::OK(); - return ValidateBuffer<true>(originalBuffer, maxLength).validate(); + return ValidateBuffer<true>(originalBuffer, maxLength, validationVersion).validate(); +} + +Status validateBSON(const BSONObj& obj, ValidationVersion validationVersion) noexcept { + return validateBSON(obj.objdata(), obj.objsize(), validationVersion); } -Status validateBSONColumn(const char* originalBuffer, int maxLength) noexcept { - return ColumnValidator::doValidateBSONColumn(originalBuffer, maxLength); +Status validateBSONColumn(const char* originalBuffer, + int maxLength, + ValidationVersion validationVersion) noexcept { + return _doValidateColumn<true>(originalBuffer, maxLength, validationVersion); } } // namespace mongo diff --git a/src/mongo/bson/bson_validate.h b/src/mongo/bson/bson_validate.h index a737f61b129..add1c18b55b 100644 --- a/src/mongo/bson/bson_validate.h +++ b/src/mongo/bson/bson_validate.h @@ -36,6 +36,17 @@ namespace mongo { +enum ValidationVersion { + /* Original validator */ + V1_Original = 1, + /* Adds validation for the content of Column-typed BinData */ + V2_Column = 2 +}; + +// When adding new versions of BSON validation, update both this and the range and the +// default for the server parameter in src/mongo/bson/bson_validate.idl +static constexpr ValidationVersion currentValidationVersion = V2_Column; + /** * Checks that the buf holds a BSON object as defined in http://bsonspec.org/spec.html. * Note that maxLength is the buffer size, NOT the BSON size. @@ -49,8 +60,15 @@ namespace mongo { * validity, code validity, correct length and formatting of binary subtypes, etc. * Length is only limited by the buffer's maxLength and the inherent 2GB - 1 format limitation. */ -Status validateBSON(const char* buf, uint64_t maxLength) noexcept; +Status validateBSON(const char* buf, + uint64_t maxLength, + ValidationVersion validationVersion = currentValidationVersion) noexcept; + +Status validateBSON(const BSONObj& obj, + ValidationVersion validationVersion = currentValidationVersion) noexcept; -Status validateBSONColumn(const char* buf, int maxLength) noexcept; +Status validateBSONColumn(const char* buf, + int maxLength, + ValidationVersion validationVersion = currentValidationVersion) noexcept; } // namespace mongo diff --git a/src/mongo/bson/bson_validate.idl b/src/mongo/bson/bson_validate.idl new file mode 100644 index 00000000000..9d4bb7b9fbe --- /dev/null +++ b/src/mongo/bson/bson_validate.idl @@ -0,0 +1,61 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +# BSON validate modes + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/idl/basic_types.idl" + +enums: + BSONValidateMode: + description: "The validate mode that dbCheck will run for BSON consistency checks" + type: string + # kDefault: Only fast structural BSON consistency checks + # kExtended: Structural BSON consistency and extra fast checks on BSON specifications. + # kFull: Structural BSON consistency and extra comprehensive checks on BSON specifications. + values: + kDefault: "kDefault" + kExtended: "kExtended" + kFull: "kFull" + +# Server paramaters for validation. +# Range and default value should be kept in sync with src/mongo/bson/bson_validate.h + +server_parameters: + bsonTestValidationVersion: + description: "The version of the BSON validator that will be used to check correctness of data in test environments" + set_at: startup + cpp_vartype: int + cpp_varname: bsonTestValidationVersion + validator: + gte: 1 + lte: 2 + default: 2
\ No newline at end of file diff --git a/src/mongo/bson/bson_validate_test.cpp b/src/mongo/bson/bson_validate_test.cpp index b105e0f58e2..29e5f1536d2 100644 --- a/src/mongo/bson/bson_validate_test.cpp +++ b/src/mongo/bson/bson_validate_test.cpp @@ -51,17 +51,17 @@ using std::unique_ptr; void appendInvalidStringElement(const char* fieldName, BufBuilder* bb) { // like a BSONObj string, but without a NUL terminator. bb->appendChar(String); - bb->appendStr(fieldName, /*withNUL*/ true); + bb->appendCStr(fieldName); bb->appendNum(4); - bb->appendStr("asdf", /*withNUL*/ false); + bb->appendStrBytes("asdf"); // Missing required final NUL. } TEST(BSONValidate, Basic) { BSONObj x; - ASSERT_TRUE(x.valid()); + ASSERT_TRUE(validateBSON(x).isOK()); x = BSON("x" << 1); - ASSERT_TRUE(x.valid()); + ASSERT_TRUE(validateBSON(x).isOK()); } TEST(BSONValidate, RandomData) { @@ -87,7 +87,7 @@ TEST(BSONValidate, RandomData) { ASSERT_EQUALS(size, o.objsize()); - if (o.valid()) { + if (validateBSON(o).isOK()) { numValid++; jsonSize += o.jsonString().size(); ASSERT_OK(validateBSON(o.objdata(), o.objsize())); @@ -138,7 +138,7 @@ TEST(BSONValidate, MuckingData1) { data[i] = 0xc8U; numToRun++; - if (mine.valid()) { + if (validateBSON(mine).isOK()) { numValid++; jsonSize += mine.jsonString().size(); ASSERT_OK(validateBSON(mine.objdata(), mine.objsize())); @@ -368,7 +368,7 @@ TEST(BSONValidateFast, StringHasSomething) { BufBuilder bb; BSONObjBuilder ob(bb); bb.appendChar(String); - bb.appendStr("x", /*withNUL*/ true); + bb.appendCStr("x"); bb.appendNum(0); const BSONObj x = ob.done(); ASSERT_EQUALS(5 // overhead @@ -483,6 +483,7 @@ TEST_F(BSONValidateColumn, BSONColumnInBSON) { TEST_F(BSONValidateColumn, BSONColumnMissingEOO) { BSONColumnBuilder cb(""); + cb.append(BSON("a" << "deadbeef") .getField("a")); @@ -694,4 +695,12 @@ TEST_F(BSONValidateColumn, BSONColumnBadExtendedSelector) { ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); } +TEST(BSONValidateColumn, BSONColumnWithCodeWScope) { + BSONObj obj = BSON("a" << BSONCodeWScope("code", BSON("c" << 1))); + BSONColumnBuilder cb(""); + cb.append(obj.getField("a")); + BSONBinData columnData = cb.finalize(); + ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); +} + } // namespace diff --git a/src/mongo/bson/bsonelement.cpp b/src/mongo/bson/bsonelement.cpp index 2409084bcfc..0990b1a18c7 100644 --- a/src/mongo/bson/bsonelement.cpp +++ b/src/mongo/bson/bsonelement.cpp @@ -46,6 +46,7 @@ #include "mongo/logv2/log.h" #include "mongo/platform/strnlen.h" #include "mongo/util/base64.h" +#include "mongo/util/decimal_counter.h" #include "mongo/util/duration.h" #include "mongo/util/hex.h" #include "mongo/util/scopeguard.h" @@ -444,6 +445,7 @@ int BSONElement::compareElements(const BSONElement& l, */ std::vector<BSONElement> BSONElement::Array() const { chk(mongo::Array); + std::vector<BSONElement> v; BSONObjIterator i(Obj()); while (i.more()) { @@ -464,6 +466,23 @@ std::vector<BSONElement> BSONElement::Array() const { return v; } +std::vector<BSONElement> BSONElement::ArrayVerifyIndexes() const { + chk(mongo::Array); + + std::vector<BSONElement> v; + DecimalCounter<std::uint32_t> counter(0); + for (auto element : Obj()) { + auto fieldName = element.fieldNameStringData(); + uassert(ErrorCodes::BadValue, + fmt::format( + "Invalid array index field name: \"{}\", expected \"{}\"", fieldName, counter), + fieldName == counter); + counter++; + v.push_back(element); + } + return v; +} + int BSONElement::woCompare(const BSONElement& elem, ComparisonRulesSet rules, const StringData::ComparatorInterface* comparator) const { @@ -646,41 +665,13 @@ BSONElement BSONElement::operator[](StringData field) const { } namespace { -MONGO_COMPILER_NOINLINE void msgAssertedBadType [[noreturn]] (const char* data) { - // We intentionally read memory that may be out of the allocated memory's boundary, so do not - // do this when the address sanitizer is enabled. We do this in an attempt to log as much - // context about the failure, even if that risks undefined behavior or a segmentation fault. -#if !__has_feature(address_sanitizer) - bool logMemory = true; -#else - bool logMemory = false; -#endif - str::stream output; - if (!logMemory) { - output << fmt::format("BSONElement: bad type {0:d} @ {1:p}", *data, data); - } else { - // To reduce the risk of a segmentation fault, only print the bytes in the 32-bit aligned - // block in which the address is located (i.e. round down to the lowest multiple of 32). The - // hope is that it's safe to read memory that may fall within the same cache line. Generate - // a mask to zero-out the last bits for a block-aligned address. - // Ex: Inverse of 0x1F (32 - 1) looks like 0xFFFFFFE0, and ANDed with the pointer, zeroes - // the lowest 5 bits, giving the starting address of a 32-bit block. - const size_t blockSize = 32; - const size_t mask = ~(blockSize - 1); - const char* startAddr = - reinterpret_cast<const char*>(reinterpret_cast<uintptr_t>(data) & mask); - const size_t offset = data - startAddr; - - output << fmt::format( - "BSONElement: bad type {0:d} @ {1:p} at offset {2:d} in block: ", *data, data, offset); - - for (size_t i = 0; i < blockSize; i++) { - output << fmt::format("{0:#x} ", static_cast<uint8_t>(startAddr[i])); - } - } - msgasserted(10320, output); +MONGO_COMPILER_NOINLINE void msgAssertedBadType [[noreturn]] (int8_t type) { + int err = 10320; // work around linter + LOGV2_ERROR(err, "BSONElement: bad type", "type"_attr = zeroPaddedHex(type)); + uasserted(err, "BSONElement: bad type"); } + } // namespace int BSONElement::computeSize(int8_t type, const char* elem, int fieldNameSize, int bufSize) { @@ -751,7 +742,7 @@ int BSONElement::computeSize(int8_t type, const char* elem, int fieldNameSize, i if (type == MaxKey || type == MinKey) return fieldNameSize + 1; if (type != BSONType::RegEx) - msgAssertedBadType(elem); + msgAssertedBadType(type); // RegEx is two c-strings back-to-back. const char* p = elem + fieldNameSize + 1; diff --git a/src/mongo/bson/bsonelement.h b/src/mongo/bson/bsonelement.h index 251ed41dd2d..f319c2981fb 100644 --- a/src/mongo/bson/bsonelement.h +++ b/src/mongo/bson/bsonelement.h @@ -143,7 +143,14 @@ public: bool Bool() const { return chk(mongo::Bool).boolean(); } + std::vector<BSONElement> Array() const; // see implementation for detailed comments + + /** + * Like Array() above, but, if the array keys are not in sequential order or are otherwise + * invalid, an exception is thrown. + */ + std::vector<BSONElement> ArrayVerifyIndexes() const; mongo::OID OID() const { return chk(jstOID).__oid(); } diff --git a/src/mongo/bson/bsonelement_test.cpp b/src/mongo/bson/bsonelement_test.cpp index 4e5bc3f6f07..0921823f92f 100644 --- a/src/mongo/bson/bsonelement_test.cpp +++ b/src/mongo/bson/bsonelement_test.cpp @@ -446,5 +446,55 @@ TEST(BSONElementTryCoeceToLongLongTest, CoerceFails) { ASSERT_NOT_OK(result) << " for input document " << testCase.toString(); } } + +TEST(BSONElement, ArrayToVectorFunctionsBehaveCorrectlyWithValidArray) { + // Create a valid array by creating a BSONObj with contiguous array indexes that is then + // passed to the BSONArray ctor. + BSONObj updateArrAsObj = BSON("0" + << "foo" + << "1" + << "bar"); + BSONArray updateArr(updateArrAsObj); + BSONObj parentObj = BSON("arr" << updateArr); + auto arrElem = parentObj.getField("arr"); + + // Both 'Array()' and 'ArrayVerifyIndexes()' will not throw, and instead create vectors of + // size 2. + auto elementVector = arrElem.Array(); + ASSERT_EQ(elementVector.size(), 2); + ASSERT(elementVector[0].binaryEqual(updateArrAsObj.getField("0"))); + ASSERT(elementVector[1].binaryEqual(updateArrAsObj.getField("1"))); + + auto verifiedVector = arrElem.ArrayVerifyIndexes(); + ASSERT_EQ(verifiedVector.size(), 2); + + // The two vectors should have the same elements. + ASSERT(elementVector[0].binaryEqual(verifiedVector[0])); + ASSERT(elementVector[1].binaryEqual(verifiedVector[1])); +} + +TEST(BSONElement, ArrayVerifyIndexesThrowsOnInvalidArrayIndexes) { + // Create our invalid array by creating a BSONObj with non contiguous array indexes that is then + // passed to the BSONArray ctor. + BSONObj updateArrAsObj = BSON("0" + << "foo" + << "2" + << "bar"); + BSONArray updateArr(updateArrAsObj); + BSONObj parentObj = BSON("badArray" << updateArr); + auto arrElem = parentObj.getField("badArray"); + + // The regular 'Array()' will not throw, and instead create an EOO BSONElement at index 1. + auto elementVector = arrElem.Array(); + ASSERT_EQ(elementVector.size(), 3); + + ASSERT(elementVector[0].binaryEqual(updateArrAsObj.getField("0"))); + ASSERT(elementVector[1].binaryEqual(BSONElement())); + ASSERT(elementVector[2].binaryEqual(updateArrAsObj.getField("2"))); + + // 'ArrayVerifyIndexes()', on the other hand, will throw. + ASSERT_THROWS(arrElem.ArrayVerifyIndexes(), ExceptionFor<ErrorCodes::BadValue>); +} + } // namespace } // namespace mongo diff --git a/src/mongo/bson/bsonobj.cpp b/src/mongo/bson/bsonobj.cpp index 7b5fa7dae04..f0c5946565c 100644 --- a/src/mongo/bson/bsonobj.cpp +++ b/src/mongo/bson/bsonobj.cpp @@ -138,41 +138,71 @@ BSONObj BSONObj::getOwned(const BSONObj& obj) { return obj.getOwned(); } -BSONObj BSONObj::redact(bool onlyEncryptedFields) const { +BSONObj BSONObj::redact(RedactLevel level, + std::function<std::string(const BSONElement&)> fieldNameRedactor) const { _validateUnownedSize(objsize()); // Helper to get an "internal function" to be able to do recursion struct redactor { - void appendRedactedElem(BSONObjBuilder& builder, const BSONElement& e, bool appendMask) { + void appendRedactedElem(BSONObjBuilder& builder, + const StringData& fieldNameString, + bool appendMask) { if (appendMask) { - builder.append(e.fieldNameStringData(), "###"_sd); + builder.append(fieldNameString, "###"_sd); } else { - builder.appendNull(e.fieldNameStringData()); + builder.appendNull(fieldNameString); } } void operator()(BSONObjBuilder& builder, const BSONObj& obj, bool appendMask, - bool onlyEncryptedFields) { + RedactLevel level, + std::function<std::string(const BSONElement&)> fieldNameRedactor) { for (BSONElement e : obj) { + StringData fieldNameString; + // Temporarily allocated string that must live long enough to be copied by builder. + std::string tempString; + if (!fieldNameRedactor) { + fieldNameString = e.fieldNameStringData(); + } else { + tempString = fieldNameRedactor(e); + fieldNameString = {tempString}; + } if (e.type() == Object) { - BSONObjBuilder subBuilder = builder.subobjStart(e.fieldNameStringData()); - operator()(subBuilder, e.Obj(), appendMask, onlyEncryptedFields); + BSONObjBuilder subBuilder = builder.subobjStart(fieldNameString); + operator()(subBuilder, e.Obj(), appendMask, level, fieldNameRedactor); subBuilder.done(); } else if (e.type() == Array) { - BSONObjBuilder subBuilder = builder.subarrayStart(e.fieldNameStringData()); - operator()(subBuilder, e.Obj(), appendMask, onlyEncryptedFields); + BSONObjBuilder subBuilder = builder.subarrayStart(fieldNameString); + operator()(subBuilder, e.Obj(), appendMask, level, fieldNameRedactor); subBuilder.done(); } else { - if (onlyEncryptedFields) { - if (e.type() == BinData && e.binDataType() == BinDataType::Encrypt) { - appendRedactedElem(builder, e, appendMask); - } else { - builder.append(e); + // SERVER-79068 Templatizing this could be a good opportunity for performance + // improvements. + switch (level) { + case RedactLevel::all: { + appendRedactedElem(builder, fieldNameString, appendMask); + break; + } + case RedactLevel::encryptedAndSensitive: { + if (e.type() == BinData && + (e.binDataType() == BinDataType::Encrypt || + e.binDataType() == BinDataType::Sensitive)) { + appendRedactedElem(builder, fieldNameString, appendMask); + } else { + builder.append(e); + } + break; + } + case RedactLevel::sensitiveOnly: { + if (e.type() == BinData && e.binDataType() == BinDataType::Sensitive) { + appendRedactedElem(builder, fieldNameString, appendMask); + } else { + builder.append(e); + } + break; } - } else { - appendRedactedElem(builder, e, appendMask); } } } @@ -181,7 +211,7 @@ BSONObj BSONObj::redact(bool onlyEncryptedFields) const { try { BSONObjBuilder builder; - redactor()(builder, *this, /*appendMask=*/true, onlyEncryptedFields); + redactor()(builder, *this, /*appendMask=*/true, level, fieldNameRedactor); return builder.obj(); } catch (const ExceptionFor<ErrorCodes::BSONObjectTooLarge>&) { } @@ -191,7 +221,7 @@ BSONObj BSONObj::redact(bool onlyEncryptedFields) const { // we use BSONType::jstNull, which ensures the redacted object will not be larger than the // original. BSONObjBuilder builder; - redactor()(builder, *this, /*appendMask=*/false, onlyEncryptedFields); + redactor()(builder, *this, /*appendMask=*/false, level, fieldNameRedactor); return builder.obj(); } @@ -298,10 +328,6 @@ BSONObj BSONObj::jsonStringBuffer(JsonStringFormat format, } } -bool BSONObj::valid() const { - return validateBSON(objdata(), objsize()).isOK(); -} - int BSONObj::woCompare(const BSONObj& r, const Ordering& o, ComparisonRulesSet rules, diff --git a/src/mongo/bson/bsonobj.h b/src/mongo/bson/bsonobj.h index a5270adfd25..5793f9e5212 100644 --- a/src/mongo/bson/bsonobj.h +++ b/src/mongo/bson/bsonobj.h @@ -264,10 +264,14 @@ public: */ BSONObj copy() const; + enum class RedactLevel : int8_t { all, encryptedAndSensitive, sensitiveOnly }; + /** * @return a new full (and owned) redacted copy of the object. */ - BSONObj redact(bool onlyEncryptedFields = false) const; + BSONObj redact( + RedactLevel level = RedactLevel::all, + std::function<std::string(const BSONElement&)> fieldNameRedactor = nullptr) const; /** * Readable representation of a BSON object in an extended JSON-style notation. @@ -638,11 +642,6 @@ public: bool hasFieldNames() const; /** - * Returns true if this object is valid and returns false otherwise. - */ - bool valid() const; - - /** * add all elements of the object to the specified vector */ void elems(std::vector<BSONElement>&) const; diff --git a/src/mongo/bson/bsonobjbuilder.cpp b/src/mongo/bson/bsonobjbuilder.cpp index e8e50fd38a7..096ba297e99 100644 --- a/src/mongo/bson/bsonobjbuilder.cpp +++ b/src/mongo/bson/bsonobjbuilder.cpp @@ -182,7 +182,7 @@ Derived& BSONObjBuilderBase<Derived, B>::appendMaxForType(StringData fieldName, template <class Derived, class B> Derived& BSONObjBuilderBase<Derived, B>::appendDate(StringData fieldName, Date_t dt) { _b.appendNum((char)Date); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum(dt.toMillisSinceEpoch()); return static_cast<Derived&>(*this); } diff --git a/src/mongo/bson/bsonobjbuilder.h b/src/mongo/bson/bsonobjbuilder.h index be826d5a824..008ad7953fb 100644 --- a/src/mongo/bson/bsonobjbuilder.h +++ b/src/mongo/bson/bsonobjbuilder.h @@ -153,7 +153,7 @@ public: // do not append eoo, that would corrupt us. the builder auto appends when done() is called. verify(!e.eoo()); _b.appendNum((char)e.type()); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendBuf((void*)e.value(), e.valuesize()); return static_cast<Derived&>(*this); } @@ -161,7 +161,7 @@ public: /** add a subobject as a member */ Derived& append(StringData fieldName, BSONObj subObj) { _b.appendNum((char)Object); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendBuf((void*)subObj.objdata(), subObj.objsize()); return static_cast<Derived&>(*this); } @@ -176,7 +176,7 @@ public: verify(size > 4 && size < 100000000); _b.appendNum((char)Object); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendBuf((void*)objdata, size); return static_cast<Derived&>(*this); } @@ -194,7 +194,7 @@ public: */ B& subobjStart(StringData fieldName) { _b.appendNum((char)Object); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); return _b; } @@ -203,7 +203,7 @@ public: */ Derived& appendArray(StringData fieldName, const BSONObj& subObj) { _b.appendNum((char)Array); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendBuf((void*)subObj.objdata(), subObj.objsize()); return static_cast<Derived&>(*this); @@ -216,14 +216,14 @@ public: the subarray's body */ B& subarrayStart(StringData fieldName) { _b.appendNum((char)Array); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); return _b; } /** Append a boolean element */ Derived& appendBool(StringData fieldName, int val) { _b.appendNum((char)Bool); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum((char)(val ? 1 : 0)); return static_cast<Derived&>(*this); } @@ -233,7 +233,7 @@ public: Derived& append(StringData fieldName, const T& n) { constexpr BSONType type = BSONObjAppendFormat<T>::value; _b.appendNum(static_cast<char>(type)); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); if constexpr (type == Bool) { _b.appendNum(static_cast<char>(n)); } else if constexpr (type == NumberInt) { @@ -284,7 +284,7 @@ public: */ Derived& appendOID(StringData fieldName, OID* oid = nullptr, bool generateIfBlank = false) { _b.appendNum((char)jstOID); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); if (oid) _b.appendBuf(oid->view().view(), OID::kOIDSize); else { @@ -305,7 +305,7 @@ public: */ Derived& append(StringData fieldName, OID oid) { _b.appendNum((char)jstOID); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendBuf(oid.view().view(), OID::kOIDSize); return static_cast<Derived&>(*this); } @@ -324,7 +324,7 @@ public: */ Derived& appendTimeT(StringData fieldName, time_t dt) { _b.appendNum((char)Date); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum(static_cast<unsigned long long>(dt) * 1000); return static_cast<Derived&>(*this); } @@ -343,9 +343,9 @@ public: */ Derived& appendRegex(StringData fieldName, StringData regex, StringData options = "") { _b.appendNum((char)RegEx); - _b.appendStr(fieldName); - _b.appendStr(regex); - _b.appendStr(options); + _b.appendCStr(fieldName); + _b.appendCStr(regex); + _b.appendCStr(options); return static_cast<Derived&>(*this); } @@ -356,9 +356,9 @@ public: Derived& appendCode(StringData fieldName, StringData code) { _b.appendNum((char)Code); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum((int)code.size() + 1); - _b.appendStr(code); + _b.appendStrBytesAndNul(code); return static_cast<Derived&>(*this); } @@ -370,7 +370,7 @@ public: @param sz size includes terminating null character */ Derived& append(StringData fieldName, const char* str, int sz) { _b.appendNum((char)String); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum((int)sz); _b.appendBuf(str, sz); @@ -383,17 +383,17 @@ public: /** Append a string element */ Derived& append(StringData fieldName, StringData str) { _b.appendNum((char)String); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum((int)str.size() + 1); - _b.appendStr(str, true); + _b.appendStrBytesAndNul(str); return static_cast<Derived&>(*this); } Derived& appendSymbol(StringData fieldName, StringData symbol) { _b.appendNum((char)Symbol); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum((int)symbol.size() + 1); - _b.appendStr(symbol); + _b.appendStrBytesAndNul(symbol); return static_cast<Derived&>(*this); } @@ -404,7 +404,7 @@ public: /** Append a Null element to the object */ Derived& appendNull(StringData fieldName) { _b.appendNum((char)jstNULL); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); return static_cast<Derived&>(*this); } @@ -412,13 +412,13 @@ public: // Append an element that is less than all other keys. Derived& appendMinKey(StringData fieldName) { _b.appendNum((char)MinKey); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); return static_cast<Derived&>(*this); } // Append an element that is greater than all other keys. Derived& appendMaxKey(StringData fieldName) { _b.appendNum((char)MaxKey); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); return static_cast<Derived&>(*this); } @@ -439,9 +439,9 @@ public: */ Derived& appendDBRef(StringData fieldName, StringData ns, const OID& oid) { _b.appendNum((char)DBRef); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum((int)ns.size() + 1); - _b.appendStr(ns); + _b.appendStrBytesAndNul(ns); _b.appendBuf(oid.view().view(), OID::kOIDSize); return static_cast<Derived&>(*this); @@ -460,7 +460,7 @@ public: */ Derived& appendBinData(StringData fieldName, int len, BinDataType type, const void* data) { _b.appendNum((char)BinData); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum(len); _b.appendNum((char)type); _b.appendBuf(data, len); @@ -480,7 +480,7 @@ public: */ Derived& appendBinDataArrayDeprecated(const char* fieldName, const void* data, int len) { _b.appendNum((char)BinData); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum(len + 4); _b.appendNum((char)0x2); _b.appendNum(len); @@ -494,10 +494,10 @@ public: */ Derived& appendCodeWScope(StringData fieldName, StringData code, const BSONObj& scope) { _b.appendNum((char)CodeWScope); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); _b.appendNum((int)(4 + 4 + code.size() + 1 + scope.objsize())); _b.appendNum((int)code.size() + 1); - _b.appendStr(code); + _b.appendStrBytesAndNul(code); _b.appendBuf((void*)scope.objdata(), scope.objsize()); return static_cast<Derived&>(*this); @@ -509,7 +509,7 @@ public: Derived& appendUndefined(StringData fieldName) { _b.appendNum((char)Undefined); - _b.appendStr(fieldName); + _b.appendCStr(fieldName); return static_cast<Derived&>(*this); } diff --git a/src/mongo/bson/bsontypes.cpp b/src/mongo/bson/bsontypes.cpp index 8be3c5e1d04..1f9112f2dc4 100644 --- a/src/mongo/bson/bsontypes.cpp +++ b/src/mongo/bson/bsontypes.cpp @@ -199,6 +199,8 @@ const char* typeName(BinDataType type) { return "encrypt"; case Column: return "column"; + case Sensitive: + return "sensitive"; case bdtCustom: return "Custom"; default: @@ -217,6 +219,7 @@ bool isValidBinDataType(int type) { case Encrypt: case Column: case bdtCustom: + case Sensitive: return true; default: return false; diff --git a/src/mongo/bson/bsontypes.h b/src/mongo/bson/bsontypes.h index 0dc2fe8a4ab..5204f1c3f72 100644 --- a/src/mongo/bson/bsontypes.h +++ b/src/mongo/bson/bsontypes.h @@ -198,8 +198,9 @@ enum BinDataType { bdtUUID = 3, /* deprecated */ newUUID = 4, /* language-independent UUID format across all drivers */ MD5Type = 5, - Encrypt = 6, /* encryption placeholder or encrypted data */ - Column = 7, /* compressed column */ + Encrypt = 6, /* encryption placeholder or encrypted data */ + Column = 7, /* compressed column */ + Sensitive = 8, /* data that should be redacted and protected from unnecessary exposure */ bdtCustom = 128 }; diff --git a/src/mongo/bson/simple_bsonobj_comparator.h b/src/mongo/bson/simple_bsonobj_comparator.h index ed08cbdd939..b073491747f 100644 --- a/src/mongo/bson/simple_bsonobj_comparator.h +++ b/src/mongo/bson/simple_bsonobj_comparator.h @@ -95,6 +95,10 @@ public: }; }; +inline auto simpleHash(const BSONObj& obj) { + return SimpleBSONObjComparator::kInstance.hash(obj); +} + /** * A set of BSONObjs that performs comparisons with simple binary semantics. */ diff --git a/src/mongo/bson/timestamp.h b/src/mongo/bson/timestamp.h index 3d0befd7450..28bf9cb31cd 100644 --- a/src/mongo/bson/timestamp.h +++ b/src/mongo/bson/timestamp.h @@ -139,7 +139,7 @@ public: // No endian conversions needed, since we store in-memory representation // in little endian format, regardless of target endian. builder.appendNum(static_cast<char>(bsonTimestamp)); - builder.appendStr(fieldName); + builder.appendCStr(fieldName); builder.appendNum(asULL()); } BSONObj toBSON() const; diff --git a/src/mongo/bson/util/builder.h b/src/mongo/bson/util/builder.h index 5d7d2d5673c..883b19a0ffe 100644 --- a/src/mongo/bson/util/builder.h +++ b/src/mongo/bson/util/builder.h @@ -57,6 +57,7 @@ #include "mongo/util/itoa.h" #include "mongo/util/shared_buffer.h" #include "mongo/util/shared_buffer_fragment.h" +#include "mongo/util/str_basic.h" namespace mongo { @@ -325,7 +326,7 @@ public: @return point to region that was skipped. pointer may change later (on realloc), so for immediate use only */ - char* skip(int n) { + char* skip(size_t n) { return grow(n); } @@ -394,7 +395,7 @@ public: } void appendBuf(const void* src, size_t len) { if (len) - memcpy(grow((int)len), src, len); + memcpy(grow(len), src, len); } template <class T> @@ -402,9 +403,36 @@ public: appendBuf(&s, sizeof(T)); } - void appendStr(StringData str, bool includeEndingNull = true) { - const int len = str.size() + (includeEndingNull ? 1 : 0); - str.copyTo(grow(len), includeEndingNull); + /** + * Appends the raw bytes of str with no NUL terminator. + */ + void appendStrBytes(StringData str) { + str.copy(grow(str.size()), str.size()); + } + + /** + * Appends the raw bytes of str followed by a final NUL byte. + * + * WARNING: only use this method for formats with explicit string lengths where the NUL byte is + * not used to find the end. This method does not check for embedded NUL bytes, so they can + * trick a parser into thinking the string has ended. Use appendCStr() instead for that use + * case. + */ + void appendStrBytesAndNul(StringData str) { + auto dest = grow(str.size() + 1); + dest += str.copy(dest, str.size()); + *dest = '\0'; + } + + /** + * Appends the raw bytes of str followed by a final NUL byte, throwing if str already has an + * embedded NUL byte. + * + * This method is intended to pair with BufReader::readCStr() on the parse side. + */ + void appendCStr(StringData str) { + str::uassertNoEmbeddedNulBytes(str); + appendStrBytesAndNul(str); } /** Returns the length of data in the current buffer */ @@ -423,8 +451,8 @@ public: } /* returns the pre-grow write position */ - char* grow(int by) { - if (MONGO_likely(by <= _end - _nextByte)) { + char* grow(size_t by) { + if (MONGO_likely(by <= static_cast<size_t>(_end - _nextByte))) { char* oldNextByte = _nextByte; _nextByte += by; return oldNextByte; @@ -751,7 +779,7 @@ public: } void append(StringData str) { - str.copyTo(_buf.grow(str.size()), false); + _buf.appendStrBytes(str); } void reset(int maxSize = 0) { diff --git a/src/mongo/bson/util/builder_test.cpp b/src/mongo/bson/util/builder_test.cpp index a4ed4c87115..0ad7518ffe5 100644 --- a/src/mongo/bson/util/builder_test.cpp +++ b/src/mongo/bson/util/builder_test.cpp @@ -38,12 +38,43 @@ TEST(Builder, String1) { ASSERT_EQUALS(small, "eliot"); BufBuilder bb; - bb.appendStr(small); + bb.appendCStr(small); + + ASSERT_EQUALS(bb.len(), small.size() + 1); + ASSERT_EQUALS(bb.buf()[small.size()], 0); ASSERT_EQUALS(0, strcmp(bb.buf(), "eliot")); ASSERT_EQUALS(0, strcmp("eliot", bb.buf())); } +TEST(Builder, StringNulByteHandling) { + auto hasNulByte = "hello\0world"_sd; + + { + // appendCStr() throws without changing bb; + BufBuilder bb; + ASSERT_THROWS_CODE(bb.appendCStr(hasNulByte), DBException, 9527900); + ASSERT_EQ(bb.len(), 0); + } + + { + // appendStrBytes appends embedded NUL without terminator. + BufBuilder bb; + bb.appendStrBytes(hasNulByte); + ASSERT_EQ(StringData(bb.buf(), bb.len()), hasNulByte); + } + + { + // appendStrBytesAndNul appends embedded NUL and NUL terminator. + BufBuilder bb; + bb.appendStrBytesAndNul(hasNulByte); + // Since hasNulByte points to a string literal, we know that + // *(hasNulByte.data() + hasNulByte.size()) is valid and == '\0' + ASSERT_EQ(StringData(bb.buf(), bb.len()), + StringData(hasNulByte.rawData(), hasNulByte.size() + 1)); + } +} + TEST(Builder, StringBuilderAddress) { const void* longPtr = reinterpret_cast<const void*>(-1); const void* shortPtr = reinterpret_cast<const void*>(static_cast<uintptr_t>(0xDEADBEEF)); diff --git a/src/mongo/bson/util/simple8b_test.cpp b/src/mongo/bson/util/simple8b_test.cpp index c612c221aec..df8c849f73b 100644 --- a/src/mongo/bson/util/simple8b_test.cpp +++ b/src/mongo/bson/util/simple8b_test.cpp @@ -1437,6 +1437,7 @@ TEST(Simple8b, ResetRLEAfterLargeValue) { // The second block should be an RLE block ASSERT_GT(size, 16); - uint64_t secondBlock = *((uint64_t*)(data.get() + sizeof(uint64_t))); + uint64_t secondBlock = + ConstDataView(data.get() + sizeof(uint64_t)).read<LittleEndian<uint64_t>>(); ASSERT_TRUE((secondBlock & kBaseSelectorMask) == kRleSelector); } diff --git a/src/mongo/client/SConscript b/src/mongo/client/SConscript index 5010be181f0..cdaa0819473 100644 --- a/src/mongo/client/SConscript +++ b/src/mongo/client/SConscript @@ -72,6 +72,7 @@ if get_option('ssl') == 'on': ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/server_options_core', # For object_check.h '$BUILD_DIR/mongo/idl/idl_parser', '$BUILD_DIR/third_party/shim_kms_message', @@ -117,6 +118,7 @@ saslClientEnv.Library( source=saslClientSource, LIBDEPS=[ '$BUILD_DIR/mongo/base/secure_allocator', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/bson/util/bson_extract', '$BUILD_DIR/mongo/executor/remote_command', '$BUILD_DIR/mongo/rpc/command_status', diff --git a/src/mongo/client/cyrus_sasl_client_session.cpp b/src/mongo/client/cyrus_sasl_client_session.cpp index 84ae1ab0b5a..2bca7e1b1de 100644 --- a/src/mongo/client/cyrus_sasl_client_session.cpp +++ b/src/mongo/client/cyrus_sasl_client_session.cpp @@ -47,7 +47,8 @@ void saslSetError(sasl_conn_t* conn, const std::string& msg) { } SaslClientSession* createCyrusSaslClientSession(const std::string& mech) { - if ((mech == "SCRAM-SHA-1") || (mech == "SCRAM-SHA-256") || mech == "MONGODB-AWS") { + if ((mech == "SCRAM-SHA-1") || (mech == "SCRAM-SHA-256") || (mech == "PLAIN") || + mech == "MONGODB-AWS") { return new NativeSaslClientSession(); } return new CyrusSaslClientSession(); @@ -121,6 +122,28 @@ int saslClientLogSwallow(void* context, int priority, const char* message) throw } /** + * Implements the Cyrus SASL default_verifyfile_cb interface registered in the + * Cyrus SASL library to verify, and then accept or reject, the loading of + * plugin libraries from the target directory. + * + * On Windows environments, disable loading of plugin files. + */ +int saslClientVerifyPluginFile(void*, const char*, sasl_verify_type_t type) { + + if (type != SASL_VRFY_PLUGIN) { + return SASL_OK; + } + +#ifdef _WIN32 + return SASL_CONTINUE; // A non-SASL_OK response indicates to Cyrus SASL that it + // should not load a file. This effectively disables + // loading plugins from path on Windows. +#else + return SASL_OK; +#endif +} + +/** * Initializes the client half of the SASL library, but is effectively a no-op if the client * application has already done it. * @@ -136,6 +159,7 @@ MONGO_INITIALIZER_WITH_PREREQUISITES(CyrusSaslClientContext, (InitializerContext* context) { static sasl_callback_t saslClientGlobalCallbacks[] = { {SASL_CB_LOG, SaslCallbackFn(saslClientLogSwallow), nullptr /* context */}, + {SASL_CB_VERIFYFILE, SaslCallbackFn(saslClientVerifyPluginFile), nullptr /*context*/}, {SASL_CB_LIST_END}}; // If the client application has previously called sasl_client_init(), the callbacks passed @@ -240,7 +264,7 @@ void CyrusSaslClientSession::setParameter(Parameter id, StringData value) { _secret.reset(new char[sizeof(sasl_secret_t) + value.size() + 1]); sasl_secret_t* secret = static_cast<sasl_secret_t*>(static_cast<void*>(_secret.get())); secret->len = value.size(); - value.copyTo(static_cast<char*>(static_cast<void*>(&secret->data[0])), false); + value.copy(static_cast<char*>(static_cast<void*>(&secret->data[0])), value.size()); } SaslClientSession::setParameter(id, value); } diff --git a/src/mongo/client/mongo_uri_test.cpp b/src/mongo/client/mongo_uri_test.cpp index f8c3027fb20..723f03f6c60 100644 --- a/src/mongo/client/mongo_uri_test.cpp +++ b/src/mongo/client/mongo_uri_test.cpp @@ -34,6 +34,7 @@ #include <fstream> #include "mongo/base/string_data.h" +#include "mongo/bson/bson_validate.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsontypes.h" #include "mongo/bson/json.h" @@ -582,7 +583,7 @@ BSONObj getBsonFromJsonFile(std::string fileName) { std::ifstream infile(filename.c_str()); std::string data((std::istreambuf_iterator<char>(infile)), std::istreambuf_iterator<char>()); BSONObj obj = fromjson(data); - ASSERT_TRUE(obj.valid()); + ASSERT_TRUE(validateBSON(obj).isOK()); ASSERT_TRUE(obj.hasField("tests")); BSONObj arr = obj.getField("tests").embeddedObject().getOwned(); ASSERT_TRUE(arr.couldBeArray()); diff --git a/src/mongo/client/read_preference.cpp b/src/mongo/client/read_preference.cpp index 6851e1a8af7..6322e5ac84f 100644 --- a/src/mongo/client/read_preference.cpp +++ b/src/mongo/client/read_preference.cpp @@ -106,13 +106,17 @@ ReadPreferenceSetting::ReadPreferenceSetting(ReadPreference pref, hedgingMode(std::move(hedgingMode)) {} ReadPreferenceSetting::ReadPreferenceSetting(ReadPreference pref, Seconds maxStalenessSeconds) - : ReadPreferenceSetting(pref, defaultTagSetForMode(pref), maxStalenessSeconds) {} + : ReadPreferenceSetting(pref, defaultTagSetForMode(pref), maxStalenessSeconds) { + _usedDefaultReadPrefValue = true; +} ReadPreferenceSetting::ReadPreferenceSetting(ReadPreference pref, TagSet tags) : pref(std::move(pref)), tags(std::move(tags)) {} ReadPreferenceSetting::ReadPreferenceSetting(ReadPreference pref) - : ReadPreferenceSetting(pref, defaultTagSetForMode(pref)) {} + : ReadPreferenceSetting(pref, defaultTagSetForMode(pref)) { + _usedDefaultReadPrefValue = true; +} StatusWith<ReadPreferenceSetting> ReadPreferenceSetting::fromInnerBSON(const BSONObj& readPrefObj) { std::string modeStr; diff --git a/src/mongo/client/read_preference.h b/src/mongo/client/read_preference.h index 837d7391f41..b660f8a5709 100644 --- a/src/mongo/client/read_preference.h +++ b/src/mongo/client/read_preference.h @@ -123,7 +123,9 @@ struct ReadPreferenceSetting { ReadPreferenceSetting(ReadPreference pref, Seconds maxStalenessSeconds); ReadPreferenceSetting(ReadPreference pref, TagSet tags); explicit ReadPreferenceSetting(ReadPreference pref); - ReadPreferenceSetting() : ReadPreferenceSetting(ReadPreference::PrimaryOnly) {} + ReadPreferenceSetting() : ReadPreferenceSetting(ReadPreference::PrimaryOnly) { + _usedDefaultReadPrefValue = true; + } inline bool equals(const ReadPreferenceSetting& other) const { auto hedgingModeEquals = [](const boost::optional<HedgingMode>& hedgingModeA, @@ -168,7 +170,9 @@ struct ReadPreferenceSetting { toContainingBSON(&bob); return bob.obj(); } - + bool usedDefaultReadPrefValue() const { + return _usedDefaultReadPrefValue; + } /** * Parses a ReadPreferenceSetting from a BSON document of the form: * { mode: <mode>, tags: <array of tags>, maxStalenessSeconds: Number, hedge: <hedgingMode>}. @@ -226,6 +230,9 @@ struct ReadPreferenceSetting { * Either way, it must be that a node opTime of X implies ClusterTime >= X. */ Timestamp minClusterTime{}; + +private: + bool _usedDefaultReadPrefValue = false; }; } // namespace mongo diff --git a/src/mongo/client/replica_set_monitor_manager.cpp b/src/mongo/client/replica_set_monitor_manager.cpp index 5fe13f2be81..c440f5b0b8f 100644 --- a/src/mongo/client/replica_set_monitor_manager.cpp +++ b/src/mongo/client/replica_set_monitor_manager.cpp @@ -141,10 +141,6 @@ void ReplicaSetMonitorManager::_setupTaskExecutorAndStatsInLock() { return; } - if (!_stats) { - _stats = std::make_shared<ReplicaSetMonitorManagerStats>(); - } - // construct task executor auto hookList = std::make_unique<rpc::EgressMetadataHookList>(); auto networkConnectionHook = std::make_unique<ReplicaSetMonitorManagerNetworkConnectionHook>(); @@ -223,7 +219,6 @@ shared_ptr<ReplicaSetMonitor> ReplicaSetMonitorManager::getMonitorForHost(const vector<string> ReplicaSetMonitorManager::getAllSetNames() const { vector<string> allNames; - stdx::lock_guard<Latch> lk(_mutex); for (const auto& entry : _monitors) { @@ -317,30 +312,31 @@ void ReplicaSetMonitorManager::removeAllMonitors() { } void ReplicaSetMonitorManager::report(BSONObjBuilder* builder, bool forFTDC) { - // Don't hold _mutex the whole time to avoid ever taking a monitor's mutex while holding the - // manager's mutex. Otherwise we could get a deadlock between the manager's, monitor's, and - // ShardRegistry's mutex due to the ReplicaSetMonitor's AsynchronousConfigChangeHook - // potentially calling ShardRegistry::updateConfigServerConnectionString. - auto setNames = getAllSetNames(); - - builder->appendNumber("numReplicaSetMonitorsCreated", _numMonitorsCreated); + std::vector<std::shared_ptr<ReplicaSetMonitor>> monitors; + int numMonitorsCreated; + // Gather relevant data under the lock. Separate out writing it to BSON. + { + stdx::lock_guard lk(_mutex); + _doGarbageCollectionLocked(lk); + for (const auto& [_, weakMonitor] : _monitors) { + if (auto monitor = weakMonitor.lock()) + monitors.push_back(std::move(monitor)); + } + numMonitorsCreated = _numMonitorsCreated; + } + // Now write out the data. + builder->appendNumber("numReplicaSetMonitorsCreated", numMonitorsCreated); { BSONObjBuilder setStats( builder->subobjStart(forFTDC ? "replicaSetPingTimesMillis" : "replicaSets")); - for (const auto& setName : setNames) { - auto monitor = getMonitor(setName); - if (!monitor) { - continue; - } + for (const auto& monitor : monitors) { monitor->appendInfo(setStats, forFTDC); } } - if (_stats) { - _stats->report(builder, forFTDC); - } + _stats->report(builder, forFTDC); } std::shared_ptr<executor::TaskExecutor> ReplicaSetMonitorManager::getExecutor() { diff --git a/src/mongo/client/replica_set_monitor_manager.h b/src/mongo/client/replica_set_monitor_manager.h index 59c9b0b9853..1bb632ab85c 100644 --- a/src/mongo/client/replica_set_monitor_manager.h +++ b/src/mongo/client/replica_set_monitor_manager.h @@ -227,8 +227,9 @@ private: // Used for tests. Counter64 _monitorsGarbageCollected; - // Internally synchronized. - std::shared_ptr<ReplicaSetMonitorManagerStats> _stats; + // Pointee is internally synchronized. + const std::shared_ptr<ReplicaSetMonitorManagerStats> _stats = + std::make_shared<ReplicaSetMonitorManagerStats>(); }; } // namespace mongo diff --git a/src/mongo/client/sasl_client_session.cpp b/src/mongo/client/sasl_client_session.cpp index 4c92134a094..f1be7a64b10 100644 --- a/src/mongo/client/sasl_client_session.cpp +++ b/src/mongo/client/sasl_client_session.cpp @@ -54,7 +54,7 @@ void SaslClientSession::setParameter(Parameter id, StringData value) { // Note that we append a terminal NUL to buffer.data, so it may be treated as a C-style // string. This is required for parameterServiceName, parameterServiceHostname, // parameterMechanism and parameterUser. - value.copyTo(buffer.data.get(), true); + str::copyAsCString(buffer.data.get(), value); } bool SaslClientSession::hasParameter(Parameter id) { diff --git a/src/mongo/crypto/SConscript b/src/mongo/crypto/SConscript index d6e291f712f..82385f5238c 100644 --- a/src/mongo/crypto/SConscript +++ b/src/mongo/crypto/SConscript @@ -98,6 +98,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base/secure_allocator', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/common', '$BUILD_DIR/mongo/idl/idl_parser', ], diff --git a/src/mongo/crypto/fle_crypto.cpp b/src/mongo/crypto/fle_crypto.cpp index f55db25f970..a028efb47de 100644 --- a/src/mongo/crypto/fle_crypto.cpp +++ b/src/mongo/crypto/fle_crypto.cpp @@ -1074,7 +1074,7 @@ BSONObj toBSON(BSONType type, ConstDataRange cdr) { builder.appendNum(static_cast<uint32_t>(docLength)); builder.appendChar(static_cast<uint8_t>(type)); - builder.appendStr(valueString, true); + builder.appendCStr(valueString); builder.appendBuf(cdr.data(), cdr.length()); builder.appendChar('\0'); diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index 51535188946..c7e4b6ff146 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -125,6 +125,7 @@ env.Library( "dbmessage.cpp", ], LIBDEPS=[ + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/transport/transport_layer_common', '$BUILD_DIR/mongo/util/net/network', @@ -166,45 +167,13 @@ env.Library( ) env.Library( - target='curop', - source=[ - 'curop.cpp', - - ], - LIBDEPS=[ - '$BUILD_DIR/mongo/bson/mutable/mutable_bson', - '$BUILD_DIR/mongo/db/commands', - '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/query/command_request_response', - '$BUILD_DIR/mongo/db/service_context', - '$BUILD_DIR/mongo/db/stats/timer_stats', - '$BUILD_DIR/mongo/rpc/client_metadata', - '$BUILD_DIR/mongo/transport/service_executor', - '$BUILD_DIR/mongo/util/diagnostic_info' if get_option( - 'use-diagnostic-latches') == 'on' else [], - '$BUILD_DIR/mongo/util/fail_point', - '$BUILD_DIR/mongo/util/net/network', - '$BUILD_DIR/mongo/util/progress_meter', - 'generic_cursor', - 'profile_filter', - 'server_options', - ], - LIBDEPS_PRIVATE=[ - 'auth/auth', - 'auth/user_acquisition_stats', - 'prepare_conflict_tracker', - 'stats/resource_consumption_metrics', - ], -) - -env.Library( target='curop_failpoint_helpers', source=[ 'curop_failpoint_helpers.cpp', ], LIBDEPS=[ + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/util/fail_point', - 'curop', ], ) @@ -730,7 +699,6 @@ env.Library( ], LIBDEPS=[ 'catalog_raii', - 'curop', 's/sharding_api_d', 'stats/top', ], @@ -957,7 +925,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/client/clientdriver_minimal', - 'curop', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', 'not_primary_error_tracker', 'ops/write_ops_parsers', ], @@ -1005,7 +973,6 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/transport/service_entry_point', - 'curop', 'curop_metrics', 'rw_concern_d', 'storage/storage_engine_common', @@ -1058,6 +1025,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/catalog/catalog_helpers', '$BUILD_DIR/mongo/db/catalog/collection_catalog', '$BUILD_DIR/mongo/db/pipeline/change_stream_preimage', '$BUILD_DIR/mongo/db/repl/tenant_migration_access_blocker', @@ -1202,11 +1170,11 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog/collection_catalog', '$BUILD_DIR/mongo/db/catalog/index_build_entry_idl', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/repl/tenant_migration_access_blocker', '$BUILD_DIR/mongo/db/s/forwardable_operation_metadata', '$BUILD_DIR/mongo/db/storage/two_phase_index_build_knobs_idl', '$BUILD_DIR/mongo/executor/task_executor_interface', - 'curop', 'db_raii', 'index_build_entry_helpers', 'resumable_index_builds_idl', @@ -1304,8 +1272,8 @@ env.Library( "curop_metrics.cpp", ], LIBDEPS_PRIVATE=[ + "$BUILD_DIR/mongo/db/query/query_stats/query_stats", "commands/server_status_core", - "curop", ] ) @@ -1329,14 +1297,14 @@ env.Library( "write_concern.cpp", ], LIBDEPS=[ - "concurrency/lock_manager", - "curop", + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', + 'concurrency/lock_manager', 'read_write_concern_defaults', - "repl/read_concern_args", - "repl/repl_coordinator_interface", - "repl/speculative_majority_read_info", - "stats/timer_stats", - "storage/storage_options", + 'repl/read_concern_args', + 'repl/repl_coordinator_interface', + 'repl/speculative_majority_read_info', + 'stats/timer_stats', + 'storage/storage_options', ], LIBDEPS_PRIVATE=[ "$BUILD_DIR/mongo/db/storage/storage_control", @@ -1359,7 +1327,6 @@ env.Library( "$BUILD_DIR/mongo/idl/server_parameter", "$BUILD_DIR/mongo/s/grid", "catalog_raii", - "curop", "repl/repl_coordinator_interface", "repl/speculative_majority_read_info", "s/sharding_api_d", @@ -1438,6 +1405,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog/clustered_collection_options', @@ -1568,7 +1536,6 @@ env.Library( 'catalog/index_catalog', 'catalog/index_catalog_entry', 'commands', - 'curop', 'curop_failpoint_helpers', 'cursor_server_params', 'db_raii', @@ -1746,6 +1713,7 @@ env.Library( target='sessions_collection', source=[ 'sessions_collection.cpp', + 'sessions_server_parameters.idl' ], LIBDEPS=[ '$BUILD_DIR/mongo/base', @@ -1843,8 +1811,8 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/pipeline/lite_parsed_document_source', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', 'commands', - 'curop', 'logical_session_id', ], ) @@ -2320,8 +2288,8 @@ env.Library( 'update/update_driver', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', 'concurrency/exception_util', - 'curop', 'repl/image_collection_entry', ], ) @@ -2434,7 +2402,6 @@ env.Library( 'common', 'concurrency/flow_control_ticketholder', 'concurrency/lock_manager', - 'curop', 'curop_metrics', 'db_raii', 'dbdirectclient', @@ -2788,7 +2755,6 @@ if wiredtiger: 'collection_index_usage_tracker', 'commands', 'common', - 'curop', 'dbdirectclient', 'dbmessage', 'fcv_op_observer', diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript index 88981e3f85a..b8639a62846 100644 --- a/src/mongo/db/auth/SConscript +++ b/src/mongo/db/auth/SConscript @@ -189,9 +189,9 @@ env.Library( '$BUILD_DIR/mongo/base/secure_allocator', '$BUILD_DIR/mongo/bson/util/bson_extract', '$BUILD_DIR/mongo/db/common', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/global_settings', '$BUILD_DIR/mongo/db/namespace_string', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/idl/feature_flag', '$BUILD_DIR/mongo/util/concurrency/thread_pool', '$BUILD_DIR/mongo/util/icu', diff --git a/src/mongo/db/auth/action_type.idl b/src/mongo/db/auth/action_type.idl index 005ccbf5808..2c09bfca837 100644 --- a/src/mongo/db/auth/action_type.idl +++ b/src/mongo/db/auth/action_type.idl @@ -140,6 +140,8 @@ enums: planCacheIndexFilter : "planCacheIndexFilter" # view/update index filters planCacheRead : "planCacheRead" # view contents of plan cache planCacheWrite : "planCacheWrite" # clear cache, drop cache entry, pin/unpin/shun plans + queryStatsRead: "queryStatsRead" # view untransformed contents of queryStats store + queryStatsReadTransformed: "queryStatsReadTransformed" # view transformed contents of queryStats store refineCollectionShardKey : "refineCollectionShardKey" reIndex : "reIndex" remove : "remove" diff --git a/src/mongo/db/auth/authorization_manager_impl.cpp b/src/mongo/db/auth/authorization_manager_impl.cpp index 93450c8b1f5..f350dcdc140 100644 --- a/src/mongo/db/auth/authorization_manager_impl.cpp +++ b/src/mongo/db/auth/authorization_manager_impl.cpp @@ -389,6 +389,7 @@ AuthorizationManagerImpl::AuthorizationManagerImpl( AuthorizationManagerImpl::~AuthorizationManagerImpl() = default; std::unique_ptr<AuthorizationSession> AuthorizationManagerImpl::makeAuthorizationSession() { + invariant(_externalState != nullptr); return std::make_unique<AuthorizationSessionImpl>( _externalState->makeAuthzSessionExternalState(this), AuthorizationSessionImpl::InstallMockForTestingOrAuthImpl{}); @@ -675,35 +676,35 @@ void AuthorizationManagerImpl::_pinnedUsersThreadRoutine() noexcept try { void AuthorizationManagerImpl::invalidateUserByName(OperationContext* opCtx, const UserName& userName) { LOGV2_DEBUG(20235, 2, "Invalidating user", "user"_attr = userName); - _updateCacheGeneration(); _authSchemaVersionCache.invalidateAll(); // Invalidate the named User, assuming no externally provided roles. When roles are defined // externally, there exists no user document which may become invalid. _userCache.invalidateKey(UserRequest(userName, boost::none)); + _updateCacheGeneration(); } void AuthorizationManagerImpl::invalidateUsersFromDB(OperationContext* opCtx, StringData dbname) { LOGV2_DEBUG(20236, 2, "Invalidating all users from database", "database"_attr = dbname); - _updateCacheGeneration(); _authSchemaVersionCache.invalidateAll(); _userCache.invalidateKeyIf( [&](const UserRequest& userRequest) { return userRequest.name.getDB() == dbname; }); + _updateCacheGeneration(); } void AuthorizationManagerImpl::invalidateUsersByTenant(OperationContext* opCtx, const TenantId& tenant) { LOGV2_DEBUG(6323600, 2, "Invalidating tenant users", "tenant"_attr = tenant); - _updateCacheGeneration(); _authSchemaVersionCache.invalidateAll(); _userCache.invalidateKeyIf( [&](const UserRequest& userRequest) { return userRequest.name.getTenant() == tenant; }); + _updateCacheGeneration(); } void AuthorizationManagerImpl::invalidateUserCache(OperationContext* opCtx) { LOGV2_DEBUG(20237, 2, "Invalidating user cache"); - _updateCacheGeneration(); _authSchemaVersionCache.invalidateAll(); _userCache.invalidateAll(); + _updateCacheGeneration(); } Status AuthorizationManagerImpl::refreshExternalUsers(OperationContext* opCtx) { diff --git a/src/mongo/db/auth/authorization_session_test_fixture.cpp b/src/mongo/db/auth/authorization_session_test_fixture.cpp index 330837ee719..db2500a6a2c 100644 --- a/src/mongo/db/auth/authorization_session_test_fixture.cpp +++ b/src/mongo/db/auth/authorization_session_test_fixture.cpp @@ -44,16 +44,18 @@ namespace mongo { void AuthorizationSessionTestFixture::setUp() { + // AuthorizationManager must be initialized prior to creating Client objects. + auto localManagerState = std::make_unique<FailureCapableAuthzManagerExternalStateMock>(); + managerState = localManagerState.get(); + auto uniqueAuthzManager = std::make_unique<AuthorizationManagerImpl>( + getServiceContext(), std::move(localManagerState)); + _session = transportLayer.createSession(); _client = getServiceContext()->makeClient("testClient", _session); RestrictionEnvironment::set(_session, std::make_unique<RestrictionEnvironment>(SockAddr(), SockAddr())); _opCtx = _client->makeOperationContext(); - auto localManagerState = std::make_unique<FailureCapableAuthzManagerExternalStateMock>(); - managerState = localManagerState.get(); managerState->setAuthzVersion(AuthorizationManager::schemaVersion26Final); - auto uniqueAuthzManager = std::make_unique<AuthorizationManagerImpl>( - getServiceContext(), std::move(localManagerState)); authzManager = uniqueAuthzManager.get(); AuthorizationManager::set(getServiceContext(), std::move(uniqueAuthzManager)); auto localSessionState = std::make_unique<AuthzSessionExternalStateMock>(authzManager); diff --git a/src/mongo/db/auth/builtin_roles.cpp b/src/mongo/db/auth/builtin_roles.cpp index 90b097800d0..9eddd117f80 100644 --- a/src/mongo/db/auth/builtin_roles.cpp +++ b/src/mongo/db/auth/builtin_roles.cpp @@ -205,7 +205,9 @@ MONGO_INITIALIZER(AuthorizationBuiltinRoles)(InitializerContext* context) { << ActionType::inprog << ActionType::shardingState << ActionType::allCollectionStats - << ActionType::shardedDataDistribution; + << ActionType::shardedDataDistribution + << ActionType::queryStatsRead + << ActionType::queryStatsReadTransformed; // clusterMonitor role actions that target a database (or collection) resource clusterMonitorRoleDatabaseActions diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index 0321036103f..e20adc6cf10 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -38,6 +38,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/pipeline/change_stream_pre_and_post_images_options', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', @@ -134,8 +135,8 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog_raii', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/index/index_access_method', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/storage/key_string', 'validate_state', ] @@ -150,8 +151,8 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/catalog_raii', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/query/query_knobs', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/storage/record_store_base', '$BUILD_DIR/mongo/db/storage/storage_repair_observer', 'index_repair', @@ -264,8 +265,8 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/catalog_raii', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/index/index_access_method', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/resumable_index_builds_idl', '$BUILD_DIR/mongo/db/service_context', @@ -369,14 +370,15 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/collection_index_usage_tracker', '$BUILD_DIR/mongo/db/commands/server_status_core', '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/db_raii', '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/op_observer', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/drop_pending_collection_reaper', '$BUILD_DIR/mongo/db/repl/oplog', @@ -413,6 +415,7 @@ env.Library( 'index_key_validate', 'index_repair', 'local_oplog_info', + 'storage_engine_collection_options_flags_parser', 'throttle_cursor', 'validate_idl', 'validate_state', @@ -442,17 +445,20 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/bson/util/bson_column', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/db_raii', '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/index_names', '$BUILD_DIR/mongo/db/multi_key_path_tracker', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', "$BUILD_DIR/mongo/db/service_context", '$BUILD_DIR/mongo/db/storage/execution_context', '$BUILD_DIR/mongo/db/storage/key_string', + '$BUILD_DIR/mongo/db/timeseries/bucket_catalog', + '$BUILD_DIR/mongo/db/timeseries/timeseries_options', '$BUILD_DIR/mongo/idl/basic_types', 'catalog_impl', 'collection_options', @@ -469,8 +475,8 @@ env.Library( 'throttle_cursor.cpp', ], LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/index/index_access_method', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/util/fail_point', 'validate_idl', ], @@ -482,8 +488,8 @@ env.Library( 'local_oplog_info.cpp', ], LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/logical_time', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/repl/optime', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/storage/flow_control', @@ -496,6 +502,7 @@ env.Library( env.Library( target='catalog_helpers', source=[ + 'backwards_compatible_collection_options_util.cpp', 'capped_utils.cpp', 'collection_catalog_helper.cpp', 'coll_mod.cpp', @@ -557,8 +564,8 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/collection_index_usage_tracker', '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/fts/base_fts', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/s/sharding_api_d', '$BUILD_DIR/mongo/db/service_context', 'index_catalog', @@ -613,7 +620,21 @@ env.Library( ) if wiredtiger: - env.CppUnitTest( + wtEnv = env.Clone() + wtEnv.InjectThirdParty(libraries=["wiredtiger"]) + + wtEnv.Library( + target="storage_engine_collection_options_flags_parser", + source=[ + "storage_engine_collection_options_flags_parser.cpp", + ], + LIBDEPS_PRIVATE=[ + "$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger", + "$BUILD_DIR/third_party/shim_pcrecpp", + ], + ) + + wtEnv.CppUnitTest( target='db_catalog_test', source=[ 'capped_utils_test.cpp', @@ -635,14 +656,17 @@ if wiredtiger: 'index_spec_validate_test.cpp', 'multi_index_block_test.cpp', 'rename_collection_test.cpp', + 'storage_engine_collection_options_flags_parser_test.cpp', 'throttle_cursor_test.cpp', 'validate_state_test.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/db/auth/authmocks', + '$BUILD_DIR/mongo/db/commands/create_command', '$BUILD_DIR/mongo/db/commands/test_commands_enabled', '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/db_raii', + '$BUILD_DIR/mongo/db/dbhelpers', '$BUILD_DIR/mongo/db/index_builds_coordinator_mongod', '$BUILD_DIR/mongo/db/matcher/expressions', '$BUILD_DIR/mongo/db/multitenancy', @@ -653,6 +677,7 @@ if wiredtiger: '$BUILD_DIR/mongo/db/query/query_test_service_context', '$BUILD_DIR/mongo/db/repl/drop_pending_collection_reaper', '$BUILD_DIR/mongo/db/repl/oplog', + '$BUILD_DIR/mongo/db/repl/oplog_application', '$BUILD_DIR/mongo/db/repl/optime', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/replmocks', @@ -661,6 +686,7 @@ if wiredtiger: '$BUILD_DIR/mongo/db/service_context_d_test_fixture', '$BUILD_DIR/mongo/db/service_context_test_fixture', '$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger', + '$BUILD_DIR/mongo/db/timeseries/timeseries_collmod', '$BUILD_DIR/mongo/db/timeseries/timeseries_options', '$BUILD_DIR/mongo/unittest/unittest', '$BUILD_DIR/mongo/util/clock_source_mock', @@ -679,6 +705,7 @@ if wiredtiger: 'index_builds_manager', 'index_key_validate', 'multi_index_block', + 'storage_engine_collection_options_flags_parser', 'throttle_cursor', 'validate_idl', 'validate_state', diff --git a/src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp b/src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp new file mode 100644 index 00000000000..d966d6108c6 --- /dev/null +++ b/src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +// TODO SERVER-92265 evaluate getting rid of this util + +#include "mongo/db/catalog/backwards_compatible_collection_options_util.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/simple_bsonobj_comparator.h" +#include "mongo/db/repl/oplog_entry.h" + +namespace mongo { +namespace backwards_compatible_collection_options { +std::pair<BSONObj, BSONObj> getCollModCmdAndAdditionalO2Field(const BSONObj& collModCmd) { + const BSONObj collModCmdStrippedBackwardsIncompatibleParams = + collModCmd.removeFields(kBackwardsCompatibleCollectionOptions); + if (SimpleBSONObjComparator::kInstance.evaluate(collModCmdStrippedBackwardsIncompatibleParams == + collModCmd)) { + return {collModCmd, BSONObj()}; + } + + const BSONObj backwardsIncompatibleFields = [&]() { + BSONObjBuilder bob; + for (auto [fieldName, elem] : collModCmd) { + if (kBackwardsCompatibleCollectionOptions.count(fieldName.toString())) { + bob.append(elem); + } + } + return bob.obj(); + }(); + + return {collModCmdStrippedBackwardsIncompatibleParams, backwardsIncompatibleFields}; +} + +BSONObj parseCollModCmdFromOplogEntry(const repl::OplogEntry& entry) { + uassert(ErrorCodes::IllegalOperation, + str::stream() << "Can't extract `collMod` command from non-collMod oplog entry: " + << entry.toBSONForLogging(), + entry.getCommandType() == repl::OplogEntry::CommandType::kCollMod); + + if (!entry.getObject2()) { + return entry.getObject(); + } + + BSONObj incompatibleFields = entry.getObject2()->getObjectField(additionalCollModO2Field); + if (incompatibleFields.isEmpty()) { + return entry.getObject(); + } + + // Only consider backwards incompatible fields supported in the current [sub-]version + for (auto [fieldName, elem] : incompatibleFields) { + if (!kBackwardsCompatibleCollectionOptions.count(fieldName.toString())) { + incompatibleFields = incompatibleFields.removeField(fieldName); + } + } + + return entry.getObject().addFields(incompatibleFields); +} + +} // namespace backwards_compatible_collection_options +} // namespace mongo diff --git a/src/mongo/db/catalog/backwards_compatible_collection_options_util.h b/src/mongo/db/catalog/backwards_compatible_collection_options_util.h new file mode 100644 index 00000000000..d5c47b45e6d --- /dev/null +++ b/src/mongo/db/catalog/backwards_compatible_collection_options_util.h @@ -0,0 +1,130 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +// TODO SERVER-92265 evaluate getting rid of this file + +#include "mongo/db/repl/oplog_entry.h" + +namespace mongo { + +/** + * + * This utility is providing primitives to manage catalog parameters for which the actual value may + * have been missing or incorrect in previous mongod [sub-]versions. It is only meant to fix catalog + * issues in existing versions and must NOT be used for implementing new features. + * + * The functions under this namespace are offering an abstraction to work around the following + * limitations: + * - collMod command is strict (so can't simply add a parameter to previous mongod [sub-]versions) + * - Collection options are strict (so can't simply add an option to previous mongod [sub-]versions) + * + */ +namespace backwards_compatible_collection_options { + +const std::string kTimeseriesBucketsMayHaveMixedSchemaData = + "timeseriesBucketsMayHaveMixedSchemaData"; + +const std::string additionalCollModO2Field = "backwardsIncompatibleCollModParameters"; + +/** + * Backwards incompatible catalog parameters for which the actual value may have been missing or + * incorrect in previous mongod [sub-]versions. + */ +const std::set<std::string> kBackwardsCompatibleCollectionOptions{ + kTimeseriesBucketsMayHaveMixedSchemaData}; + +/** + * Strips backwards incompatible fields from a collMod command and places them into a + * different BSON object. + * + * Returns two BSON objects: + * - A backwards compatible collMod oplog entry (not to generate crashes when applied by + * incompatible mongod [sub-]versions). + * - A field meant to be added to the `o2` sub-object (parsable by new mongod [sub-]versions). + * + * Example: + * + * - Original command: + * {"collMod":"testdb.system.buckets.testcoll", "timeseriesBucketsMayHaveMixedSchemaData":true } + * + * - Expected oplog entry with `timeseriesBucketsMayHaveMixedSchemaData` backwards incompatible + * collMod parameter. + * + * { + * "oplogEntry":{ + * "op":"c", + * "ns":"testdb.$cmd", + * "ui":"UUID(""7302d025-cb9c-4a16-9222-0d5aeefbc039"")", + * "o":{ + * "collMod":"system.buckets.testcoll" + * }, + * "o2":{ + * "collectionOptions_old":{ + * "uuid": UUID("7302d025-cb9c-4a16-9222-0d5aeefbc039"), + * "Validator":{ "...REDACTED..." }, + * "clusteredIndex":true, + * "timeseries":{ + * "timeField":"t", + * “granularity":"seconds", + * "bucketMaxSpanSeconds":3600 + * } + * }, + * "backwardsIncompatibleCollModParameters":{ + * "timeseriesBucketsMayHaveMixedSchemaData":true + * } + * }, + * "ts":Timestamp(1720003401,4), + * "t":1, + * "v":2, + * "wall":new Date(1720003401165) + * } + * } + * + */ +std::pair<BSONObj, BSONObj> getCollModCmdAndAdditionalO2Field(const BSONObj& collModCmd); + +/** + * Rebuilds a collMod command from an oplog entry. + * + * Returns a bson object: + * - A collMod command inclusive of potential backwards incompatible fields present in the oplog + * entry's `o2` sub-object. + * + * Example: collMod command parsed from the sample oplog entry documented above. + * + * { + * "collMod":"system.buckets.testcoll", + * "timeseriesBucketsMayHaveMixedSchemaData":true + * } + * + */ +BSONObj parseCollModCmdFromOplogEntry(const repl::OplogEntry& entry); + +} // namespace backwards_compatible_collection_options +} // namespace mongo diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index 3bb76fee38b..2c9f03053d0 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -118,6 +118,7 @@ struct ParsedCollModRequest { bool dryRun = false; boost::optional<long long> cappedSize; boost::optional<long long> cappedMax; + boost::optional<bool> timeseriesBucketsMayHaveMixedSchemaData; }; Status getNotSupportedOnViewError(StringData fieldName) { @@ -400,6 +401,11 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati } if (cmdIndex.getPrepareUnique()) { + // Check if prepareUnique is being set on a time-series collection. + if (isTimeseries) { + return {ErrorCodes::InvalidOptions, + "cannot set 'prepareUnique' for indexes of a time-series collection."}; + } parsed.numModifications++; // Attempting to modify with the same value should be treated as a no-op. if (cmrIndex->idx->prepareUnique() == *cmdIndex.getPrepareUnique() || @@ -605,6 +611,17 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati timeseries->serialize(&subObjBuilder); } + if (auto mixedSchema = cmr.getTimeseriesBucketsMayHaveMixedSchemaData()) { + if (!isTimeseries) { + return getOnlySupportedOnTimeseriesError( + CollMod::kTimeseriesBucketsMayHaveMixedSchemaDataFieldName); + } + + parsed.timeseriesBucketsMayHaveMixedSchemaData = mixedSchema; + oplogEntryBuilder.append(CollMod::kTimeseriesBucketsMayHaveMixedSchemaDataFieldName, + *mixedSchema); + } + if (auto& dryRun = cmr.getDryRun()) { parsed.dryRun = *dryRun; // The dry run option should never be included in a collMod oplog entry. @@ -930,6 +947,11 @@ Status _collModInternal(OperationContext* opCtx, *cmd.getExpireAfterSeconds()); } + if (auto mixedSchema = cmrNew.timeseriesBucketsMayHaveMixedSchemaData) { + coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData( + opCtx, mixedSchema); + } + // Handle index modifications. processCollModIndexRequest( opCtx, &coll, cmrNew.indexRequest, &indexCollModInfo, result, mode); @@ -985,24 +1007,24 @@ Status _collModInternal(OperationContext* opCtx, // (Generic FCV reference): TODO SERVER-60912: When kLastLTS is 6.0, remove this FCV-gated // upgrade/downgrade code. const auto currentVersion = serverGlobalParams.featureCompatibility.getVersion(); - if (coll->getTimeseriesOptions() && !coll->getTimeseriesBucketsMayHaveMixedSchemaData() && - (currentVersion == multiversion::GenericFCV::kUpgradingFromLastLTSToLatest || - currentVersion == multiversion::GenericFCV::kLatest)) { - // (Generic FCV reference): While upgrading the FCV from kLastLTS to kLatest, collMod is - // called as part of the upgrade process to add the - // 'timeseriesBucketsMayHaveMixedSchemaData=true' catalog entry flag for time-series - // collections that are missing the flag. This indicates that the time-series collection - // existed in earlier server versions and may have mixed-schema data. - coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx, - true); - } else if (coll->getTimeseriesBucketsMayHaveMixedSchemaData() && - (currentVersion == multiversion::GenericFCV::kDowngradingFromLatestToLastLTS || - currentVersion == multiversion::GenericFCV::kLastLTS)) { - // (Generic FCV reference): While downgrading the FCV to kLastLTS, collMod is called as - // part of the downgrade process to remove the 'timeseriesBucketsMayHaveMixedSchemaData' - // catalog entry flag for time-series collections that have the flag. - coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData( - opCtx, boost::none); + if (coll->getTimeseriesOptions()) { + if (currentVersion == multiversion::GenericFCV::kUpgradingFromLastLTSToLatest) { + // (Generic FCV reference): While upgrading the FCV from kLastLTS to kLatest, + // collMod is called as part of the upgrade process to add the + // 'timeseriesBucketsMayHaveMixedSchemaData=true' catalog entry flag for time-series + // collections that are missing the flag. This indicates that the time-series + // collection existed in earlier server versions and may have mixed-schema data. + coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx, + true); + } else if (currentVersion == + multiversion::GenericFCV::kDowngradingFromLatestToLastLTS) { + // (Generic FCV reference): While downgrading the FCV to kLastLTS, collMod is called + // as part of the downgrade process to remove the + // 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag for time-series + // collections that have the flag. + coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData( + opCtx, boost::none); + } } // Only observe non-view collMods, as view operations are observed as operations on the diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h index f64af95c32a..9be52a11a11 100644 --- a/src/mongo/db/catalog/collection.h +++ b/src/mongo/db/catalog/collection.h @@ -560,9 +560,10 @@ public: boost::optional<bool> setting) = 0; /** - * Returns true if the passed in time-series bucket document contains mixed-schema data. + * Returns true if the passed in time-series bucket document contains mixed-schema data. Returns + * a non-OK status if the bucket's min/max is malformed. */ - virtual bool doesTimeseriesBucketsDocContainMixedSchemaData( + virtual StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData( const BSONObj& bucketsDoc) const = 0; /** diff --git a/src/mongo/db/catalog/collection_catalog_helper.cpp b/src/mongo/db/catalog/collection_catalog_helper.cpp index 1bdd85ce3e6..ed816f2e1cf 100644 --- a/src/mongo/db/catalog/collection_catalog_helper.cpp +++ b/src/mongo/db/catalog/collection_catalog_helper.cpp @@ -66,6 +66,7 @@ void forEachCollectionFromDb(OperationContext* opCtx, CollectionCatalog::CollectionInfoFn predicate) { auto catalogForIteration = CollectionCatalog::get(opCtx); + size_t collectionCount = 0; for (auto&& coll : catalogForIteration->range(tenantDbName)) { auto uuid = coll->uuid(); if (predicate && !catalogForIteration->checkIfCollectionSatisfiable(uuid, predicate)) { @@ -99,7 +100,14 @@ void forEachCollectionFromDb(OperationContext* opCtx, if (!callback(collection)) break; + // This was a rough heuristic that was found that 400 collections would take 100 + // milliseconds with calling checkForInterrupt() (with freeStorage: 1). + // We made the checkForInterrupt() occur after 200 collections to be conservative. + if (!(collectionCount % 200)) { + opCtx->checkForInterrupt(); + } hangBeforeGettingNextCollection.pauseWhileSet(); + collectionCount += 1; } } diff --git a/src/mongo/db/catalog/collection_compact.cpp b/src/mongo/db/catalog/collection_compact.cpp index b6cc7cb444d..549aef5d9f1 100644 --- a/src/mongo/db/catalog/collection_compact.cpp +++ b/src/mongo/db/catalog/collection_compact.cpp @@ -47,64 +47,18 @@ namespace mongo { using logv2::LogComponent; -namespace { - -CollectionPtr getCollectionForCompact(OperationContext* opCtx, - const NamespaceString& collectionNss) { - invariant(opCtx->lockState()->isCollectionLockedForMode(collectionNss, MODE_IX)); - - auto collectionCatalog = CollectionCatalog::get(opCtx); - CollectionPtr collection = collectionCatalog->lookupCollectionByNamespace(opCtx, collectionNss); - - if (!collection) { - std::shared_ptr<const ViewDefinition> view = - collectionCatalog->lookupView(opCtx, collectionNss); - uassert(ErrorCodes::CommandNotSupportedOnView, "can't compact a view", !view); - uasserted(ErrorCodes::NamespaceNotFound, "collection does not exist"); - } - - return collection; -} - -} // namespace - -StatusWith<int64_t> compactCollection(OperationContext* opCtx, - const NamespaceString& collectionNss) { - AutoGetDb autoDb(opCtx, collectionNss.db(), MODE_IX); - Database* database = autoDb.getDb(); - uassert(ErrorCodes::NamespaceNotFound, "database does not exist", database); - - // The collection lock will be upgraded to an exclusive lock if the record store does not - // support online compaction. - boost::optional<Lock::CollectionLock> collLk; - collLk.emplace(opCtx, collectionNss, MODE_IX); - - CollectionPtr collection = getCollectionForCompact(opCtx, collectionNss); +StatusWith<int64_t> compactCollection(OperationContext* opCtx, const CollectionPtr& collection) { DisableDocumentValidation validationDisabler(opCtx); + auto collectionNss = collection->ns(); auto recordStore = collection->getRecordStore(); - OldClientContext ctx(opCtx, collectionNss.ns()); - if (!recordStore->compactSupported()) return Status(ErrorCodes::CommandNotSupported, str::stream() << "cannot compact collection with record store: " << recordStore->name()); - if (!recordStore->supportsOnlineCompaction()) { - // Storage engines that disallow online compaction should compact under an exclusive lock. - collLk.emplace(opCtx, collectionNss, MODE_X); - - // Ensure the collection was not dropped during the re-lock. - collection = getCollectionForCompact(opCtx, collectionNss); - recordStore = collection->getRecordStore(); - } - - LOGV2_OPTIONS(20284, - {LogComponent::kCommand}, - "compact {namespace} begin", - "Compact begin", - "namespace"_attr = collectionNss); + LOGV2_OPTIONS(20284, {LogComponent::kCommand}, "Compact begin", logAttrs(collectionNss)); auto oldTotalSize = recordStore->storageSize(opCtx) + collection->getIndexSize(opCtx); auto indexCatalog = collection->getIndexCatalog(); diff --git a/src/mongo/db/catalog/collection_compact.h b/src/mongo/db/catalog/collection_compact.h index ab22cf07155..c6307360bc0 100644 --- a/src/mongo/db/catalog/collection_compact.h +++ b/src/mongo/db/catalog/collection_compact.h @@ -40,7 +40,6 @@ namespace mongo { * Returns the number of bytes of stable storage and index size that were freed. If the total * size decreased, the return value is positive. Otherwise, the return value is negative. */ -StatusWith<int64_t> compactCollection(OperationContext* opCtx, - const NamespaceString& collectionNss); +StatusWith<int64_t> compactCollection(OperationContext* opCtx, const CollectionPtr& collection); } // namespace mongo diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index 298772562db..cdf3730fa1e 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -40,6 +40,7 @@ #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/crypto/fle_crypto.h" #include "mongo/db/auth/security_token.h" +#include "mongo/db/catalog/backwards_compatible_collection_options_util.h" #include "mongo/db/catalog/catalog_stats.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_options.h" @@ -48,6 +49,7 @@ #include "mongo/db/catalog/index_consistency.h" #include "mongo/db/catalog/index_key_validate.h" #include "mongo/db/catalog/local_oplog_info.h" +#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h" #include "mongo/db/catalog/uncommitted_multikey.h" #include "mongo/db/clientcursor.h" #include "mongo/db/commands/server_status_metric.h" @@ -352,35 +354,42 @@ bool indexTypeSupportsPathLevelMultikeyTracking(StringData accessMethod) { return accessMethod == IndexNames::BTREE || accessMethod == IndexNames::GEO_2DSPHERE; } -bool doesMinMaxHaveMixedSchemaData(const BSONObj& min, const BSONObj& max) { +StatusWith<bool> doesMinMaxHaveMixedSchemaData(const BSONObj& min, const BSONObj& max) { auto minIt = min.begin(); auto minEnd = min.end(); auto maxIt = max.begin(); auto maxEnd = max.end(); while (minIt != minEnd && maxIt != maxEnd) { - bool typeMatch = minIt->canonicalType() == maxIt->canonicalType(); - if (!typeMatch) { + // The 'control.min' and 'control.max' fields have the same ordering. + if (minIt->fieldNameStringData() != maxIt->fieldNameStringData()) { + return Status{ + ErrorCodes::BadValue, + "Encountered inconsistent field name ordering in time-series bucket min/max"}; + } + + if (minIt->canonicalType() != maxIt->canonicalType()) { return true; } else if (minIt->type() == Object) { - // The 'control.min' and 'control.max' fields have the same ordering. - invariant(minIt->fieldNameStringData() == maxIt->fieldNameStringData()); - if (doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj())) { - return true; + auto result = doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj()); + if (!result.isOK() || result.getValue()) { + return result; } } else if (minIt->type() == Array) { - if (doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj())) { - return true; + auto result = doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj()); + if (!result.isOK() || result.getValue()) { + return result; } } - invariant(typeMatch); minIt++; maxIt++; } - // The 'control.min' and 'control.max' fields have the same cardinality. - invariant(minIt == minEnd && maxIt == maxEnd); + if (minIt != minEnd || maxIt != maxEnd) { + return Status{ErrorCodes::BadValue, + "Encountered extra field(s) in time-series bucket min/max"}; + } return false; } @@ -1589,6 +1598,19 @@ bool CollectionImpl::isTemporary() const { } boost::optional<bool> CollectionImpl::getTimeseriesBucketsMayHaveMixedSchemaData() const { + if (!getTimeseriesOptions()) { + return boost::none; + } + + // If present, reuse storageEngine options to work around the issue described in SERVER-91194 + boost::optional<bool> optBackwardsCompatibleFlag = getFlagFromStorageEngineBson( + _metadata->options.storageEngine, + backwards_compatible_collection_options::kTimeseriesBucketsMayHaveMixedSchemaData); + if (optBackwardsCompatibleFlag) { + return *optBackwardsCompatibleFlag; + } + + // Else, fallback to legacy parameter return _metadata->timeseriesBucketsMayHaveMixedSchemaData; } @@ -1604,11 +1626,21 @@ void CollectionImpl::setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext "setting"_attr = setting); _writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) { + // Reuse storageEngine options to work around the issue described in SERVER-91194 + if (setting.has_value()) { + md.options.storageEngine = setFlagToStorageEngineBson( + md.options.storageEngine, + backwards_compatible_collection_options::kTimeseriesBucketsMayHaveMixedSchemaData, + *setting); + } + + // Also update legacy parameter for compatibility when downgrading to older sub-versions + // only relying on this option (best-effort because it may be lost due to SERVER-91194) md.timeseriesBucketsMayHaveMixedSchemaData = setting; }); } -bool CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData( +StatusWith<bool> CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData( const BSONObj& bucketsDoc) const { if (!getTimeseriesOptions()) { return false; diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h index a8e7dd99863..58662679df1 100644 --- a/src/mongo/db/catalog/collection_impl.h +++ b/src/mongo/db/catalog/collection_impl.h @@ -329,7 +329,8 @@ public: void setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext* opCtx, boost::optional<bool> setting) final; - bool doesTimeseriesBucketsDocContainMixedSchemaData(const BSONObj& bucketsDoc) const final; + StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData( + const BSONObj& bucketsDoc) const final; bool getRequiresTimeseriesExtendedRangeSupport() const final; void setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const final; diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h index a9b7fe8c27b..820ed47cdf4 100644 --- a/src/mongo/db/catalog/collection_mock.h +++ b/src/mongo/db/catalog/collection_mock.h @@ -253,7 +253,8 @@ public: std::abort(); } - bool doesTimeseriesBucketsDocContainMixedSchemaData(const BSONObj& bucketsDoc) const { + StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData( + const BSONObj& bucketsDoc) const { std::abort(); } diff --git a/src/mongo/db/catalog/collection_test.cpp b/src/mongo/db/catalog/collection_test.cpp index fd73c68bad8..00c131756ac 100644 --- a/src/mongo/db/catalog/collection_test.cpp +++ b/src/mongo/db/catalog/collection_test.cpp @@ -403,7 +403,9 @@ TEST_F(CollectionTest, CheckTimeseriesBucketDocsForMixedSchemaData) { "max" : { "x" : [ 2, 3 ] } } })")}; for (const auto& controlDoc : mixedSchemaControlDocs) { - ASSERT_TRUE(coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc)); + auto mixedSchema = coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc); + ASSERT_OK(mixedSchema) << controlDoc; + ASSERT_TRUE(mixedSchema.getValue()) << controlDoc; } std::vector<BSONObj> nonMixedSchemaControlDocs = { @@ -460,7 +462,27 @@ TEST_F(CollectionTest, CheckTimeseriesBucketDocsForMixedSchemaData) { for (const auto& controlDoc : nonMixedSchemaControlDocs) { - ASSERT_FALSE(coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc)); + auto mixedSchema = coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc); + ASSERT_OK(mixedSchema) << controlDoc; + ASSERT_FALSE(mixedSchema.getValue()) << controlDoc; + } + + std::vector<BSONObj> malformedControlDocs = { + // Inconsistent field name ordering + ::mongo::fromjson(R"({ "control" : { "min" : { "x" : 1, "y" : 1 }, + "max" : { "y" : 2, "x" : 2 } } })"), + + // Extra field in min + ::mongo::fromjson(R"({ "control" : { "min" : { "x" : 1, "y" : 1 }, + "max" : { "x" : 2 } } })"), + + // Extra field in max + ::mongo::fromjson(R"({ "control" : { "min" : { "y" : 1 }, + "max" : { "y" : 2, "x" : 2 } } })")}; + + for (const auto& controlDoc : malformedControlDocs) { + ASSERT_NOT_OK(coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc)) + << controlDoc; } } diff --git a/src/mongo/db/catalog/collection_validation.cpp b/src/mongo/db/catalog/collection_validation.cpp index 015bf7e4b5a..9d0c487b41a 100644 --- a/src/mongo/db/catalog/collection_validation.cpp +++ b/src/mongo/db/catalog/collection_validation.cpp @@ -162,7 +162,8 @@ void _gatherIndexEntryErrors(OperationContext* opCtx, ValidateResults tempValidateResults; BSONObjBuilder tempBuilder; - indexValidator->traverseRecordStore(opCtx, &tempValidateResults, &tempBuilder); + indexValidator->traverseRecordStore( + opCtx, &tempValidateResults, &tempBuilder, validateState->validationVersion()); } LOGV2_OPTIONS( @@ -454,8 +455,7 @@ void _validateCatalogEntry(OperationContext* opCtx, index_key_validate::validateIndexSpec(opCtx, indexEntry->descriptor()->infoObj()) .getStatus(); if (!status.isOK()) { - results->valid = false; - results->errors.push_back( + results->warnings.push_back( fmt::format("The index specification for index '{}' contains invalid fields. {}. " "Run the 'collMod' command on the collection without any arguments " "to fix the invalid index options", @@ -621,6 +621,14 @@ Status validate(OperationContext* opCtx, opCtx->recoveryUnit()->abandonSnapshot(); opCtx->recoveryUnit()->setPrepareConflictBehavior(oldPrepareConflictBehavior); }); + + // Relax corruption detection so that we log and continue scanning instead of failing early. + auto oldDataCorruptionMode = opCtx->recoveryUnit()->getDataCorruptionDetectionMode(); + opCtx->recoveryUnit()->setDataCorruptionDetectionMode( + DataCorruptionDetectionMode::kLogAndContinue); + ON_BLOCK_EXIT( + [&] { opCtx->recoveryUnit()->setDataCorruptionDetectionMode(oldDataCorruptionMode); }); + if (validateState.fixErrors()) { // Note: cannot set PrepareConflictBehavior here, since the validate command with repair // needs kIngnoreConflictsAllowWrites, but validate repair at startup cannot set that here @@ -697,7 +705,8 @@ Status validate(OperationContext* opCtx, // the collection. For clustered collections, the validator also verifies that the // record key (RecordId) matches the cluster key field in the record value (document's // cluster key). - indexValidator.traverseRecordStore(opCtx, results, output); + indexValidator.traverseRecordStore( + opCtx, results, output, additionalOptions.validationVersion); // Pause collection validation while a lock is held and between collection and index data // validation. @@ -771,8 +780,7 @@ Status validate(OperationContext* opCtx, return e.toStatus(); } string err = str::stream() << "exception during collection validation: " << e.toString(); - results->errors.push_back(err); - results->valid = false; + results->warnings.push_back(err); LOGV2_OPTIONS(5160302, {LogComponent::kIndex}, "Validation failed due to exception", diff --git a/src/mongo/db/catalog/collection_validation.h b/src/mongo/db/catalog/collection_validation.h index 2a0ea7a6929..f68258fee14 100644 --- a/src/mongo/db/catalog/collection_validation.h +++ b/src/mongo/db/catalog/collection_validation.h @@ -29,6 +29,9 @@ #pragma once +#include "mongo/base/status.h" +#include "mongo/bson/bson_validate.h" +#include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/catalog/validate_results.h" #include "mongo/db/namespace_string.h" @@ -90,7 +93,7 @@ enum class RepairMode { * Additional validation options that can run in any mode. */ struct AdditionalOptions { - bool warnOnSchemaValidation = false; // only warn on schema validation failure + ValidationVersion validationVersion = currentValidationVersion; }; /** diff --git a/src/mongo/db/catalog/database_test.cpp b/src/mongo/db/catalog/database_test.cpp index 59308be7233..c4669cd85b2 100644 --- a/src/mongo/db/catalog/database_test.cpp +++ b/src/mongo/db/catalog/database_test.cpp @@ -261,7 +261,7 @@ void _testDropCollectionThrowsExceptionIfThereAreIndexesInProgress(OperationCont collection->ns(), indexInfoObj, IndexBuildMethod::kHybrid, UUID::gen()); { WriteUnitOfWork wuow(opCtx); - ASSERT_OK(indexBuildBlock->init(opCtx, collection)); + ASSERT_OK(indexBuildBlock->init(opCtx, collection, /*forRecovery=*/false)); wuow.commit(); } ON_BLOCK_EXIT([&indexBuildBlock, opCtx, collection] { diff --git a/src/mongo/db/catalog/index_build_block.cpp b/src/mongo/db/catalog/index_build_block.cpp index a5645e9de72..34dbf0c49ad 100644 --- a/src/mongo/db/catalog/index_build_block.cpp +++ b/src/mongo/db/catalog/index_build_block.cpp @@ -75,6 +75,11 @@ void IndexBuildBlock::_completeInit(OperationContext* opCtx, Collection* collect .registerIndex(desc->indexName(), desc->keyPattern(), IndexFeatures::make(desc, collection->ns().isOnInternalDb())); + opCtx->recoveryUnit()->onRollback( + [collectionDecorations = collection->getSharedDecorations(), indexName = _indexName] { + CollectionIndexUsageTrackerDecoration::get(collectionDecorations) + .unregisterIndex(indexName); + }); } Status IndexBuildBlock::initForResume(OperationContext* opCtx, @@ -122,7 +127,7 @@ Status IndexBuildBlock::initForResume(OperationContext* opCtx, return Status::OK(); } -Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection) { +Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection, bool forRecovery) { // Being in a WUOW means all timestamping responsibility can be pushed up to the caller. invariant(opCtx->lockState()->inAWriteUnitOfWork()); @@ -150,14 +155,25 @@ Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection) { !replCoord->getMemberState().primary() && isBackgroundIndex; } - // Setup on-disk structures. - Status status = collection->prepareForIndexBuild( - opCtx, descriptor.get(), _buildUUID, isBackgroundSecondaryBuild); - if (!status.isOK()) - return status; + if (!forRecovery) { + // Setup on-disk structures. We skip this during startup recovery for unfinished indexes as + // everything is already in-place. + Status status = collection->prepareForIndexBuild( + opCtx, descriptor.get(), _buildUUID, isBackgroundSecondaryBuild); + if (!status.isOK()) + return status; + } - auto indexCatalogEntry = collection->getIndexCatalog()->createIndexEntry( - opCtx, collection, std::move(descriptor), CreateIndexEntryFlags::kNone); + auto indexCatalog = collection->getIndexCatalog(); + IndexCatalogEntry* indexCatalogEntry = nullptr; + if (forRecovery) { + auto desc = indexCatalog->findIndexByName( + opCtx, _indexName, IndexCatalog::InclusionPolicy::kUnfinished); + indexCatalogEntry = desc->getEntry(); + } else { + indexCatalogEntry = indexCatalog->createIndexEntry( + opCtx, collection, std::move(descriptor), CreateIndexEntryFlags::kNone); + } if (_method == IndexBuildMethod::kHybrid) { _indexBuildInterceptor = std::make_unique<IndexBuildInterceptor>(opCtx, indexCatalogEntry); diff --git a/src/mongo/db/catalog/index_build_block.h b/src/mongo/db/catalog/index_build_block.h index b1086aac5ed..2b94849b46c 100644 --- a/src/mongo/db/catalog/index_build_block.h +++ b/src/mongo/db/catalog/index_build_block.h @@ -62,7 +62,7 @@ public: * * Must be called from within a `WriteUnitOfWork` */ - Status init(OperationContext* opCtx, Collection* collection); + Status init(OperationContext* opCtx, Collection* collection, bool forRecovery); /** * Makes sure that an entry for the index was created at startup in the IndexCatalog. Returns diff --git a/src/mongo/db/catalog/index_build_entry_test.cpp b/src/mongo/db/catalog/index_build_entry_test.cpp index ffa6a870874..d53c7e0046c 100644 --- a/src/mongo/db/catalog/index_build_entry_test.cpp +++ b/src/mongo/db/catalog/index_build_entry_test.cpp @@ -32,6 +32,7 @@ #include <string> #include <vector> +#include "mongo/bson/bson_validate.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/bsontypes.h" @@ -124,7 +125,7 @@ TEST(IndexBuildEntryTest, SerializeAndDeserialize) { entry.setCommitReadyMembers(generateCommitReadyMembers(3)); BSONObj obj = entry.toBSON(); - ASSERT_TRUE(obj.valid()); + ASSERT_TRUE(validateBSON(obj).isOK()); IDLParserErrorContext ctx("IndexBuildsEntry Parser"); IndexBuildEntry rebuiltEntry = IndexBuildEntry::parse(ctx, obj); diff --git a/src/mongo/db/catalog/index_builds_manager.cpp b/src/mongo/db/catalog/index_builds_manager.cpp index a5db8f4536f..0c44c44de48 100644 --- a/src/mongo/db/catalog/index_builds_manager.cpp +++ b/src/mongo/db/catalog/index_builds_manager.cpp @@ -113,7 +113,8 @@ Status IndexBuildsManager::setUpIndexBuild(OperationContext* opCtx, std::vector<BSONObj> indexes; try { indexes = writeConflictRetry(opCtx, "IndexBuildsManager::setUpIndexBuild", nss.ns(), [&]() { - return uassertStatusOK(builder->init(opCtx, collection, specs, onInit, resumeInfo)); + return uassertStatusOK( + builder->init(opCtx, collection, specs, onInit, options.forRecovery, resumeInfo)); }); } catch (const DBException& ex) { return ex.toStatus(); diff --git a/src/mongo/db/catalog/index_builds_manager.h b/src/mongo/db/catalog/index_builds_manager.h index cddb46c3d16..37614751eb5 100644 --- a/src/mongo/db/catalog/index_builds_manager.h +++ b/src/mongo/db/catalog/index_builds_manager.h @@ -73,6 +73,7 @@ public: IndexConstraints indexConstraints = IndexConstraints::kEnforce; IndexBuildProtocol protocol = IndexBuildProtocol::kSinglePhase; IndexBuildMethod method = IndexBuildMethod::kHybrid; + bool forRecovery = false; }; IndexBuildsManager() = default; diff --git a/src/mongo/db/catalog/index_catalog.h b/src/mongo/db/catalog/index_catalog.h index 58e48e08be5..6f4fa0c8ae3 100644 --- a/src/mongo/db/catalog/index_catalog.h +++ b/src/mongo/db/catalog/index_catalog.h @@ -419,6 +419,16 @@ public: const IndexDescriptor* desc) = 0; /** + * Resets the index given its descriptor. + * + * This can only be called during startup recovery as it involves recreating the index table to + * allow bulk cursors to be used again. + */ + virtual Status resetUnfinishedIndexForRecovery(OperationContext* opCtx, + Collection* collection, + const IndexDescriptor* desc) = 0; + + /** * Drops an unfinished index given its descriptor. * * The caller must hold the collection X lock. diff --git a/src/mongo/db/catalog/index_catalog_entry.h b/src/mongo/db/catalog/index_catalog_entry.h index 9761f590c9d..2cf80bb8d3f 100644 --- a/src/mongo/db/catalog/index_catalog_entry.h +++ b/src/mongo/db/catalog/index_catalog_entry.h @@ -95,6 +95,7 @@ public: /// --------------------- virtual void setIsReady(bool newIsReady) = 0; + virtual void setIsFrozen(bool newIsFrozen) = 0; virtual void setDropped() = 0; virtual bool isDropped() const = 0; diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.cpp b/src/mongo/db/catalog/index_catalog_entry_impl.cpp index 498165df637..e104623ef2d 100644 --- a/src/mongo/db/catalog/index_catalog_entry_impl.cpp +++ b/src/mongo/db/catalog/index_catalog_entry_impl.cpp @@ -179,6 +179,10 @@ void IndexCatalogEntryImpl::setIsReady(bool newIsReady) { _isReady = newIsReady; } +void IndexCatalogEntryImpl::setIsFrozen(bool newIsFrozen) { + _isFrozen = newIsFrozen; +} + void IndexCatalogEntryImpl::setMultikey(OperationContext* opCtx, const CollectionPtr& collection, const KeyStringSet& multikeyMetadataKeys, diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.h b/src/mongo/db/catalog/index_catalog_entry_impl.h index 30ef5a80921..0760989b99a 100644 --- a/src/mongo/db/catalog/index_catalog_entry_impl.h +++ b/src/mongo/db/catalog/index_catalog_entry_impl.h @@ -110,6 +110,8 @@ public: void setIsReady(bool newIsReady) final; + void setIsFrozen(bool newIsFrozen) final; + void setDropped() final { _isDropped.store(true); } diff --git a/src/mongo/db/catalog/index_catalog_impl.cpp b/src/mongo/db/catalog/index_catalog_impl.cpp index 4a4d0b64826..2c3086a54cf 100644 --- a/src/mongo/db/catalog/index_catalog_impl.cpp +++ b/src/mongo/db/catalog/index_catalog_impl.cpp @@ -623,7 +623,7 @@ StatusWith<BSONObj> IndexCatalogImpl::createIndexOnEmptyCollection(OperationCont boost::optional<UUID> buildUUID = boost::none; IndexBuildBlock indexBuildBlock( collection->ns(), spec, IndexBuildMethod::kForeground, buildUUID); - status = indexBuildBlock.init(opCtx, collection); + status = indexBuildBlock.init(opCtx, collection, /*forRecovery=*/false); if (!status.isOK()) return status; @@ -1230,6 +1230,73 @@ Status IndexCatalogImpl::dropIndex(OperationContext* opCtx, return dropIndexEntry(opCtx, collection, entry); } +Status IndexCatalogImpl::resetUnfinishedIndexForRecovery(OperationContext* opCtx, + Collection* collection, + const IndexDescriptor* desc) { + invariant(opCtx->lockState()->isCollectionLockedForMode(collection->ns(), MODE_X)); + invariant(opCtx->lockState()->inAWriteUnitOfWork()); + + IndexCatalogEntry* entry = desc->getEntry(); + const std::string indexName = entry->descriptor()->indexName(); + + // Only indexes that aren't ready can be reset. + invariant(!collection->isIndexReady(indexName)); + + auto released = [&] { + if (auto released = _readyIndexes.release(entry->descriptor())) { + invariant(!released, "Cannot reset a ready index"); + } + if (auto released = _buildingIndexes.release(entry->descriptor())) { + return released; + } + if (auto released = _frozenIndexes.release(entry->descriptor())) { + return released; + } + MONGO_UNREACHABLE; + }(); + + LOGV2(6987700, + "Resetting unfinished index", + logAttrs(collection->ns()), + "index"_attr = indexName, + "ident"_attr = released->getIdent()); + + invariant(released.get() == entry); + + // Drop the ident if it exists. The storage engine will return OK if the ident is not found. + auto engine = opCtx->getServiceContext()->getStorageEngine(); + const std::string ident = released->getIdent(); + Status status = engine->getEngine()->dropIdent(opCtx->recoveryUnit(), ident); + if (!status.isOK()) { + return status; + } + + // Recreate the ident on-disk. DurableCatalog::createIndex() will lookup the ident internally + // using the catalogId and index name. + status = DurableCatalog::get(opCtx)->createIndex(opCtx, + collection->getCatalogId(), + collection->ns(), + collection->getCollectionOptions(), + released->descriptor()); + if (!status.isOK()) { + return status; + } + + // Update the index entry state in preparation to rebuild the index. + if (!released->accessMethod()) { + std::unique_ptr<SortedDataInterface> sdi = engine->getEngine()->getSortedDataInterface( + opCtx, collection->ns(), collection->getCollectionOptions(), ident, desc); + std::unique_ptr<IndexAccessMethod> accessMethod = + IndexAccessMethod::make(released.get(), std::move(sdi)); + released->setAccessMethod(std::move(accessMethod)); + } + + released->setIsFrozen(false); + _buildingIndexes.add(std::move(released)); + + return Status::OK(); +} + Status IndexCatalogImpl::dropUnfinishedIndex(OperationContext* opCtx, Collection* collection, const IndexDescriptor* desc) { diff --git a/src/mongo/db/catalog/index_catalog_impl.h b/src/mongo/db/catalog/index_catalog_impl.h index f9baf272f11..223399e5bf0 100644 --- a/src/mongo/db/catalog/index_catalog_impl.h +++ b/src/mongo/db/catalog/index_catalog_impl.h @@ -200,6 +200,9 @@ public: Status dropIndex(OperationContext* opCtx, Collection* collection, const IndexDescriptor* desc) override; + Status resetUnfinishedIndexForRecovery(OperationContext* opCtx, + Collection* collection, + const IndexDescriptor* desc) override; Status dropUnfinishedIndex(OperationContext* opCtx, Collection* collection, const IndexDescriptor* desc) override; diff --git a/src/mongo/db/catalog/index_key_validate.cpp b/src/mongo/db/catalog/index_key_validate.cpp index 8433b958894..20dd9f37fc0 100644 --- a/src/mongo/db/catalog/index_key_validate.cpp +++ b/src/mongo/db/catalog/index_key_validate.cpp @@ -497,6 +497,9 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in << "' is only allowed when '" << IndexDescriptor::kKeyPatternFieldName << "' is {\"$**\": ±1}"}; } + if (key.nFields() != 1) { + return {ErrorCodes::CannotCreateIndex, "wildcard indexes do not allow compounding"}; + } if (indexSpecElem.embeddedObject().isEmpty()) { return {ErrorCodes::FailedToParse, diff --git a/src/mongo/db/catalog/multi_index_block.cpp b/src/mongo/db/catalog/multi_index_block.cpp index 10ac2a47fdd..a0e8382959e 100644 --- a/src/mongo/db/catalog/multi_index_block.cpp +++ b/src/mongo/db/catalog/multi_index_block.cpp @@ -185,7 +185,7 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(OperationContext* opCtx, const BSONObj& spec, OnInitFn onInit) { const auto indexes = std::vector<BSONObj>(1, spec); - return init(opCtx, collection, indexes, onInit, boost::none); + return init(opCtx, collection, indexes, onInit, /*forRecovery=*/false, boost::none); } StatusWith<std::vector<BSONObj>> MultiIndexBlock::init( @@ -193,6 +193,7 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init( CollectionWriter& collection, const std::vector<BSONObj>& indexSpecs, OnInitFn onInit, + bool forRecovery, const boost::optional<ResumeIndexInfo>& resumeInfo) { invariant(opCtx->lockState()->isCollectionLockedForMode(collection->ns(), MODE_X), str::stream() << "Collection " << collection->ns() << " with UUID " @@ -246,27 +247,31 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init( for (size_t i = 0; i < indexSpecs.size(); i++) { BSONObj info = indexSpecs[i]; - StatusWith<BSONObj> statusWithInfo = - collection->getIndexCatalog()->prepareSpecForCreate( - opCtx, collection.get(), info, resumeInfo); - Status status = statusWithInfo.getStatus(); - if (!status.isOK()) { - // If we were given two identical indexes to build, we will run into an error trying - // to set up the same index a second time in this for-loop. This is the only way to - // encounter this error because callers filter out ready/in-progress indexes and - // start the build while holding a lock throughout. - if (status == ErrorCodes::IndexBuildAlreadyInProgress) { - invariant(indexSpecs.size() > 1, - str::stream() - << "Collection: " << collection->ns() << " (" << _collectionUUID - << "), Index spec: " << indexSpecs.front()); - return { - ErrorCodes::OperationFailed, - "Cannot build two identical indexes. Try again without duplicate indexes."}; + if (!forRecovery) { + // We skip this step when initializing unfinished index builds during startup + // recovery as they are already in the index catalog. + StatusWith<BSONObj> statusWithInfo = + collection->getIndexCatalog()->prepareSpecForCreate( + opCtx, collection.get(), info, resumeInfo); + Status status = statusWithInfo.getStatus(); + if (!status.isOK()) { + // If we were given two identical indexes to build, we will run into an error + // trying to set up the same index a second time in this for-loop. This is the + // only way to encounter this error because callers filter out ready/in-progress + // indexes and start the build while holding a lock throughout. + if (status == ErrorCodes::IndexBuildAlreadyInProgress) { + invariant(indexSpecs.size() > 1, + str::stream() << "Collection: " << collection->ns() << " (" + << _collectionUUID + << "), Index spec: " << indexSpecs.front()); + return {ErrorCodes::OperationFailed, + "Cannot build two identical indexes. Try again without duplicate " + "indexes."}; + } + return status; } - return status; + info = statusWithInfo.getValue(); } - info = statusWithInfo.getValue(); indexInfoObjs.push_back(info); boost::optional<TimeseriesOptions> options = collection->getTimeseriesOptions(); @@ -302,7 +307,7 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init( status = index.block->initForResume( opCtx, collection.getWritableCollection(), *stateInfo, resumeInfo->getPhase()); } else { - status = index.block->init(opCtx, collection.getWritableCollection()); + status = index.block->init(opCtx, collection.getWritableCollection(), forRecovery); } if (!status.isOK()) return status; @@ -685,10 +690,10 @@ Status MultiIndexBlock::_insert(OperationContext* opCtx, // collection to have it. if (_containsIndexBuildOnTimeseriesMeasurement && *collection->getTimeseriesBucketsMayHaveMixedSchemaData()) { - bool docHasMixedSchemaData = + auto docHasMixedSchemaData = collection->doesTimeseriesBucketsDocContainMixedSchemaData(doc); - if (docHasMixedSchemaData) { + if (docHasMixedSchemaData.isOK() && docHasMixedSchemaData.getValue()) { LOGV2(6057700, "Detected mixed-schema data in time-series bucket collection", logAttrs(collection->ns()), @@ -704,7 +709,8 @@ Status MultiIndexBlock::_insert(OperationContext* opCtx, auto replCoord = repl::ReplicationCoordinator::get(opCtx); const bool replSetAndNotPrimary = !replCoord->canAcceptWritesFor(opCtx, collection->ns()); - if (docHasMixedSchemaData && !replSetAndNotPrimary) { + if (docHasMixedSchemaData.isOK() && docHasMixedSchemaData.getValue() && + !replSetAndNotPrimary) { return timeseriesMixedSchemaDataFailure(collection.get()); } } @@ -972,15 +978,21 @@ Status MultiIndexBlock::commit(OperationContext* opCtx, onCommit(); - // Update the 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag to false in order to - // allow subsequent index builds to skip checking bucket documents for mixed-schema data. + // We can't update the 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag here as it + // requires the change to be driven by the router role. It means that subsequent index builds + // and other systems needs to treat this collection as-if it contains mixed-schema data even if + // it might not. We log a warning that can be used to initiate changing the flag. Note: just + // because this node doesn't contain mixed-schema it doesn't mean that other shards can't have + // mixed schema data. This flag needs to be consistent across the shards. if (_containsIndexBuildOnTimeseriesMeasurement && !_timeseriesBucketContainsMixedSchemaData) { boost::optional<bool> mayContainMixedSchemaData = collection->getTimeseriesBucketsMayHaveMixedSchemaData(); invariant(mayContainMixedSchemaData); if (*mayContainMixedSchemaData) { - collection->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx, false); + LOGV2_WARNING(9301400, + "Index build finished for time-series collection marked as containing " + "mixed schema buckets without detecting any buckets with mixed schema."); } } diff --git a/src/mongo/db/catalog/multi_index_block.h b/src/mongo/db/catalog/multi_index_block.h index 840770595cf..5220f8dd6f0 100644 --- a/src/mongo/db/catalog/multi_index_block.h +++ b/src/mongo/db/catalog/multi_index_block.h @@ -114,6 +114,7 @@ public: CollectionWriter& collection, const std::vector<BSONObj>& specs, OnInitFn onInit, + bool forRecovery, const boost::optional<ResumeIndexInfo>& resumeInfo = boost::none); StatusWith<std::vector<BSONObj>> init(OperationContext* opCtx, CollectionWriter& collection, diff --git a/src/mongo/db/catalog/multi_index_block_test.cpp b/src/mongo/db/catalog/multi_index_block_test.cpp index 2fb9caf7371..e9153ba35dc 100644 --- a/src/mongo/db/catalog/multi_index_block_test.cpp +++ b/src/mongo/db/catalog/multi_index_block_test.cpp @@ -90,8 +90,11 @@ TEST_F(MultiIndexBlockTest, CommitWithoutInsertingDocuments) { AutoGetCollection autoColl(operationContext(), getNSS(), MODE_X); CollectionWriter coll(operationContext(), autoColl); - auto specs = unittest::assertGet(indexer->init( - operationContext(), coll, std::vector<BSONObj>(), MultiIndexBlock::kNoopOnInitFn)); + auto specs = unittest::assertGet(indexer->init(operationContext(), + coll, + std::vector<BSONObj>(), + MultiIndexBlock::kNoopOnInitFn, + /*forRecovery=*/false)); ASSERT_EQUALS(0U, specs.size()); ASSERT_OK(indexer->dumpInsertsFromBulk(operationContext(), coll.get())); @@ -113,8 +116,11 @@ TEST_F(MultiIndexBlockTest, CommitAfterInsertingSingleDocument) { AutoGetCollection autoColl(operationContext(), getNSS(), MODE_X); CollectionWriter coll(operationContext(), autoColl); - auto specs = unittest::assertGet(indexer->init( - operationContext(), coll, std::vector<BSONObj>(), MultiIndexBlock::kNoopOnInitFn)); + auto specs = unittest::assertGet(indexer->init(operationContext(), + coll, + std::vector<BSONObj>(), + MultiIndexBlock::kNoopOnInitFn, + /*forRecovery=*/false)); ASSERT_EQUALS(0U, specs.size()); ASSERT_OK( @@ -146,8 +152,11 @@ TEST_F(MultiIndexBlockTest, AbortWithoutCleanupAfterInsertingSingleDocument) { AutoGetCollection autoColl(operationContext(), getNSS(), MODE_X); CollectionWriter coll(operationContext(), autoColl); - auto specs = unittest::assertGet(indexer->init( - operationContext(), coll, std::vector<BSONObj>(), MultiIndexBlock::kNoopOnInitFn)); + auto specs = unittest::assertGet(indexer->init(operationContext(), + coll, + std::vector<BSONObj>(), + MultiIndexBlock::kNoopOnInitFn, + /*forRecovery=*/false)); ASSERT_EQUALS(0U, specs.size()); ASSERT_OK( indexer->insertSingleDocumentForInitialSyncOrRecovery(operationContext(), diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp index ee4b9477100..01bc87f7e2b 100644 --- a/src/mongo/db/catalog/rename_collection.cpp +++ b/src/mongo/db/catalog/rename_collection.cpp @@ -885,6 +885,10 @@ void validateNamespacesForRenameCollection(OperationContext* opCtx, "renaming system.views collection or renaming to system.views is not allowed", !source.isSystemDotViews() && !target.isSystemDotViews()); + uassert(ErrorCodes::IllegalOperation, + "renaming system.users collection or renaming to system.users is not allowed", + !source.isSystemDotUsers() && !target.isSystemDotUsers()); + if (source.isTimeseriesBucketsCollection()) { uassert(ErrorCodes::IllegalOperation, "Renaming system.buckets collections is not allowed", diff --git a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp new file mode 100644 index 00000000000..66ed55b6e57 --- /dev/null +++ b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp @@ -0,0 +1,161 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include <algorithm> +#include <cstring> +#include <fmt/format.h> +#include <pcrecpp.h> + +#include "mongo/base/string_data.h" +#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_util.h" +#include "mongo/util/ctype.h" + +namespace mongo { + +const static StaticImmortal<pcrecpp::RE> appMetadataRegex( + R"re(((?<=^|,)\s*(?:app_metadata|\"app_metadata\")\s*[=:]\s*[({[]\s*))re"); + +static pcrecpp::RE flagMatchRegex(StringData flagName) { + // This check is overly strict, but it suffices for now and ensures that both: + // - The flag name is a valid WiredTiger identifier, and + // - It can be used in the regular expression without needing to escape it + invariant(std::all_of(flagName.begin(), flagName.end(), ctype::isAlpha)); + + // Some examples of possible matches: + // `flag=false` + // `flag:true` + // `flag` + // ` "flag" = false ` + // ` "flag" ` + return pcrecpp::RE(fmt::format( + R"re(((?<=[,({{[])\s*(?:{0}|\"{0}\")(?:\s*[=:]\s*(true|false))?\s*(?=[,)}}\]])))re", + flagName)); +} + +static std::map<StringData, boost::optional<bool>> getFlagsFromWtConfigStringAppMetadata( + const std::string& configString, const std::vector<StringData>& flagNames) { + std::map<StringData, boost::optional<bool>> flags; + + for (const auto& flagName : flagNames) { + auto flagRegex = flagMatchRegex(flagName); + pcrecpp::StringPiece fullMatch, flagValueStr; + auto matches = flagRegex.PartialMatch(configString, &fullMatch, &flagValueStr); + flags.emplace(flagName, + matches ? boost::optional<bool>(flagValueStr == "" || flagValueStr == "true") + : boost::none); + } + + return flags; +} + +std::map<StringData, boost::optional<bool>> getFlagsFromStorageEngineBson( + const BSONObj& storageEngineOptions, const std::vector<StringData>& flagNames) { + auto configString = WiredTigerUtil::getConfigStringFromStorageOptions(storageEngineOptions); + return getFlagsFromWtConfigStringAppMetadata(configString.value_or(""), flagNames); +} + +boost::optional<bool> getFlagFromStorageEngineBson(const BSONObj& storageEngineOptions, + StringData flagName) { + return getFlagsFromStorageEngineBson(storageEngineOptions, {flagName})[flagName]; +} + +// Finds or adds the 'app_metadata=(...)' struct inside a WiredTiger config string +// Returns the position inside the struct (after the delimiter, before the first key-value) +static size_t findOrAddAppMetadataStructToConfigString(std::string& configString) { + pcrecpp::StringPiece fullMatch; + auto matches = appMetadataRegex->PartialMatch(configString, &fullMatch); + if (!matches) + configString += configString.empty() ? "app_metadata=()" : ",app_metadata=()"; + return matches ? (fullMatch.data() + fullMatch.size() - configString.data()) + : configString.size() - 1; +} + +// Expand a [pos, len) range inside a config string to include a leading or trailing comma separator +static void expandRangeToIncludeSeparator(const std::string& configString, + size_t& pos, + size_t& len) { + if (pos > 0 && configString[pos - 1] == ',') { + pos--; + len++; + } else if (pos + len < configString.size() && configString[pos + len] == ',') { + len++; + } +} + +static void setFlagsToWtConfigStringAppMetadata( + std::string& configString, const std::map<StringData, boost::optional<bool>>& flags) { + auto metadataPos = findOrAddAppMetadataStructToConfigString(configString); + + for (const auto& [flagName, flagValue] : flags) { + auto flagRegex = flagMatchRegex(flagName); + pcrecpp::StringPiece fullMatch; + // "- 1" allows the positive lookbehind (?<=) at the start of the regex to work + auto matches = flagRegex.PartialMatch(&configString[metadataPos - 1], &fullMatch); + if (matches) { + size_t pos = fullMatch.data() - configString.data(), len = fullMatch.size(); + + if (flagValue.has_value()) { // Replace existing flag + auto flagItem = fmt::format("{}={}", flagName, *flagValue); + configString.replace(pos, len, flagItem); + } else { // Unset existing flag + expandRangeToIncludeSeparator(configString, pos, len); + configString.erase(pos, len); + } + } else if (flagValue.has_value()) { // Add new flag + auto metadataEmpty = strchr(")]}", configString[metadataPos]) != nullptr; + auto flagItem = fmt::format("{}={}{}", flagName, *flagValue, metadataEmpty ? "" : ","); + configString.insert(metadataPos, flagItem); + } + } +} + +BSONObj setFlagsToStorageEngineBson(const BSONObj& storageEngineOptions, + const std::map<StringData, boost::optional<bool>>& flags) { + auto configString = + WiredTigerUtil::getConfigStringFromStorageOptions(storageEngineOptions).value_or(""); + setFlagsToWtConfigStringAppMetadata(configString, flags); + + // Both for safety, and because the regex-based parser can not handle some theoretical cases, + // sanity check that the resulting string is a valid WiredTiger configuration string + auto configStringObj = BSON(WiredTigerUtil::kConfigStringField << configString); + tassert(9218600, + "The resulting WiredTiger configuration string is not valid", + WiredTigerUtil::checkTableCreationOptions(configStringObj.firstElement()).isOK()); + + return WiredTigerUtil::setConfigStringToStorageOptions(storageEngineOptions, configString); +} + +BSONObj setFlagToStorageEngineBson(const BSONObj& storageEngineOptions, + StringData flagName, + boost::optional<bool> flagValue) { + return setFlagsToStorageEngineBson(storageEngineOptions, {{flagName, flagValue}}); +} + +} // namespace mongo diff --git a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h new file mode 100644 index 00000000000..cb4a1b20e2f --- /dev/null +++ b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include <boost/optional/optional.hpp> +#include <map> +#include <vector> + +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobj.h" + +namespace mongo { + +/** + * Utility functions to get or set boolean flags from/to a storage engine options object + * (see `CollectionOptions::storageEngine`). + * + * The idea is that for exceptional (workaround) purposes, we can use the storage engine + * options object as a flexible structure where new fields can be added retroactively, + * unlike the other parts of the catalog which generally have non-flexible / strict validations. + * For more information, see: SERVER-91195, SERVER-92186. + */ + +std::map<StringData, boost::optional<bool>> getFlagsFromStorageEngineBson( + const BSONObj& storageEngineOptions, const std::vector<StringData>& flagNames); + +boost::optional<bool> getFlagFromStorageEngineBson(const BSONObj& storageEngineOptions, + StringData flagName); + +[[nodiscard]] BSONObj setFlagsToStorageEngineBson( + const BSONObj& storageEngineOptions, const std::map<StringData, boost::optional<bool>>& flags); + +[[nodiscard]] BSONObj setFlagToStorageEngineBson(const BSONObj& storageEngineOptions, + StringData flagName, + boost::optional<bool> flagValue); + +} // namespace mongo diff --git a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp new file mode 100644 index 00000000000..5305668e8fc --- /dev/null +++ b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp @@ -0,0 +1,250 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_util.h" +#include "mongo/unittest/bson_test_util.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + +static BSONObj makeStorageEngineWithConfigString(StringData configString) { + return BSON(kWiredTigerEngineName << BSON(WiredTigerUtil::kConfigStringField << configString)); +} + +static BSONObj addExtraFields(const BSONObj& storageEngineOptions) { + auto wtObj = storageEngineOptions[kWiredTigerEngineName].Obj(); + return storageEngineOptions.addFields(BSON("dummy1" + << "value1" << kWiredTigerEngineName + << wtObj.addFields(BSON("dummy2" + << "value2")))); +} + +TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenNoWiredTigerConfigString) { + auto options = BSONObj(); + + auto flag = getFlagFromStorageEngineBson(options, "flagA"); + + ASSERT_EQ(boost::none, flag); +} + +TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenOptionsDoesNotContainMetadata) { + auto options = makeStorageEngineWithConfigString("access_pattern_hint=random"); + + auto flag = getFlagFromStorageEngineBson(options, "flagA"); + + ASSERT_EQ(boost::none, flag); +} + +TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenMetadataDoesNotContainTheFlag) { + auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1)"); + + auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); + + ASSERT_EQ(boost::none, flags["flagA"]); + ASSERT_EQ(boost::none, flags["flagB"]); +} + +TEST(StorageEngineFlagsParserTest, GetValueWhenMetadataContainsASingleFlag) { + auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA=true)"); + + auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); + + ASSERT_EQ(true, flags["flagA"]); + ASSERT_EQ(boost::none, flags["flagB"]); +} + +TEST(StorageEngineFlagsParserTest, GetValueWhenMetadataContainsMultipleFlags) { + auto options = makeStorageEngineWithConfigString("app_metadata=(flagB=true,flagA=false)"); + + auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); + + ASSERT_EQ(false, flags["flagA"]); + ASSERT_EQ(true, flags["flagB"]); +} + +TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenMetadataContainsAnInvalidValue) { + auto options = + makeStorageEngineWithConfigString("app_metadata=(flagB=(hello=world),flagA=true)"); + + auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); + + ASSERT_EQ(true, flags["flagA"]); + ASSERT_EQ(boost::none, flags["flagB"]); +} + +TEST(StorageEngineFlagsParserTest, GetTrueWhenMetadataContainsAKeyWithNoValue) { + auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA)"); + + auto flag = getFlagFromStorageEngineBson(options, "flagA"); + + ASSERT_EQ(true, flag); +} + +TEST(StorageEngineFlagsParserTest, GetIgnoresUnknownStorageEngineFields) { + auto options = addExtraFields(makeStorageEngineWithConfigString("app_metadata=(flagA=true)")); + + auto flag = getFlagFromStorageEngineBson(options, "flagA"); + + ASSERT_EQ(true, flag); +} + +TEST(StorageEngineFlagsParserTest, GetHandlesTrickyFormatting) { + auto options = addExtraFields(makeStorageEngineWithConfigString( + " access_pattern_hint = random , \"app_metadata\" : [ x=y , " + "\"flagB\": true , z : t ]")); + + auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); + + ASSERT_EQ(boost::none, flags["flagA"]); + ASSERT_EQ(true, flags["flagB"]); +} + +TEST(StorageEngineFlagsParserTest, AddFlagToEmptyStorageEngineBson) { + auto options = BSONObj(); + + auto newOptions = setFlagToStorageEngineBson(options, "flagA", true); + + ASSERT_BSONOBJ_EQ(newOptions, makeStorageEngineWithConfigString("app_metadata=(flagA=true)")); +} + +TEST(StorageEngineFlagsParserTest, AddFlagToEmptyConfigString) { + auto options = makeStorageEngineWithConfigString(""); + + auto newOptions = setFlagToStorageEngineBson(options, "flagA", true); + + ASSERT_BSONOBJ_EQ(newOptions, makeStorageEngineWithConfigString("app_metadata=(flagA=true)")); +} + +TEST(StorageEngineFlagsParserTest, AddFlagToExistingConfigStringWithNoMetadata) { + auto options = makeStorageEngineWithConfigString("access_pattern_hint=random"); + + auto newOptions = setFlagToStorageEngineBson(options, "flagA", true); + + ASSERT_BSONOBJ_EQ( + newOptions, + makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=true)")); +} + +TEST(StorageEngineFlagsParserTest, AddFlagToExistingConfigStringWithEmptyMetadata) { + auto options = makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=()"); + + auto newOptions = setFlagToStorageEngineBson(options, "flagA", false); + + ASSERT_BSONOBJ_EQ( + newOptions, + makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=false)")); +} + +TEST(StorageEngineFlagsParserTest, AddFlagToExistingConfigStringWithOtherFlags) { + auto options = makeStorageEngineWithConfigString( + "access_pattern_hint=random,app_metadata=(hello2=world2,flagB=true)"); + + auto newOptions = setFlagToStorageEngineBson(options, "flagA", false); + + ASSERT_BSONOBJ_EQ( + newOptions, + makeStorageEngineWithConfigString( + "access_pattern_hint=random,app_metadata=(flagA=false,hello2=world2,flagB=true)")); +} + +TEST(StorageEngineFlagsParserTest, SetExistingFlag) { + auto options = + makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=false)"); + + auto newOptions = setFlagToStorageEngineBson(options, "flagA", true); + + ASSERT_BSONOBJ_EQ( + newOptions, + makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=true)")); +} + +TEST(StorageEngineFlagsParserTest, RemoveExistingFlag) { + auto options = makeStorageEngineWithConfigString( + "access_pattern_hint=random,app_metadata=(flagB=true,x=y,z=t)"); + + auto newOptions = setFlagToStorageEngineBson(options, "flagB", boost::none); + + ASSERT_BSONOBJ_EQ( + newOptions, + makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(x=y,z=t)")); +} + +TEST(StorageEngineFlagsParserTest, SetMultipleFlags) { + auto options = makeStorageEngineWithConfigString( + "access_pattern_hint=random,app_metadata=(x=y,flagB=true,z=t,flagC=true)"); + + auto newOptions = setFlagsToStorageEngineBson( + options, {{"flagB", false}, {"flagA", true}, {"flagC", boost::none}}); + + ASSERT_BSONOBJ_EQ( + newOptions, + makeStorageEngineWithConfigString( + "access_pattern_hint=random,app_metadata=(flagA=true,x=y,flagB=false,z=t)")); +} + +TEST(StorageEngineFlagsParserTest, SetFlagWhenMetadataContainsAKeyWithNoValue) { + auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA)"); + + auto newOptions = setFlagToStorageEngineBson(options, "flagA", false); + + ASSERT_BSONOBJ_EQ( + newOptions, + makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA=false)")); +} + +TEST(StorageEngineFlagsParserTest, SetPreservesUnknownStorageEngineFields) { + auto options = addExtraFields(makeStorageEngineWithConfigString( + "access_pattern_hint=random,app_metadata=(x=y,flagB=false,z=t)")); + + auto newOptions = setFlagToStorageEngineBson(options, "flagB", true); + + auto expected = addExtraFields(makeStorageEngineWithConfigString( + "access_pattern_hint=random,app_metadata=(x=y,flagB=true,z=t)")); + ASSERT_BSONOBJ_EQ(newOptions, expected); +} + +TEST(StorageEngineFlagsParserTest, SetHandlesTrickyFormatting) { + auto options = addExtraFields(makeStorageEngineWithConfigString( + " access_pattern_hint = random , \"app_metadata\" : [ x=y , " + "\"flagB\": false , z : t , flagC : true ]")); + + auto newOptions = setFlagsToStorageEngineBson( + options, {{"flagA", false}, {"flagB", true}, {"flagC", boost::none}}); + + auto expected = addExtraFields(makeStorageEngineWithConfigString( + " access_pattern_hint = random , \"app_metadata\" : [ flagA=false,x=y " + " ,flagB=true, z : t ]")); + ASSERT_BSONOBJ_EQ(newOptions, expected); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/catalog/validate_adaptor.cpp b/src/mongo/db/catalog/validate_adaptor.cpp index 3967bd93d82..9dc4a8dc71b 100644 --- a/src/mongo/db/catalog/validate_adaptor.cpp +++ b/src/mongo/db/catalog/validate_adaptor.cpp @@ -53,6 +53,10 @@ #include "mongo/db/storage/execution_context.h" #include "mongo/db/storage/key_string.h" #include "mongo/db/storage/record_store.h" +#include "mongo/db/storage/storage_parameters_gen.h" +#include "mongo/db/timeseries/flat_bson.h" +#include "mongo/db/timeseries/timeseries_constants.h" +#include "mongo/db/timeseries/timeseries_options.h" #include "mongo/logv2/log.h" #include "mongo/rpc/object_check.h" #include "mongo/util/fail_point.h" @@ -64,6 +68,7 @@ namespace { MONGO_FAIL_POINT_DEFINE(crashOnMultikeyValidateFailure); MONGO_FAIL_POINT_DEFINE(failIndexKeyOrdering); +MONGO_FAIL_POINT_DEFINE(failRecordStoreTraversal); // Set limit for size of corrupted records that will be reported. const long long kMaxErrorSizeBytes = 1 * 1024 * 1024; @@ -73,6 +78,9 @@ const long long kInterruptIntervalNumBytes = 50 * 1024 * 1024; // 50MB. static constexpr const char* kSchemaValidationFailedReason = "Detected one or more documents not compliant with the collection's schema. Check logs for log " "id 5363500."; +static constexpr const char* kTimeseriesValidationInconsistencyReason = + "Detected one or more documents in this collection incompatible with time-series " + "specifications. For more info, see logs with log id 6698300."; /** * Validate that for each record in a clustered RecordStore the record key (RecordId) matches the @@ -119,17 +127,163 @@ void schemaValidationFailed(CollectionValidation::ValidateState* state, state->setCollectionSchemaViolated(); - // TODO SERVER-65078: remove the testing proctor check. - // When testing is enabled, only warn about non-compliant documents to prevent test failures. - if (TestingProctor::instance().isEnabled() || - Collection::SchemaValidationResult::kWarn == result || state->warnOnSchemaValidation()) { + if (result != Collection::SchemaValidationResult::kPass) { results->warnings.push_back(kSchemaValidationFailedReason); - } else if (Collection::SchemaValidationResult::kError == result) { - results->errors.push_back(kSchemaValidationFailedReason); - results->valid = false; } } +/** + * Checks the value of the bucket's version and if it matches the types of 'data' fields. + */ +Status _validateTimeseriesControlVersion(const BSONObj& recordBson) { + int controlVersion = recordBson.getField(timeseries::kBucketControlFieldName) + .Obj() + .getField(timeseries::kBucketControlVersionFieldName) + .Number(); + if (controlVersion != 1 && controlVersion != 2) { + return Status( + ErrorCodes::BadValue, + fmt::format("Invalid value for 'control.version'. Expected 1 or 2, but got {}.", + controlVersion)); + } + auto dataType = controlVersion == 1 ? BSONType::Object : BSONType::BinData; + // In addition to checking dataType, make sure that closed buckets have BinData Column subtype + auto isCorrectType = [&](BSONElement el) { + if (controlVersion == 1) { + return el.type() == BSONType::Object; + } else { + return el.type() == BSONType::BinData && el.binDataType() == BinDataType::Column; + } + }; + BSONObj data = recordBson.getField(timeseries::kBucketDataFieldName).Obj(); + for (BSONObjIterator bi(data); bi.more();) { + BSONElement e = bi.next(); + if (!isCorrectType(e)) { + return Status(ErrorCodes::TypeMismatch, + fmt::format("Mismatch between time-series schema version and data field " + "type. Expected type {}, but got {}.", + mongo::typeName(dataType), + mongo::typeName(e.type()))); + } + } + return Status::OK(); +} + +/** + * Checks the equivalence between the min and max fields in 'control' for a bucket and + * the corresponding value in 'data'. + */ +Status _validateTimeseriesMinMax(const BSONObj& recordBson, const CollectionPtr& coll) { + BSONObj data = recordBson.getField(timeseries::kBucketDataFieldName).Obj(); + BSONObj control = recordBson.getField(timeseries::kBucketControlFieldName).Obj(); + BSONObj controlMin = control.getField(timeseries::kBucketControlMinFieldName).Obj(); + BSONObj controlMax = control.getField(timeseries::kBucketControlMaxFieldName).Obj(); + + auto dataFields = data.getFieldNames<std::set<std::string>>(); + auto controlMinFields = controlMin.getFieldNames<std::set<std::string>>(); + auto controlMaxFields = controlMax.getFieldNames<std::set<std::string>>(); + + // Checks that the number of 'control.min' and 'control.max' fields agrees with number of 'data' + // fields. + if (dataFields.size() != controlMinFields.size() || + dataFields.size() != controlMaxFields.size()) { + return Status( + ErrorCodes::BadValue, + fmt::format( + "Mismatch between the number of time-series control fields and the number " + "of data fields. " + "Control had {} min fields and {} max fields, but observed data had {} fields.", + controlMinFields.size(), + controlMaxFields.size(), + dataFields.size())); + }; + + // Used when checking min timestamp, which is rounded down by granularity. + auto granularity = coll->getTimeseriesOptions()->getGranularity(); + + // Validates that the 'control.min' and 'control.max' field values agree with 'data' field + // values. + for (auto fieldName : dataFields) { + timeseries::MinMax minmax; + auto field = data.getField(fieldName); + + for (BSONElement el : field.Obj()) { + minmax.update(el.wrap(fieldName), boost::none, coll->getDefaultCollator()); + } + auto controlFieldMin = controlMin.getField(fieldName); + auto controlFieldMax = controlMax.getField(fieldName); + auto min = minmax.min(); + auto max = minmax.max(); + + // Checks whether the min and max values between 'control' and 'data' match, taking + // timestamp granularity into account. + auto checkMinAndMaxMatch = [&]() { + if (fieldName == coll->getTimeseriesOptions()->getTimeField()) { + return controlFieldMin.Date() == + timeseries::roundTimestampToGranularity(min.getField(fieldName).Date(), + granularity) && + controlFieldMax.Date() == max.getField(fieldName).Date(); + } else { + return controlFieldMin.wrap().woCompare(min) == 0 && + controlFieldMax.wrap().woCompare(max) == 0; + } + }; + + if (!checkMinAndMaxMatch()) { + return Status( + ErrorCodes::BadValue, + fmt::format( + "Mismatch between time-series control and observed min or max for field {}. " + "Control had min {} and max {}, but observed data had min {} and max {}.", + fieldName, + controlFieldMin.toString(), + controlFieldMax.toString(), + min.toString(), + max.toString())); + } + } + + return Status::OK(); +} + +/** + * Validates the consistency of a time-series bucket. + */ +Status _validateTimeSeriesBucketRecord(const CollectionPtr& collection, + const BSONObj& recordBson, + ValidateResults* results) { + + if (Status status = _validateTimeseriesControlVersion(recordBson); !status.isOK()) { + return status; + } + + int version = recordBson.getField(timeseries::kBucketControlFieldName) + .Obj() + .getField(timeseries::kBucketControlVersionFieldName) + .Number(); + + // TODO(SERVER-67023): Check closed bucket as part of validation. + if (version == 1) { + if (Status status = _validateTimeseriesMinMax(recordBson, collection); !status.isOK()) { + return status; + } + } + + + return Status::OK(); +} + + +void _timeseriesValidationFailed(CollectionValidation::ValidateState* state, + ValidateResults* results) { + if (state->isTimeseriesDataInconsistent()) { + // Only report the warning message once. + return; + } + state->setTimeseriesDataInconsistent(); + + results->warnings.push_back(kTimeseriesValidationInconsistencyReason); +} BSONObj rehydrateKey(const BSONObj& keyPattern, const BSONObj& indexKey) { // We need to rehydrate the indexKey for improved readability. @@ -154,8 +308,9 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, const RecordId& recordId, const RecordData& record, size_t* dataSize, - ValidateResults* results) { - const Status status = validateBSON(record.data(), record.size()); + ValidateResults* results, + ValidationVersion validationVersion) { + const Status status = validateBSON(record.data(), record.size(), validationVersion); if (!status.isOK()) return status; @@ -328,15 +483,16 @@ namespace { // Ensures that index entries are in increasing or decreasing order. void _validateKeyOrder(OperationContext* opCtx, const IndexCatalogEntry* index, - const KeyString::Value& currKey, - const KeyString::Value& prevKey, + const KeyStringEntry& currKey, + const KeyStringEntry& prevKey, IndexValidateResults* results) { auto descriptor = index->descriptor(); bool unique = descriptor->unique(); // KeyStrings will be in strictly increasing order because all keys are sorted and they are in // the format (Key, RID), and all RecordIDs are unique. - if (currKey.compare(prevKey) <= 0 || MONGO_unlikely(failIndexKeyOrdering.shouldFail())) { + if (currKey.keyString.compare(prevKey.keyString) <= 0 || + MONGO_unlikely(failIndexKeyOrdering.shouldFail())) { if (results && results->valid) { results->errors.push_back(str::stream() << "index '" << descriptor->indexName() @@ -350,21 +506,20 @@ void _validateKeyOrder(OperationContext* opCtx, if (unique) { // Unique indexes must not have duplicate keys. - int cmp = currKey.compareWithoutRecordIdLong(prevKey); + int cmp = currKey.loc.isLong() + ? currKey.keyString.compareWithoutRecordIdLong(prevKey.keyString) + : currKey.keyString.compareWithoutRecordIdStr(prevKey.keyString); if (cmp != 0) { return; } if (results && results->valid) { - auto bsonKey = KeyString::toBson(currKey, Ordering::make(descriptor->keyPattern())); - auto firstRecordId = - KeyString::decodeRecordIdLongAtEnd(prevKey.getBuffer(), prevKey.getSize()); - auto secondRecordId = - KeyString::decodeRecordIdLongAtEnd(currKey.getBuffer(), currKey.getSize()); + auto bsonKey = + KeyString::toBson(currKey.keyString, Ordering::make(descriptor->keyPattern())); results->errors.push_back(str::stream() << "Unique index '" << descriptor->indexName() << "' has duplicate key: " << bsonKey - << ", first record: " << firstRecordId - << ", second record: " << secondRecordId); + << ", first record: " << prevKey.loc + << ", second record: " << currKey.loc); } if (results) { results->valid = false; @@ -383,8 +538,6 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, IndexInfo& indexInfo = _indexConsistency->getIndexInfo(indexName); int64_t numKeys = 0; - bool isFirstEntry = true; - // The progress meter will be inactive after traversing the record store to allow the message // and the total to be set to different values. if (!_progress->isActive()) { @@ -399,7 +552,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, KeyString::Builder firstKeyStringBuilder( version, BSONObj(), indexInfo.ord, KeyString::Discriminator::kExclusiveBefore); KeyString::Value firstKeyString = firstKeyStringBuilder.getValueCopy(); - KeyString::Value prevIndexKeyStringValue; + boost::optional<KeyStringEntry> prevIndexKeyStringEntry; // Ensure that this index has an open index cursor. const auto indexCursorIt = _validateState->getIndexCursors().find(indexName); @@ -430,9 +583,8 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, bool foundOldUniqueIndexKeys = false; while (indexEntry) { - if (!isFirstEntry) { - _validateKeyOrder( - opCtx, index, indexEntry->keyString, prevIndexKeyStringValue, &indexResults); + if (prevIndexKeyStringEntry) { + _validateKeyOrder(opCtx, index, *indexEntry, *prevIndexKeyStringEntry, &indexResults); } if (!foundOldUniqueIndexKeys && !descriptor->isIdIndex() && descriptor->unique() && @@ -463,8 +615,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, _progress->hit(); numKeys++; - isFirstEntry = false; - prevIndexKeyStringValue = indexEntry->keyString; + prevIndexKeyStringEntry = indexEntry; if (numKeys % kInterruptIntervalNumRecords == 0) { // Periodically checks for interrupts and yields. @@ -480,7 +631,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, "Error advancing index cursor", "error"_attr = ex.toString(), "index"_attr = indexName, - "prevKey"_attr = prevIndexKeyStringValue.toString()); + "prevKey"_attr = prevIndexKeyStringEntry->keyString.toString()); } throw; } @@ -566,7 +717,8 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, ValidateResults* results, - BSONObjBuilder* output) { + BSONObjBuilder* output, + ValidationVersion validationVersion) { _numRecords = 0; // need to reset it because this function can be called more than once. long long dataSizeTotal = 0; long long interruptIntervalNumBytes = 0; @@ -590,9 +742,10 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, // Because the progress meter is intended as an approximation, it's sufficient to get the number // of records when we begin traversing, even if this number may deviate from the final number. + const auto& coll = _validateState->getCollection(); const char* curopMessage = "Validate: scanning documents"; - const auto totalRecords = _validateState->getCollection()->getRecordStore()->numRecords(opCtx); - const auto rs = _validateState->getCollection()->getRecordStore(); + const auto totalRecords = coll->getRecordStore()->numRecords(opCtx); + const auto rs = coll->getRecordStore(); { stdx::unique_lock<Client> lk(*opCtx->getClient()); _progress.set(CurOp::get(opCtx)->setProgress_inlock(curopMessage, totalRecords)); @@ -603,6 +756,9 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, return; } + bool bucketMixedSchemaDataError = false; + bool bucketMinMaxMalformedError = false; + bool bucketMixedSchemaDataWarning = false; bool corruptRecordsSizeLimitWarning = false; const std::unique_ptr<SeekableRecordThrottleCursor>& traverseRecordStoreCursor = _validateState->getTraverseRecordStoreCursor(); @@ -616,7 +772,26 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, interruptIntervalNumBytes += dataSize; dataSizeTotal += dataSize; size_t validatedSize = 0; - Status status = validateRecord(opCtx, record->id, record->data, &validatedSize, results); + Status status = validateRecord( + opCtx, record->id, record->data, &validatedSize, results, validationVersion); + + // Log the out-of-order entries as errors. + // + // Validate uses a DataCorruptionDetectionMode::kLogAndContinue mode such that data + // corruption errors are logged without throwing, so certain checks must be duplicated here + // as well. + if ((prevRecordId.isValid() && prevRecordId > record->id) || + MONGO_unlikely(failRecordStoreTraversal.shouldFail())) { + // TODO SERVER-78040: Clean this up once we can insert errors blindly into the list and + // not care about deduplication. + static constexpr auto kErrorMessage = "Detected out-of-order documents. See logs."; + if (results->valid || + std::find(results->errors.begin(), results->errors.end(), kErrorMessage) == + results->errors.end()) { + results->errors.push_back(kErrorMessage); + results->valid = false; + } + } // validatedSize = dataSize is not a general requirement as some storage engines may use // padding, but we still require that they return the unpadded record data. @@ -646,8 +821,14 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, results->numRemovedCorruptRecords++; _numRecords--; } else { - if (results->valid) { - results->errors.push_back("Detected one or more invalid documents. See logs."); + // TODO SERVER-78040: Clean this up once we can insert errors blindly into the list + // and not care about deduplication. + static constexpr auto kErrorMessage = + "Detected one or more invalid documents. See logs."; + if (results->valid || + std::find(results->errors.begin(), results->errors.end(), kErrorMessage) == + results->errors.end()) { + results->errors.push_back(kErrorMessage); results->valid = false; } @@ -666,18 +847,76 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, // If the document is not corrupted, validate the document against this collection's // schema validator. Don't treat invalid documents as errors since documents can bypass // document validation when being inserted or updated. - auto result = - _validateState->getCollection()->checkValidation(opCtx, record->data.toBson()); + auto result = coll->checkValidation(opCtx, record->data.toBson()); if (result.first != Collection::SchemaValidationResult::kPass) { LOGV2_WARNING(5363500, "Document is not compliant with the collection's schema", - logAttrs(_validateState->getCollection()->ns()), + logAttrs(coll->ns()), "recordId"_attr = record->id, "reason"_attr = result.second); nNonCompliantDocuments++; schemaValidationFailed(_validateState, result.first, results); + } else if (coll->getTimeseriesOptions()) { + // Checks for time-series collection consistency. + Status bucketStatus = + _validateTimeSeriesBucketRecord(coll, record->data.toBson(), results); + + // This log id should be kept in sync with the associated warning messages that are + // returned to the client. + if (!bucketStatus.isOK()) { + LOGV2_WARNING(6698300, + "Document is not compliant with time-series specifications", + logAttrs(coll->ns()), + "recordId"_attr = record->id, + "reason"_attr = bucketStatus); + nNonCompliantDocuments++; + _timeseriesValidationFailed(_validateState, results); + } else { + auto containsMixedSchemaDataResponse = + coll->doesTimeseriesBucketsDocContainMixedSchemaData(record->data.toBson()); + if (!containsMixedSchemaDataResponse.isOK() && !bucketMinMaxMalformedError) { + bucketMinMaxMalformedError = true; + LOGV2_WARNING(8469900, + "Detected a time-series bucket with malformed min/max values", + logAttrs(coll->ns()), + "bucketId"_attr = record->id, + "error"_attr = containsMixedSchemaDataResponse.getStatus()); + results->errors.push_back( + str::stream() + << "Detected a time-series bucket with malformed min/max values"); + results->valid = false; + } else if (containsMixedSchemaDataResponse.isOK() && + containsMixedSchemaDataResponse.getValue()) { + bool mixedSchemaAllowed = + coll->getTimeseriesBucketsMayHaveMixedSchemaData().value_or(true); + if (mixedSchemaAllowed && !bucketMixedSchemaDataWarning) { + bucketMixedSchemaDataWarning = true; + LOGV2_WARNING(8469901, + "Detected a time-series bucket with mixed schema data", + logAttrs(coll->ns()), + "bucketId"_attr = record->id); + results->warnings.push_back( + str::stream() + << "Detected a time-series bucket with mixed schema data"); + } else if (!mixedSchemaAllowed && !bucketMixedSchemaDataError) { + bucketMixedSchemaDataError = true; + LOGV2_WARNING(8469902, + "Detected a time-series bucket with mixed schema data " + "when timeseriesBucketsMayHaveMixedSchemaData is false. " + "You can run the collMod command to set this flag", + logAttrs(coll->ns()), + "bucketId"_attr = record->id); + results->errors.push_back( + str::stream() + << "Detected a time-series bucket with mixed schema data when " + "timeseriesBucketsMayHaveMixedSchemaData is false. You can run " + "the collMod command to set this flag"); + results->valid = false; + } + } + } } } @@ -700,20 +939,18 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, << " invalid documents."); } - const auto fastCount = _validateState->getCollection()->numRecords(opCtx); + const auto fastCount = coll->numRecords(opCtx); if (_validateState->shouldEnforceFastCount() && fastCount != _numRecords) { - results->errors.push_back(str::stream() << "fast count (" << fastCount - << ") does not match number of records (" - << _numRecords << ") for collection '" - << _validateState->getCollection()->ns() << "'"); + results->errors.push_back( + str::stream() << "fast count (" << fastCount << ") does not match number of records (" + << _numRecords << ") for collection '" << coll->ns() << "'"); results->valid = false; } // Do not update the record store stats if we're in the background as we've validated a // checkpoint and it may not have the most up-to-date changes. if (results->valid && !_validateState->isBackground()) { - _validateState->getCollection()->getRecordStore()->updateStatsAfterRepair( - opCtx, _numRecords, dataSizeTotal); + coll->getRecordStore()->updateStatsAfterRepair(opCtx, _numRecords, dataSizeTotal); } } diff --git a/src/mongo/db/catalog/validate_adaptor.h b/src/mongo/db/catalog/validate_adaptor.h index ba548ca6358..99c0a2fceb1 100644 --- a/src/mongo/db/catalog/validate_adaptor.h +++ b/src/mongo/db/catalog/validate_adaptor.h @@ -57,7 +57,8 @@ public: const RecordId& recordId, const RecordData& record, size_t* dataSize, - ValidateResults* results); + ValidateResults* results, + ValidationVersion validationVersion = currentValidationVersion); /** * Traverses the index getting index entries to validate them and keep track of the index keys @@ -74,7 +75,8 @@ public: */ void traverseRecordStore(OperationContext* opCtx, ValidateResults* results, - BSONObjBuilder* output); + BSONObjBuilder* output, + ValidationVersion validationVersion); /** * Validates that the number of document keys matches the number of index keys previously diff --git a/src/mongo/db/catalog/validate_state.cpp b/src/mongo/db/catalog/validate_state.cpp index f40ebe0062e..01f80ac61cc 100644 --- a/src/mongo/db/catalog/validate_state.cpp +++ b/src/mongo/db/catalog/validate_state.cpp @@ -81,16 +81,26 @@ ValidateState::ValidateState(OperationContext* opCtx, _collection = CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, _nss); if (!_collection) { - if (CollectionCatalog::get(opCtx)->lookupView(opCtx, _nss)) { - uasserted(ErrorCodes::CommandNotSupportedOnView, "Cannot validate a view"); + auto view = CollectionCatalog::get(opCtx)->lookupView(opCtx, _nss); + if (!view) { + uasserted(ErrorCodes::NamespaceNotFound, + str::stream() << "Collection '" << _nss << "' does not exist to validate."); + } else { + // Uses the bucket collection in place of the time-series collection view. + if (!view->timeseries()) { + uasserted(ErrorCodes::CommandNotSupportedOnView, "Cannot validate a view"); + } + _nss = _nss.makeTimeseriesBucketsNamespace(); + if (isBackground()) { + _collectionLock.emplace(opCtx, _nss, MODE_IS); + } else { + _collectionLock.emplace(opCtx, _nss, MODE_X); + } + _collection = CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, _nss); } - - uasserted(ErrorCodes::NamespaceNotFound, - str::stream() << "Collection '" << _nss << "' does not exist to validate."); } - // Return warnings instead of errors on schema validation failures. - _warnOnSchemaValidation = additionalOptions.warnOnSchemaValidation; + _validationVersion = additionalOptions.validationVersion; // RepairMode is incompatible with the ValidateModes kBackground and // kForegroundFullEnforceFastCount. diff --git a/src/mongo/db/catalog/validate_state.h b/src/mongo/db/catalog/validate_state.h index 42bc8660e33..bb8f431d2d6 100644 --- a/src/mongo/db/catalog/validate_state.h +++ b/src/mongo/db/catalog/validate_state.h @@ -93,6 +93,13 @@ public: _collectionSchemaViolated = true; } + bool isTimeseriesDataInconsistent() { + return _timeseriesDataInconsistency; + } + void setTimeseriesDataInconsistent() { + _timeseriesDataInconsistency = true; + } + bool fixErrors() const { return _repairMode == RepairMode::kFixErrors; } @@ -168,8 +175,8 @@ public: return _logDiagnostics; } - bool warnOnSchemaValidation() const { - return _warnOnSchemaValidation; + ValidationVersion validationVersion() const { + return _validationVersion; } boost::optional<Timestamp> getValidateTimestamp() { @@ -214,7 +221,8 @@ private: ValidateMode _mode; RepairMode _repairMode; bool _collectionSchemaViolated = false; - bool _warnOnSchemaValidation = false; + bool _timeseriesDataInconsistency = false; + ValidationVersion _validationVersion = currentValidationVersion; boost::optional<ShouldNotConflictWithSecondaryBatchApplicationBlock> _noPBWM; boost::optional<Lock::GlobalLock> _globalLock; diff --git a/src/mongo/db/clientcursor.cpp b/src/mongo/db/clientcursor.cpp index 12aa1cedab0..ac60a6b14be 100644 --- a/src/mongo/db/clientcursor.cpp +++ b/src/mongo/db/clientcursor.cpp @@ -51,6 +51,7 @@ #include "mongo/db/cursor_server_params.h" #include "mongo/db/jsobj.h" #include "mongo/db/query/explain.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/util/background.h" @@ -81,6 +82,49 @@ static ServerStatusMetricField<Counter64> dCursorStatsTotalOpened("cursor.totalO static ServerStatusMetricField<Counter64> dCursorStatsMoreThanOneBatch( "cursor.moreThanOneBatch", &cursorStatsMoreThanOneBatch); +static Counter64 cursorStatsLifespanLessThan1Second; +static Counter64 cursorStatsLifespanLessThan5Seconds; +static Counter64 cursorStatsLifespanLessThan15Seconds; +static Counter64 cursorStatsLifespanLessThan30Seconds; +static Counter64 cursorStatsLifespanLessThan1Minute; +static Counter64 cursorStatsLifespanLessThan10Minutes; +static Counter64 cursorStatsLifespanGreaterThanOrEqual10Minutes; + +static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan1Second( + "cursor.lifespan.lessThan1Second", &cursorStatsLifespanLessThan1Second); +static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan5Seconds( + "cursor.lifespan.lessThan5Seconds", &cursorStatsLifespanLessThan5Seconds); +static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan15Seconds( + "cursor.lifespan.lessThan15Seconds", &cursorStatsLifespanLessThan15Seconds); +static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan30Seconds( + "cursor.lifespan.lessThan30Seconds", &cursorStatsLifespanLessThan30Seconds); +static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan1Minute( + "cursor.lifespan.lessThan1Minute", &cursorStatsLifespanLessThan1Minute); +static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan10Minutes( + "cursor.lifespan.lessThan10Minutes", &cursorStatsLifespanLessThan10Minutes); +static ServerStatusMetricField<Counter64> dCursorStatsLifespanGreaterThanOrEqual10Minutes( + "cursor.lifespan.greaterThanOrEqual10Minutes", &cursorStatsLifespanGreaterThanOrEqual10Minutes); + +void incrementCursorLifespanMetric(Date_t birth, Date_t death) { + auto elapsed = death - birth; + + if (elapsed < Seconds(1)) { + cursorStatsLifespanLessThan1Second.increment(); + } else if (elapsed < Seconds(5)) { + cursorStatsLifespanLessThan5Seconds.increment(); + } else if (elapsed < Seconds(15)) { + cursorStatsLifespanLessThan15Seconds.increment(); + } else if (elapsed < Seconds(30)) { + cursorStatsLifespanLessThan30Seconds.increment(); + } else if (elapsed < Minutes(1)) { + cursorStatsLifespanLessThan1Minute.increment(); + } else if (elapsed < Minutes(10)) { + cursorStatsLifespanLessThan10Minutes.increment(); + } else { + cursorStatsLifespanGreaterThanOrEqual10Minutes.increment(); + } +} + ClientCursor::ClientCursor(ClientCursorParams params, CursorId cursorId, OperationContext* operationUsingCursor, @@ -105,6 +149,10 @@ ClientCursor::ClientCursor(ClientCursorParams params, _planSummary(_exec->getPlanExplainer().getPlanSummary()), _planCacheKey(CurOp::get(operationUsingCursor)->debug().planCacheKey), _queryHash(CurOp::get(operationUsingCursor)->debug().queryHash), + _queryStatsKeyHash(CurOp::get(operationUsingCursor)->debug().queryStatsInfo.keyHash), + _queryStatsKey(std::move(CurOp::get(operationUsingCursor)->debug().queryStatsInfo.key)), + _queryStatsWillNeverExhaust( + CurOp::get(operationUsingCursor)->debug().queryStatsInfo.willNeverExhaust), _opKey(operationUsingCursor->getOperationKey()) { invariant(_exec); invariant(_operationUsingCursor); @@ -129,19 +177,33 @@ ClientCursor::~ClientCursor() { // needs to keep data pinned. _stashedRecoveryUnit->setAbandonSnapshotMode(RecoveryUnit::AbandonSnapshotMode::kAbort); } +} + +void ClientCursor::dispose(OperationContext* opCtx, boost::optional<Date_t> now) { + if (_disposed) { + return; + } + + query_stats::writeQueryStatsOnCursorDisposeOrKill( + opCtx, + _queryStatsKeyHash, + std::move(_queryStatsKey), + _queryStatsWillNeverExhaust, + _metrics.executionTime.value_or(Microseconds{0}).count(), + _firstResponseExecutionTime.value_or(Microseconds{0}).count(), + _metrics.nreturned.value_or(0)); + + if (now) { + incrementCursorLifespanMetric(_createdDate, *now); + } cursorStatsOpen.decrement(); if (isNoTimeout()) { cursorStatsOpenNoTimeout.decrement(); } - if (_nBatchesReturned > 1) + if (_metrics.nBatches && *_metrics.nBatches > 1) { cursorStatsMoreThanOneBatch.increment(); -} - -void ClientCursor::dispose(OperationContext* opCtx) { - if (_disposed) { - return; } _exec->dispose(opCtx); @@ -152,7 +214,7 @@ GenericCursor ClientCursor::toGenericCursor() const { GenericCursor gc; gc.setCursorId(cursorid()); gc.setNs(nss()); - gc.setNDocsReturned(_nReturnedSoFar); + gc.setNDocsReturned(_metrics.nreturned.value_or(0)); gc.setTailable(isTailable()); gc.setAwaitData(isAwaitData()); gc.setNoCursorTimeout(isNoTimeout()); @@ -265,24 +327,12 @@ void ClientCursorPin::deleteUnderlying() { invariant(_cursor); invariant(_cursor->_operationUsingCursor); invariant(_cursorManager); - // Note the following subtleties of this method's implementation: - // - We must unpin the cursor (by clearing the '_operationUsingCursor' field) before - // destruction, since it is an error to delete a pinned cursor. - // - In addition, we must deregister the cursor before clearing the '_operationUsingCursor' - // field, since it is an error to unpin a registered cursor without holding the appropriate - // cursor manager mutex. By first deregistering the cursor, we ensure that no other thread can - // access '_cursor', meaning that it is safe for us to write to '_operationUsingCursor' - // without holding the CursorManager mutex. - - _cursorManager->deregisterCursor(_cursor); - - // Make sure the cursor is disposed and unpinned before being destroyed. - _cursor->dispose(_opCtx); - _cursor->_operationUsingCursor = nullptr; - delete _cursor; - cursorStatsOpenPinned.decrement(); + std::unique_ptr<ClientCursor, ClientCursor::Deleter> ownedCursor(_cursor); _cursor = nullptr; + _cursorManager->deregisterAndDestroyCursor(_opCtx, std::move(ownedCursor)); + + cursorStatsOpenPinned.decrement(); _shouldSaveRecoveryUnit = false; } @@ -354,4 +404,37 @@ void startClientCursorMonitor() { getClientCursorMonitor(getGlobalServiceContext()).go(); } +void collectQueryStatsMongod(OperationContext* opCtx, ClientCursorPin& pinnedCursor) { + pinnedCursor->incrementCursorMetrics(CurOp::get(opCtx)->debug().additiveMetrics); + + // For a change stream query, we want to collect and update query stats on the initial query and + // for every getMore. + // TODO SERVER-89058 Modify comment to include tailable cursors. + if (pinnedCursor->getQueryStatsWillNeverExhaust()) { + auto& opDebug = CurOp::get(opCtx)->debug(); + + int64_t execTime = opDebug.additiveMetrics.executionTime.value_or(Microseconds{0}).count(); + query_stats::writeQueryStats(opCtx, + opDebug.queryStatsInfo.keyHash, + pinnedCursor->takeKey(), + execTime, + execTime, + opDebug.additiveMetrics.nreturned.value_or(0), + pinnedCursor->getQueryStatsWillNeverExhaust()); + } +} + +void collectQueryStatsMongod(OperationContext* opCtx, std::unique_ptr<query_stats::Key> key) { + // If we haven't registered a cursor to prepare for getMore requests, we record + // query stats directly. + auto& opDebug = CurOp::get(opCtx)->debug(); + int64_t execTime = opDebug.additiveMetrics.executionTime.value_or(Microseconds{0}).count(); + query_stats::writeQueryStats(opCtx, + opDebug.queryStatsInfo.keyHash, + std::move(key), + execTime, + execTime, + opDebug.additiveMetrics.nreturned.value_or(0)); +} + } // namespace mongo diff --git a/src/mongo/db/clientcursor.h b/src/mongo/db/clientcursor.h index 66b1438fdfb..6ac10c21d86 100644 --- a/src/mongo/db/clientcursor.h +++ b/src/mongo/db/clientcursor.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/bson/bsonobj.h" #include <boost/optional.hpp> #include <functional> @@ -36,6 +37,7 @@ #include "mongo/db/api_parameters.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/auth/user_name.h" +#include "mongo/db/curop.h" #include "mongo/db/cursor_id.h" #include "mongo/db/jsobj.h" #include "mongo/db/logical_session_id.h" @@ -160,6 +162,13 @@ public: ReadPreferenceSetting getReadPreferenceSetting() const { return _readPreferenceSetting; } + bool getQueryStatsWillNeverExhaust() const { + return _queryStatsWillNeverExhaust; + } + + std::unique_ptr<query_stats::Key> takeKey() { + return std::move(_queryStatsKey); + } /** * Returns a pointer to the underlying query plan executor. All cursors manage a PlanExecutor, @@ -205,29 +214,28 @@ public: * Increments the cursor's tracked number of query results returned so far by 'n'. */ void incNReturnedSoFar(std::uint64_t n) { - _nReturnedSoFar += n; + _metrics.incrementNreturned(n); } - /** - * Sets the cursor's tracked number of query results returned so far to 'n'. - */ - void setNReturnedSoFar(std::uint64_t n) { - invariant(n >= _nReturnedSoFar); - _nReturnedSoFar = n; + void incrementCursorMetrics(OpDebug::AdditiveMetrics newMetrics) { + _metrics.add(newMetrics); + if (!_firstResponseExecutionTime) { + _firstResponseExecutionTime = _metrics.executionTime; + } } /** * Returns the number of batches returned by this cursor so far. */ std::uint64_t getNBatches() const { - return _nBatchesReturned; + return _metrics.nBatches.value_or(0); } /** * Increments the number of batches returned so far by one. */ void incNBatches() { - ++_nBatchesReturned; + _metrics.incrementNBatches(); } Date_t getLastUseDate() const { @@ -353,9 +361,10 @@ private: /** * Disposes this ClientCursor's PlanExecutor. Must be called before deleting a ClientCursor to * ensure it has a chance to clean up any resources it is using. Can be called multiple times. - * It is an error to call any other method after calling dispose(). + * It is an error to call any other method after calling dispose(). If 'now' is specified, + * will track cursor lifespan metrics. */ - void dispose(OperationContext* opCtx); + void dispose(OperationContext* opCtx, boost::optional<Date_t> now); bool isNoTimeout() const { return _isNoTimeout; @@ -389,13 +398,6 @@ private: // an error to use a ClientCursor once it has been disposed. bool _disposed = false; - // Tracks the number of results returned by this cursor so far. Tracked only as debugging info - // for display in $currentOp output. - std::uint64_t _nReturnedSoFar = 0; - - // Tracks the number of batches returned by this cursor so far. - std::uint64_t _nBatchesReturned = 0; - // Holds an owned copy of the command specification received from the client. const BSONObj _originatingCommand; @@ -450,11 +452,25 @@ private: boost::optional<uint32_t> _planCacheKey; boost::optional<uint32_t> _queryHash; + // If boost::none, query stats should not be collected for this cursor. + boost::optional<std::size_t> _queryStatsKeyHash; + // Metrics that are accumulated over the lifetime of the cursor, incremented with each getMore. + // Useful for diagnostics like queryStats. + OpDebug::AdditiveMetrics _metrics; + // The Key used by query stats to generate the query stats store key. + std::unique_ptr<query_stats::Key> _queryStatsKey; + + // Flag for query stats on if the current cursor is used for a tailable or change stream query. + bool _queryStatsWillNeverExhaust{false}; + // The client OperationKey associated with this cursor. boost::optional<OperationKey> _opKey; // Flag indicating that a client has requested to kill the cursor. bool _killPending = false; + + // The execution time collected from the initial operation prior to any getMore requests. + boost::optional<Microseconds> _firstResponseExecutionTime; }; /** @@ -562,4 +578,17 @@ private: void startClientCursorMonitor(); + +/** + * Records certain metrics for the current operation on OpDebug and aggregates those metrics for + * query stats use. If a cursor pin is provided, metrics are aggregated on the cursor; otherwise, + * metrics are written directly to the query stats store. + * NOTE: Metrics are taken from opDebug.additiveMetrics, so CurOp::setEndOfOpMetrics must be called + * *prior* to calling these. + * + * Currently, query stats are only collected for find and aggregate requests (and their subsequent + * getMore requests), so these should only be called from those request paths. + */ +void collectQueryStatsMongod(OperationContext* opCtx, ClientCursorPin& cursor); +void collectQueryStatsMongod(OperationContext* opCtx, std::unique_ptr<query_stats::Key> key); } // namespace mongo diff --git a/src/mongo/db/coll_mod.idl b/src/mongo/db/coll_mod.idl index 10f29489aad..b57eef9744f 100644 --- a/src/mongo/db/coll_mod.idl +++ b/src/mongo/db/coll_mod.idl @@ -184,6 +184,13 @@ structs: optional: true type: bool unstable: false + timeseriesBucketsMayHaveMixedSchemaData: + description: >- + Whether the time-series collection may have mixed schema data, which was only + possible to write in FCV 5.1 and earlier. + optional: true + type: safeBool + unstable: true dryRun: description: "Runs the requested modification without modifying any database state. This can be used to determine in advance if a particular collMod diff --git a/src/mongo/db/collection_index_usage_tracker_test.cpp b/src/mongo/db/collection_index_usage_tracker_test.cpp index a532e91a152..ed428331208 100644 --- a/src/mongo/db/collection_index_usage_tracker_test.cpp +++ b/src/mongo/db/collection_index_usage_tracker_test.cpp @@ -237,12 +237,14 @@ TEST_F(CollectionIndexUsageTrackerTest, GlobalFeatureUsageBasic) { ASSERT_EQ(1, getGlobalIndexUsage()->getCount()); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "unique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "unique")); @@ -255,12 +257,14 @@ TEST_F(CollectionIndexUsageTrackerTest, GlobalFeatureUsageBasic) { ASSERT_EQ(2, getGlobalIndexUsage()->getCount()); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "unique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "unique")); @@ -271,12 +275,14 @@ TEST_F(CollectionIndexUsageTrackerTest, GlobalFeatureUsageBasic) { ASSERT_EQ(2, getGlobalIndexUsage()->getCount()); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "unique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "unique")); @@ -285,12 +291,14 @@ TEST_F(CollectionIndexUsageTrackerTest, GlobalFeatureUsageBasic) { ASSERT_EQ(2, getGlobalIndexUsage()->getCount()); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "unique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "unique")); @@ -299,12 +307,14 @@ TEST_F(CollectionIndexUsageTrackerTest, GlobalFeatureUsageBasic) { ASSERT_EQ(1, getGlobalIndexUsage()->getCount()); ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "unique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "unique")); @@ -313,16 +323,63 @@ TEST_F(CollectionIndexUsageTrackerTest, GlobalFeatureUsageBasic) { ASSERT_EQ(0, getGlobalIndexUsage()->getCount()); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "unique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "id")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "prepareUnique")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "single")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "sparse")); ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "unique")); } +// Unregister and re-register an index with prepareUnique +TEST_F(CollectionIndexUsageTrackerTest, RegisterPrepareUnique) { + auto spec = BSON("key" << BSON("foo" << 1) << "v" << 2); + auto desc = IndexDescriptor("", spec); + getTracker()->registerIndex("foo", spec, IndexFeatures::make(&desc, false /* internal */)); + getTracker()->recordIndexAccess("foo"); + + ASSERT_EQ(1, getGlobalIndexUsage()->getCount()); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "id")); + ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "prepareUnique")); + ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "single")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "sparse")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "unique")); + + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "id")); + ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "prepareUnique")); + ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "single")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "sparse")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "unique")); + + // Unregister an re-register with different options. + getTracker()->unregisterIndex("foo"); + auto spec2 = BSON("key" << BSON("foo" << 1) << "prepareUnique" << true << "v" << 2); + auto desc2 = IndexDescriptor("", spec2); + getTracker()->registerIndex("foo", spec2, IndexFeatures::make(&desc2, false /* internal */)); + getTracker()->recordIndexAccess("foo"); + + ASSERT_EQ(1, getGlobalIndexUsage()->getCount()); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "id")); + ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "prepareUnique")); + ASSERT_EQ(1, getFeatureUseCount(getGlobalIndexUsage(), "single")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "sparse")); + ASSERT_EQ(0, getFeatureUseCount(getGlobalIndexUsage(), "unique")); + + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "id")); + ASSERT_EQ(2, getFeatureAccessCount(getGlobalIndexUsage(), "normal")); + ASSERT_EQ(1, getFeatureAccessCount(getGlobalIndexUsage(), "prepareUnique")); + ASSERT_EQ(2, getFeatureAccessCount(getGlobalIndexUsage(), "single")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "sparse")); + ASSERT_EQ(0, getFeatureAccessCount(getGlobalIndexUsage(), "unique")); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/collection_type.h b/src/mongo/db/collection_type.h new file mode 100644 index 00000000000..0c58804bf2a --- /dev/null +++ b/src/mongo/db/collection_type.h @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/util/assert_util.h" + +namespace mongo::query_shape { + +// This enum is not compatible with the QUERY_UTIL_NAMED_ENUM_DEFINE util since the "virtual" type +// conflicts with the C++ keyword "virtual". Instead, we manually define the enum and the +// toStringData function below. +enum class CollectionType { + kUnknown, + kCollection, + kView, + kTimeseries, + kChangeStream, + kVirtual, + kNonExistent, +}; + +static StringData toStringData(CollectionType type) { + switch (type) { + case CollectionType::kUnknown: + return "unknown"_sd; + case CollectionType::kCollection: + return "collection"_sd; + case CollectionType::kView: + return "view"_sd; + case CollectionType::kTimeseries: + return "timeseries"_sd; + case CollectionType::kChangeStream: + return "changeStream"_sd; + case CollectionType::kVirtual: + return "virtual"_sd; + case CollectionType::kNonExistent: + return "nonExistent"_sd; + default: + MONGO_UNREACHABLE_TASSERT(7804900); + } +} + +} // namespace mongo::query_shape diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp index 3ec877b7d4c..646578c855d 100644 --- a/src/mongo/db/commands.cpp +++ b/src/mongo/db/commands.cpp @@ -205,7 +205,6 @@ void CommandHelpers::runCommandInvocation(OperationContext* opCtx, CommandInvocation* invocation, rpc::ReplyBuilderInterface* response) { auto&& hooks = getCommandInvocationHooks(opCtx->getServiceContext()); - if (hooks) { hooks->onBeforeRun(opCtx, request, invocation); } diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript index 9eae99e2838..d5210d9603c 100644 --- a/src/mongo/db/commands/SConscript +++ b/src/mongo/db/commands/SConscript @@ -232,8 +232,8 @@ env.Library( '$BUILD_DIR/mongo/db/auth/authprivilege', '$BUILD_DIR/mongo/db/commands', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/dbdirectclient', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/storage/backup_cursor_hooks', 'fsync_locked', ] @@ -387,7 +387,8 @@ env.Library( '$BUILD_DIR/mongo/db/query/ce/query_ce', '$BUILD_DIR/mongo/db/query/command_request_response', '$BUILD_DIR/mongo/db/query/cursor_response_idl', - '$BUILD_DIR/mongo/db/query/optimizer/optimizer', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query_exec', '$BUILD_DIR/mongo/db/repl/replica_set_messages', '$BUILD_DIR/mongo/db/repl/tenant_migration_access_blocker', @@ -496,6 +497,7 @@ env.Library( 'shutdown.idl', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/idl/idl_parser', '$BUILD_DIR/mongo/util/fail_point', ], @@ -672,8 +674,8 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/auth/auth', '$BUILD_DIR/mongo/db/auth/authprivilege', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/matcher/expressions', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/server_options_core', ], ) diff --git a/src/mongo/db/commands/compact.cpp b/src/mongo/db/commands/compact.cpp index 26ee8ab5d4b..48d899efcd1 100644 --- a/src/mongo/db/commands/compact.cpp +++ b/src/mongo/db/commands/compact.cpp @@ -27,23 +27,34 @@ * it in the license file. */ +#include <absl/container/btree_set.h> #include <string> #include <vector> #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_compact.h" #include "mongo/db/catalog/database.h" #include "mongo/db/commands.h" #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/curop.h" +#include "mongo/db/db_raii.h" #include "mongo/db/jsobj.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/repl/member_state.h" #include "mongo/db/repl/replication_coordinator.h" namespace mongo { +namespace { +static absl::btree_set<UUID> compactsRunning; +} // namespace + using std::string; using std::stringstream; @@ -65,6 +76,7 @@ public: actions.addAction(ActionType::compact); out->push_back(Privilege(parseResourcePattern(dbname, cmdObj), actions)); } + std::string help() const override { return "compact collection\n" "warning: this operation locks the database and is slow. you can cancel with " @@ -75,36 +87,59 @@ public: CompactCmd() : ErrmsgCommandDeprecated("compact") {} virtual bool errmsgRun(OperationContext* opCtx, - const string& db, + const std::string& dbName, const BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result) { - NamespaceString nss = CommandHelpers::parseNsCollectionRequired(db, cmdObj); - - repl::ReplicationCoordinator* replCoord = repl::ReplicationCoordinator::get(opCtx); - if (replCoord->getMemberState().primary() && !cmdObj["force"].trueValue()) { - errmsg = - "will not run compact on an active replica set primary as this is a slow blocking " - "operation. use force:true to force"; - return false; - } + NamespaceString collectionNss = CommandHelpers::parseNsCollectionRequired(dbName, cmdObj); + + Lock::GlobalLock lk(opCtx, + MODE_IX, + Date_t::max(), + Lock::InterruptBehavior::kThrow, + /*skipRSTLLock=*/true); + + // Hold reference to the catalog for collection lookup without locks to be safe. + auto collectionCatalog = CollectionCatalog::get(opCtx); - if (nss.isSystem()) { - // Items in system.* cannot be moved as there might be pointers to them. - errmsg = "can't compact a system namespace"; - return false; + CollectionPtr collection = [&]() { + if (CollectionPtr collection = CollectionPtr( + collectionCatalog->lookupCollectionByNamespace(opCtx, collectionNss))) { + return collection; + } + + // Check if this is a time-series collection. + auto bucketsNs = collectionNss.makeTimeseriesBucketsNamespace(); + if (CollectionPtr collection = CollectionPtr( + collectionCatalog->lookupCollectionByNamespace(opCtx, bucketsNs))) { + return collection; + } + + return CollectionPtr(); + }(); + + if (!collection) { + std::shared_ptr<const ViewDefinition> view = + collectionCatalog->lookupView(opCtx, collectionNss); + uassert(ErrorCodes::CommandNotSupportedOnView, "can't compact a view", !view); + uasserted(ErrorCodes::NamespaceNotFound, "collection does not exist"); } - // This command is internal to the storage engine and should not block oplog application. - ShouldNotConflictWithSecondaryBatchApplicationBlock noPBWMBlock(opCtx->lockState()); + AutoStatsTracker statsTracker(opCtx, + collectionNss, + Top::LockType::NotLocked, + AutoStatsTracker::LogMode::kUpdateTopAndCurOp, + collectionCatalog->getDatabaseProfileLevel(dbName)); + + StatusWith<int64_t> status = compactCollection(opCtx, collection); - StatusWith<int64_t> status = compactCollection(opCtx, nss); uassertStatusOK(status.getStatus()); int64_t bytesFreed = status.getValue(); if (bytesFreed < 0) { - // When compacting a collection that is actively being written to, it is possible that - // the collection is larger at the completion of compaction than when it started. + // When compacting a collection that is actively being written to, it is possible + // that the collection is larger at the completion of compaction than when it + // started. bytesFreed = 0; } diff --git a/src/mongo/db/commands/count_cmd.cpp b/src/mongo/db/commands/count_cmd.cpp index 7fb22de4c0f..57291c7b782 100644 --- a/src/mongo/db/commands/count_cmd.cpp +++ b/src/mongo/db/commands/count_cmd.cpp @@ -182,12 +182,8 @@ public: // An empty PrivilegeVector is acceptable because these privileges are only checked on // getMore and explain will not open a cursor. - return runAggregate(opCtx, - viewAggRequest.getNamespace(), - viewAggRequest, - viewAggregation.getValue(), - PrivilegeVector(), - result); + return runAggregate( + opCtx, viewAggRequest, viewAggregation.getValue(), PrivilegeVector(), result); } const auto& collection = ctx->getCollection(); @@ -236,6 +232,8 @@ public: &hangBeforeCollectionCount, opCtx, "hangBeforeCollectionCount", []() {}, nss); auto request = CountCommandRequest::parse(IDLParserErrorContext("count"), cmdObj); + auto curOp = CurOp::get(opCtx); + curOp->beginQueryPlanningTimer(); if (shouldDoFLERewrite(request)) { processFLECountD(opCtx, nss, &request); } @@ -285,7 +283,6 @@ public: auto exec = std::move(statusWithPlanExecutor.getValue()); // Store the plan summary string in CurOp. - auto curOp = CurOp::get(opCtx); { stdx::lock_guard<Client> lk(*opCtx->getClient()); curOp->setPlanSummary_inlock(exec->getPlanExplainer().getPlanSummary()); diff --git a/src/mongo/db/commands/current_op.cpp b/src/mongo/db/commands/current_op.cpp index 78fec805202..b4a16bc40cb 100644 --- a/src/mongo/db/commands/current_op.cpp +++ b/src/mongo/db/commands/current_op.cpp @@ -77,12 +77,7 @@ public: privileges = {Privilege(ResourcePattern::forClusterResource(), ActionType::inprog)}; } - auto status = runAggregate(opCtx, - request.getNamespace(), - request, - std::move(aggCmdObj), - privileges, - &replyBuilder); + auto status = runAggregate(opCtx, request, std::move(aggCmdObj), privileges, &replyBuilder); if (!status.isOK()) { return status; diff --git a/src/mongo/db/commands/dbcheck.cpp b/src/mongo/db/commands/dbcheck.cpp index ef4a566bc8f..5d4579691dd 100644 --- a/src/mongo/db/commands/dbcheck.cpp +++ b/src/mongo/db/commands/dbcheck.cpp @@ -53,6 +53,8 @@ #include "mongo/logv2/log.h" MONGO_FAIL_POINT_DEFINE(sleepAfterExtraIndexKeysHashing); +MONGO_FAIL_POINT_DEFINE(hangBeforeProcessingDbCheckRun); +MONGO_FAIL_POINT_DEFINE(hangBeforeAddingDBCheckBatchToOplog); namespace mongo { @@ -288,6 +290,11 @@ protected: DbCheckStartAndStopLogger startStop(opCtx); + if (MONGO_unlikely(hangBeforeProcessingDbCheckRun.shouldFail())) { + LOGV2(7949000, "Hanging dbcheck due to failpoint 'hangBeforeProcessingDbCheckRun'"); + hangBeforeProcessingDbCheckRun.pauseWhileSet(); + } + for (const auto& coll : *_run) { try { _doCollection(opCtx, coll); @@ -440,11 +447,14 @@ private: WriteConcernResult unused; auto status = waitForWriteConcern(opCtx, stats.time, info.writeConcern, &unused); if (!status.isOK()) { - auto entry = dbCheckWarningHealthLogEntry(info.nss, - "dbCheck failed waiting for writeConcern", - OplogEntriesEnum::Batch, - status); + // TODO SERVER-89817: Add context with batch ID and lastKey once those are + // backported. + auto entry = dbCheckErrorHealthLogEntry(info.nss, + "dbCheck failed waiting for writeConcern", + OplogEntriesEnum::Batch, + status); HealthLogInterface::get(opCtx)->log(*entry); + return; } start = stats.lastKey; @@ -575,6 +585,12 @@ private: batch.setMaxKey(BSONKey(hasher->lastKey())); batch.setReadTimestamp(readTimestamp); + if (MONGO_unlikely(hangBeforeAddingDBCheckBatchToOplog.shouldFail())) { + LOGV2(8589000, + "Hanging dbCheck due to failpoint 'hangBeforeAddingDBCheckBatchToOplog'"); + hangBeforeAddingDBCheckBatchToOplog.pauseWhileSet(); + } + // Send information on this batch over the oplog. result.time = _logOp(opCtx, info.nss, collection->uuid(), batch.toBSON()); result.readTimestamp = readTimestamp; diff --git a/src/mongo/db/commands/distinct.cpp b/src/mongo/db/commands/distinct.cpp index f642ba2f49d..ef08e46aee7 100644 --- a/src/mongo/db/commands/distinct.cpp +++ b/src/mongo/db/commands/distinct.cpp @@ -172,7 +172,7 @@ public: // An empty PrivilegeVector is acceptable because these privileges are only checked on // getMore and explain will not open a cursor. return runAggregate( - opCtx, nss, viewAggRequest, viewAggregation.getValue(), PrivilegeVector(), result); + opCtx, viewAggRequest, viewAggregation.getValue(), PrivilegeVector(), result); } const auto& collection = ctx->getCollection(); diff --git a/src/mongo/db/commands/drop_indexes.cpp b/src/mongo/db/commands/drop_indexes.cpp index 8e0a48f013b..a2c773af89d 100644 --- a/src/mongo/db/commands/drop_indexes.cpp +++ b/src/mongo/db/commands/drop_indexes.cpp @@ -236,8 +236,8 @@ public: collection.getWritableCollection()->getIndexCatalog()->dropAllIndexes( opCtx, collection.getWritableCollection(), true, {}); - swIndexesToRebuild = - indexer->init(opCtx, collection, all, MultiIndexBlock::kNoopOnInitFn); + swIndexesToRebuild = indexer->init( + opCtx, collection, all, MultiIndexBlock::kNoopOnInitFn, /*forRecovery=*/false); uassertStatusOK(swIndexesToRebuild.getStatus()); wunit.commit(); }); diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index 9aaaa2b48c3..d94346896da 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -171,6 +171,7 @@ void makeUpdateRequest(OperationContext* opCtx, requestOut->setExplain(explain); requestOut->setYieldPolicy(PlanYieldPolicy::YieldPolicy::YIELD_AUTO); + requestOut->setBypassEmptyTsReplacement(request.getBypassEmptyTsReplacement()); } void makeDeleteRequest(OperationContext* opCtx, diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index 48dee5c1e7d..df96e09e2ce 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -36,6 +36,7 @@ #include "mongo/db/catalog/collection_uuid_mismatch.h" #include "mongo/db/client.h" #include "mongo/db/clientcursor.h" +#include "mongo/db/collection_type.h" #include "mongo/db/commands.h" #include "mongo/db/commands/run_aggregate.h" #include "mongo/db/commands/test_commands_enabled.h" @@ -54,6 +55,10 @@ #include "mongo/db/query/find_common.h" #include "mongo/db/query/get_executor.h" #include "mongo/db/query/query_knobs_gen.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_stats/find_key.h" +#include "mongo/db/query/query_stats/key.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/service_context.h" #include "mongo/db/stats/counters.h" @@ -111,25 +116,6 @@ std::unique_ptr<FindCommandRequest> translateNtoReturnToLimitOrBatchSize( return findCmd; } -// Parses the command object to a FindCommandRequest. If the client request did not specify any -// runtime constants, make them available to the query here. -std::unique_ptr<FindCommandRequest> parseCmdObjectToFindCommandRequest(OperationContext* opCtx, - NamespaceString nss, - BSONObj cmdObj) { - auto findCommand = query_request_helper::makeFromFindCommand( - std::move(cmdObj), - std::move(nss), - APIParameters::get(opCtx).getAPIStrict().value_or(false)); - - // Rewrite any FLE find payloads that exist in the query if this is a FLE 2 query. - if (shouldDoFLERewrite(findCommand)) { - invariant(findCommand->getNamespaceOrUUID().nss()); - processFLEFindD(opCtx, findCommand->getNamespaceOrUUID().nss().get(), findCommand.get()); - } - - return translateNtoReturnToLimitOrBatchSize(std::move(findCommand)); -} - boost::intrusive_ptr<ExpressionContext> makeExpressionContext( OperationContext* opCtx, const FindCommandRequest& findCommand, @@ -145,38 +131,13 @@ boost::intrusive_ptr<ExpressionContext> makeExpressionContext( // ExpressionContext. collator = collPtr->getDefaultCollator()->clone(); } - - // Although both 'find' and 'aggregate' commands have an ExpressionContext, some of the data - // members in the ExpressionContext are used exclusively by the aggregation subsystem. This - // includes the following fields which here we simply initialize to some meaningless default - // value: - // - explain - // - fromMongos - // - needsMerge - // - bypassDocumentValidation - // - mongoProcessInterface - // - resolvedNamespaces - // - uuid - // - // As we change the code to make the find and agg systems more tightly coupled, it would make - // sense to start initializing these fields for find operations as well. - auto expCtx = make_intrusive<ExpressionContext>( - opCtx, - verbosity, - false, // fromMongos - false, // needsMerge - findCommand.getAllowDiskUse().value_or(allowDiskUseByDefault.load()), - false, // bypassDocumentValidation - false, // isMapReduceCommand - findCommand.getNamespaceOrUUID().nss().value_or(NamespaceString()), - findCommand.getLegacyRuntimeConstants(), - std::move(collator), - nullptr, // mongoProcessInterface - StringMap<ExpressionContext::ResolvedNamespace>{}, - boost::none, // uuid - findCommand.getLet(), // let - CurOp::get(opCtx)->dbProfileLevel() > 0 // mayDbProfile - ); + auto expCtx = + make_intrusive<ExpressionContext>(opCtx, + findCommand, + std::move(collator), + CurOp::get(opCtx)->dbProfileLevel() > 0, // mayDbProfile + verbosity, + allowDiskUseByDefault.load()); if (storageGlobalParams.readOnly) { // Disallow disk use if in read-only mode. expCtx->allowDiskUse = false; @@ -198,6 +159,45 @@ void beginQueryOp(OperationContext* opCtx, const NamespaceString& nss, const BSO } /** + * Parses the grammar elements like 'filter', 'sort', and 'projection' from the raw + * 'FindCommandRequest', and tracks internal state like begining the operation's timer and recording + * query shape stats (if enabled). + */ +std::unique_ptr<CanonicalQuery> parseQueryAndBeginOperation( + OperationContext* opCtx, + const AutoGetCollectionForReadCommandMaybeLockFree& ctx, + const NamespaceString& nss, + BSONObj requestBody, + std::unique_ptr<FindCommandRequest> findCommand, + const CollectionPtr& collection) { + // Fill out curop information. + beginQueryOp(opCtx, nss, requestBody); + // Finish the parsing step by using the FindCommandRequest to create a CanonicalQuery. + const ExtensionsCallbackReal extensionsCallback(opCtx, &nss); + + auto expCtx = + makeExpressionContext(opCtx, *findCommand, collection, boost::none /* verbosity */); + + auto parsedRequest = uassertStatusOK( + parsed_find_command::parse(expCtx, + std::move(findCommand), + extensionsCallback, + MatchExpressionParser::kAllowAllSpecialFeatures)); + + // Register query stats collection. Exclude queries against collections with encrypted fields. + // It is important to do this before canonicalizing and optimizing the query, each of which + // would alter the query shape. + if (!(collection && collection.get()->getCollectionOptions().encryptedFieldConfig)) { + query_stats::registerRequest(opCtx, nss, [&]() { + return std::make_unique<query_stats::FindKey>( + expCtx, *parsedRequest, ctx.getCollectionType()); + }); + } + + return uassertStatusOK( + CanonicalQuery::canonicalize(std::move(expCtx), std::move(parsedRequest))); +} +/** * A command for running .find() queries. */ class FindCmd final : public Command { @@ -322,7 +322,7 @@ public: const auto nss = ctx->getNss(); // Parse the command BSON to a FindCommandRequest. - auto findCommand = parseCmdObjectToFindCommandRequest(opCtx, nss, _request.body); + auto findCommand = _parseCmdObjectToFindCommandRequest(opCtx, nss, _request.body); // Finish the parsing step by using the FindCommandRequest to create a CanonicalQuery. const ExtensionsCallbackReal extensionsCallback(opCtx, &nss); @@ -372,8 +372,8 @@ public: try { // An empty PrivilegeVector is acceptable because these privileges are only // checked on getMore and explain will not open a cursor. - uassertStatusOK(runAggregate( - opCtx, nss, aggRequest, viewAggCmd, PrivilegeVector(), result)); + uassertStatusOK( + runAggregate(opCtx, aggRequest, viewAggCmd, PrivilegeVector(), result)); } catch (DBException& error) { if (error.code() == ErrorCodes::InvalidPipelineOperator) { uasserted(ErrorCodes::InvalidPipelineOperator, @@ -419,10 +419,10 @@ public: // Parse the command BSON to a FindCommandRequest. Pass in the parsedNss in case cmdObj // does not have a UUID. auto parsedNss = NamespaceString{CommandHelpers::parseNsFromCommand(_dbName, cmdObj)}; - const bool isExplain = false; const bool isOplogNss = (parsedNss == NamespaceString::kRsOplogNamespace); auto findCommand = - parseCmdObjectToFindCommandRequest(opCtx, std::move(parsedNss), cmdObj); + _parseCmdObjectToFindCommandRequest(opCtx, std::move(parsedNss), cmdObj); + CurOp::get(opCtx)->beginQueryPlanningTimer(); // Only allow speculative majority for internal commands that specify the correct flag. uassert(ErrorCodes::ReadConcernMajorityNotEnabled, @@ -542,21 +542,8 @@ public: findCommand->getResumeAfter(), isClusteredCollection)); } - // Fill out curop information. - beginQueryOp(opCtx, nss, _request.body); - - // Finish the parsing step by using the FindCommandRequest to create a CanonicalQuery. - const ExtensionsCallbackReal extensionsCallback(opCtx, &nss); - - auto expCtx = - makeExpressionContext(opCtx, *findCommand, collection, boost::none /* verbosity */); - auto cq = uassertStatusOK( - CanonicalQuery::canonicalize(opCtx, - std::move(findCommand), - isExplain, - std::move(expCtx), - extensionsCallback, - MatchExpressionParser::kAllowAllSpecialFeatures)); + auto cq = parseQueryAndBeginOperation( + opCtx, *ctx, nss, _request.body, std::move(findCommand), collection); // If we are running a query against a view, or if we are trying to test the new // optimizer, redirect this query through the aggregation system. @@ -573,6 +560,9 @@ public: auto viewAggregationCommand = uassertStatusOK(query_request_helper::asAggregationCommand(findCommand)); + // This doesn't directly call 'runAggregate()' so it doesn't need to adapt to the + // new API on v6.0. @Alyssa this suggests we should look into view performance more + // carefully on v6.0. The perf of this code path may have different characteristics? BSONObj aggResult = CommandHelpers::runCommandDirectly( opCtx, OpMsgRequest::fromDBAndBody(_dbName, std::move(viewAggregationCommand))); auto status = getStatusFromCommandResult(aggResult); @@ -626,7 +616,7 @@ public: // there is no ClientCursor id, and then return. const long long numResults = 0; const CursorId cursorId = 0; - endQueryOp(opCtx, collection, *exec, numResults, cursorId); + endQueryOp(opCtx, collection, *exec, numResults, boost::none, cmdObj); auto bodyBuilder = result->getBodyBuilder(); appendCursorResponseObject( cursorId, nss.ns(), BSONArray(), boost::none, &bodyBuilder); @@ -725,11 +715,9 @@ public: pinnedCursor.getCursor()->setLeftoverMaxTimeMicros( opCtx->getRemainingMaxTimeMicros()); } - pinnedCursor.getCursor()->setNReturnedSoFar(numResults); - pinnedCursor.getCursor()->incNBatches(); // Fill out curop based on the results. - endQueryOp(opCtx, collection, *cursorExec, numResults, cursorId); + endQueryOp(opCtx, collection, *cursorExec, numResults, pinnedCursor, cmdObj); if (stashResourcesForGetMore) { // Collect storage stats now before we stash the recovery unit. These stats are @@ -744,7 +732,7 @@ public: opCtx->recoveryUnit()->computeOperationStatisticsSinceLastCall(); } } else { - endQueryOp(opCtx, collection, *exec, numResults, cursorId); + endQueryOp(opCtx, collection, *exec, numResults, boost::none, cmdObj); } // Generate the response object to send to the client. @@ -785,6 +773,25 @@ public: private: const OpMsgRequest _request; const StringData _dbName; + + // Parses the command object to a FindCommandRequest. If the client request did not specify + // any runtime constants, make them available to the query here. + std::unique_ptr<FindCommandRequest> _parseCmdObjectToFindCommandRequest( + OperationContext* opCtx, NamespaceString nss, BSONObj cmdObj) { + auto findCommand = query_request_helper::makeFromFindCommand( + std::move(cmdObj), + std::move(nss), + APIParameters::get(opCtx).getAPIStrict().value_or(false)); + + // Rewrite any FLE find payloads that exist in the query if this is a FLE 2 query. + if (shouldDoFLERewrite(findCommand)) { + invariant(findCommand->getNamespaceOrUUID().nss()); + processFLEFindD( + opCtx, findCommand->getNamespaceOrUUID().nss().value(), findCommand.get()); + } + + return translateNtoReturnToLimitOrBatchSize(std::move(findCommand)); + } }; } findCmd; diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp index c699a3262e7..07df3bf5a9b 100644 --- a/src/mongo/db/commands/getmore_cmd.cpp +++ b/src/mongo/db/commands/getmore_cmd.cpp @@ -362,7 +362,7 @@ public: * be returned by this getMore. * * Returns true if the cursor should be saved for subsequent getMores, and false otherwise. - * Fills out *numResults with the number of documents in the batch, which must be + * Fills out 'numResults' with the number of documents in the batch, which must be * initialized to zero by the caller. * * Throws an exception on failure. @@ -385,9 +385,14 @@ public: try { while (!FindCommon::enoughForGetMore(batchSize, *numResults) && PlanExecutor::ADVANCED == (state = exec->getNext(&obj, nullptr))) { + auto nextPostBatchResumeToken = exec->getPostBatchResumeToken(); + // If adding this object will cause us to exceed the message size limit, then we // stash it for later. - if (!FindCommon::haveSpaceForNext(obj, *numResults, nextBatch->bytesUsed())) { + if (!FindCommon::haveSpaceForNext(obj, + *numResults, + nextBatch->bytesUsed() + + nextPostBatchResumeToken.objsize())) { exec->stashResult(obj); break; } @@ -396,7 +401,7 @@ public: awaitDataState(opCtx).shouldWaitForInserts = false; // If this executor produces a postBatchResumeToken, add it to the response. - nextBatch->setPostBatchResumeToken(exec->getPostBatchResumeToken()); + nextBatch->setPostBatchResumeToken(nextPostBatchResumeToken); // At this point, we know that there will be at least one document in this // batch. Reserve an initial estimated number of bytes for the response. @@ -719,12 +724,9 @@ public: // documents. auto& metricsCollector = ResourceConsumption::MetricsCollector::get(opCtx); metricsCollector.incrementDocUnitsReturned(docUnitsReturned); - cursorPin->incNReturnedSoFar(numResults); - cursorPin->incNBatches(); - - // Ensure log and profiler include the number of results returned in this getMore's - // response batch. - curOp->debug().nreturned = numResults; + curOp->debug().additiveMetrics.nBatches = 1; + curOp->setEndOfOpMetrics(numResults); + collectQueryStatsMongod(opCtx, cursorPin); if (respondWithId) { cursorDeleter.dismiss(); diff --git a/src/mongo/db/commands/map_reduce_agg.cpp b/src/mongo/db/commands/map_reduce_agg.cpp index eb1d432f5b5..91a27c93bf4 100644 --- a/src/mongo/db/commands/map_reduce_agg.cpp +++ b/src/mongo/db/commands/map_reduce_agg.cpp @@ -138,6 +138,8 @@ bool runAggregationMapReduce(OperationContext* opCtx, Timer cmdTimer; auto parsedMr = MapReduceCommandRequest::parse(IDLParserErrorContext("mapReduce"), cmd); + auto curop = CurOp::get(opCtx); + curop->beginQueryPlanningTimer(); auto expCtx = makeExpressionContext(opCtx, parsedMr, verbosity); auto runnablePipeline = [&]() { auto pipeline = map_reduce_common::translateFromMR(parsedMr, expCtx); @@ -146,10 +148,10 @@ bool runAggregationMapReduce(OperationContext* opCtx, }(); auto exec = plan_executor_factory::make(expCtx, std::move(runnablePipeline)); auto&& explainer = exec->getPlanExplainer(); - + // Store the plan summary string in CurOp. { stdx::lock_guard<Client> lk(*opCtx->getClient()); - CurOp::get(opCtx)->setPlanSummary_inlock(explainer.getPlanSummary()); + curop->setPlanSummary_inlock(explainer.getPlanSummary()); } try { diff --git a/src/mongo/db/commands/map_reduce_agg_test.cpp b/src/mongo/db/commands/map_reduce_agg_test.cpp index 52ee67c416b..b47b2bfa0e4 100644 --- a/src/mongo/db/commands/map_reduce_agg_test.cpp +++ b/src/mongo/db/commands/map_reduce_agg_test.cpp @@ -46,15 +46,6 @@ #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/unittest/unittest.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString(); \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace mongo { namespace { diff --git a/src/mongo/db/commands/pipeline_command.cpp b/src/mongo/db/commands/pipeline_command.cpp index decfcb35fe8..2042914e16b 100644 --- a/src/mongo/db/commands/pipeline_command.cpp +++ b/src/mongo/db/commands/pipeline_command.cpp @@ -142,7 +142,6 @@ public: opCtx, !Pipeline::aggHasWriteStage(_request.body)); uassertStatusOK(runAggregate(opCtx, - _aggregationRequest.getNamespace(), _aggregationRequest, _liteParsedPipeline, _request.body, @@ -165,7 +164,6 @@ public: rpc::ReplyBuilderInterface* result) override { uassertStatusOK(runAggregate(opCtx, - _aggregationRequest.getNamespace(), _aggregationRequest, _liteParsedPipeline, _request.body, diff --git a/src/mongo/db/commands/run_aggregate.cpp b/src/mongo/db/commands/run_aggregate.cpp index b47645610f7..66d68f4413c 100644 --- a/src/mongo/db/commands/run_aggregate.cpp +++ b/src/mongo/db/commands/run_aggregate.cpp @@ -74,6 +74,9 @@ #include "mongo/db/query/query_feature_flags_gen.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/db/query/query_planner_common.h" +#include "mongo/db/query/query_stats/agg_key.h" +#include "mongo/db/query/query_stats/key.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/read_concern.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/read_concern_args.h" @@ -627,7 +630,6 @@ std::vector<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> createLegacyEx getSearchHelpers(expCtx->opCtx->getServiceContext()) ->injectSearchShardFiltererIfNeeded(pipeline.get()); - // Complete creation of the initial $cursor stage, if needed. PipelineD::attachInnerQueryExecutorToPipeline(collections, attachExecutorCallback.first, @@ -640,7 +642,6 @@ std::vector<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> createLegacyEx // There are separate ExpressionContexts for each exchange pipeline, so make sure to // pass the pipeline's ExpressionContext to the plan executor factory. auto pipelineExpCtx = pipelineIt->getContext(); - execs.emplace_back( plan_executor_factory::make(std::move(pipelineExpCtx), std::move(pipelineIt), @@ -664,15 +665,11 @@ Status runAggregateOnView(OperationContext* opCtx, const MultipleCollectionAccessor& collections, boost::optional<std::unique_ptr<CollatorInterface>> collatorToUse, const ViewDefinition* view, - const boost::intrusive_ptr<ExpressionContext>& expCtx, std::shared_ptr<const CollectionCatalog> catalog, const PrivilegeVector& privileges, - CurOp* curOp, rpc::ReplyBuilderInterface* result, const std::function<void(void)>& resetContextFn) { auto nss = request.getNamespace(); - checkCollectionUUIDMismatch( - opCtx, nss, collections.getMainCollection(), request.getCollectionUUID()); uassert(ErrorCodes::CommandNotSupportedOnView, "mapReduce on a view is not supported", @@ -719,7 +716,7 @@ Status runAggregateOnView(OperationContext* opCtx, auto status{Status::OK()}; try { - status = runAggregate(opCtx, origNss, newRequest, newCmd, privileges, result); + status = runAggregate(opCtx, newRequest, newCmd, privileges, result, resolvedView, request); } catch (const ExceptionForCat<ErrorCategory::StaleShardVersionError>& ex) { // Since we expect the view to be UNSHARDED, if we reached to this point there are // two possibilities: @@ -739,31 +736,145 @@ Status runAggregateOnView(OperationContext* opCtx, // Set the namespace of the curop back to the view namespace so ctx records // stats on this view namespace on destruction. stdx::lock_guard<Client> lk(*opCtx->getClient()); - curOp->setNS_inlock(nss.ns()); + CurOp::get(opCtx)->setNS_inlock(nss.ns()); } return status; } +/** + * Determines the collection type of the query by precedence of various configurations. The order + * of these checks is critical since there may be overlap (e.g., a view over a virtual collection + * is classified as a view). + */ +query_shape::CollectionType determineCollectionType( + const boost::optional<AutoGetCollectionForReadCommandMaybeLockFree>& ctx, + boost::optional<const ResolvedView&> resolvedView, + bool hasChangeStream, + bool isCollectionless) { + if (resolvedView.has_value()) { + if (resolvedView->timeseries()) { + return query_shape::CollectionType::kTimeseries; + } + return query_shape::CollectionType::kView; + } + if (isCollectionless) { + return query_shape::CollectionType::kVirtual; + } + if (hasChangeStream) { + return query_shape::CollectionType::kChangeStream; + } + return ctx ? ctx->getCollectionType() : query_shape::CollectionType::kUnknown; +} + +std::unique_ptr<Pipeline, PipelineDeleter> parsePipelineAndRegisterQueryStats( + OperationContext* opCtx, + const NamespaceString& origNss, + const AggregateCommandRequest& request, + const boost::optional<AutoGetCollectionForReadCommandMaybeLockFree>& ctx, + std::unique_ptr<CollatorInterface> collator, + boost::optional<UUID> uuid, + ExpressionContext::CollationMatchesDefault collationMatchesDefault, + const MultipleCollectionAccessor& collections, + stdx::unordered_set<NamespaceString> pipelineInvolvedNamespaces, + const LiteParsedPipeline& liteParsedPipeline, + bool isCollectionless, + boost::optional<const ResolvedView&> resolvedView, + boost::optional<const AggregateCommandRequest&> origRequest) { + // If we're operating over a view, we first parse just the original user-given request + // for the sake of registering query stats. Then, we'll parse the view pipeline and stitch + // the two pipelines together below. + auto expCtx = + makeExpressionContext(opCtx, request, std::move(collator), uuid, collationMatchesDefault); + // If any involved collection contains extended-range data, set a flag which individual + // DocumentSource parsers can check. + collections.forEach([&](const CollectionPtr& coll) { + if (coll->getRequiresTimeseriesExtendedRangeSupport()) + expCtx->setRequiresTimeseriesExtendedRangeSupport(true); + }); + + const bool hasChangeStream = liteParsedPipeline.hasChangeStream(); + // A pipeline with $changeStreamSplitLargeEvent requires the use of resume token format + // v2, since the 'fragmentNum' field only exists in this version and later. + if (hasChangeStream && liteParsedPipeline.endsWithChangeStreamSplitLargeEvent()) { + expCtx->changeStreamTokenVersion = 2; + } + + auto requestForQueryStats = origRequest.has_value() ? *origRequest : request; + expCtx->startExpressionCounters(); + auto pipeline = Pipeline::parse(requestForQueryStats.getPipeline(), expCtx); + expCtx->stopExpressionCounters(); + + // Register query stats with the pre-optimized pipeline. Exclude queries against collections + // with encrypted fields. We still collect query stats on collection-less aggregations. + bool hasEncryptedFields = ctx && ctx->getCollection() && + ctx->getCollection()->getCollectionOptions().encryptedFieldConfig; + if (!hasEncryptedFields) { + // If this is a query over a resolved view, we want to register query stats with the + // original user-given request and pipeline, rather than the new request generated when + // resolving the view. + auto collectionType = + determineCollectionType(ctx, resolvedView, hasChangeStream, isCollectionless); + + query_stats::registerRequest(opCtx, + origNss, + [&]() { + return std::make_unique<query_stats::AggKey>( + requestForQueryStats, + *pipeline, + expCtx, + pipelineInvolvedNamespaces, + origNss, + collectionType); + }, + hasChangeStream); + } + + if (resolvedView.has_value()) { + expCtx->startExpressionCounters(); + + if (resolvedView->timeseries()) { + // For timeseries, there may have been rewrites done on the raw BSON pipeline + // during view resolution. We must parse the request's full resolved pipeline + // which will account for those rewrites. + // TODO SERVER-82101 Re-organize timeseries rewrites so timeseries can follow the + // same pattern here as other views + pipeline = Pipeline::parse(request.getPipeline(), expCtx); + } else { + // Parse the view pipeline, then stitch the user pipeline and view pipeline together + // to build the total aggregation pipeline. + auto userPipeline = std::move(pipeline); + pipeline = Pipeline::parse(resolvedView->getPipeline(), expCtx); + pipeline->appendPipeline(std::move(userPipeline)); + } + + expCtx->stopExpressionCounters(); + } + + return pipeline; +} } // namespace Status runAggregate(OperationContext* opCtx, - const NamespaceString& nss, AggregateCommandRequest& request, const BSONObj& cmdObj, const PrivilegeVector& privileges, - rpc::ReplyBuilderInterface* result) { - return runAggregate(opCtx, nss, request, {request}, cmdObj, privileges, result); + rpc::ReplyBuilderInterface* result, + boost::optional<const ResolvedView&> resolvedView, + boost::optional<const AggregateCommandRequest&> origRequest) { + return runAggregate( + opCtx, request, {request}, cmdObj, privileges, result, resolvedView, origRequest); } Status runAggregate(OperationContext* opCtx, - const NamespaceString& origNss, AggregateCommandRequest& request, const LiteParsedPipeline& liteParsedPipeline, const BSONObj& cmdObj, const PrivilegeVector& privileges, - rpc::ReplyBuilderInterface* result) { - + rpc::ReplyBuilderInterface* result, + boost::optional<const ResolvedView&> resolvedView, + boost::optional<const AggregateCommandRequest&> origRequest) { + auto origNss = origRequest.has_value() ? origRequest->getNamespace() : request.getNamespace(); // Perform some validations on the LiteParsedPipeline and request before continuing with the // aggregation command. performValidationChecks(opCtx, request, liteParsedPipeline); @@ -848,7 +959,6 @@ Status runAggregate(OperationContext* opCtx, // Raise an error if 'origNss' is a view. We do not need to check this if we are opening // a stream on an entire db or across the cluster. - const TenantDatabaseName origTenantDbName(boost::none, origNss.db()); if (!origNss.isCollectionlessAggregateNS()) { auto view = catalog->lookupView(opCtx, origNss); uassert(ErrorCodes::CommandNotSupportedOnView, @@ -882,7 +992,7 @@ Status runAggregate(OperationContext* opCtx, nss, Top::LockType::NotLocked, AutoStatsTracker::LogMode::kUpdateTopAndCurOp, - 0); + catalog->getDatabaseProfileLevel(nss.db())); auto [collator, match] = PipelineD::resolveCollator( opCtx, request.getCollation().get_value_or(BSONObj()), nullptr); collatorToUse.emplace(std::move(collator)); @@ -904,6 +1014,13 @@ Status runAggregate(OperationContext* opCtx, } } + // If collectionUUID was provided, verify the collection exists and has the expected UUID. + checkCollectionUUIDMismatch(opCtx, + nss, + collections.getMainCollection(), + request.getCollectionUUID(), + false /* checkFeatureFlag */); + // If this is a view, resolve it by finding the underlying collection and stitching view // pipelines and this request's pipeline together. We then release our locks before // recursively calling runAggregate(), which will re-acquire locks on the underlying @@ -921,41 +1038,29 @@ Status runAggregate(OperationContext* opCtx, collections, std::move(collatorToUse), ctx->getView(), - expCtx, catalog, privileges, - curOp, result, resetContext); } - // If collectionUUID was provided, verify the collection exists and has the expected UUID. - checkCollectionUUIDMismatch(opCtx, - nss, - collections.getMainCollection(), - request.getCollectionUUID(), - false /* checkFeatureFlag */); - invariant(collatorToUse); - expCtx = makeExpressionContext( - opCtx, request, std::move(*collatorToUse), uuid, collatorToUseMatchesDefault); - - // If any involved collection contains extended-range data, set a flag which individual - // DocumentSource parsers can check. - collections.forEach([&](const CollectionPtr& coll) { - if (coll->getRequiresTimeseriesExtendedRangeSupport()) - expCtx->setRequiresTimeseriesExtendedRangeSupport(true); - }); - - // A pipeline with $changeStreamSplitLargeEvent requires the use of resume token format v2, - // since the 'fragmentNum' field only exists in this version and later. - if (hasChangeStream && liteParsedPipeline.endsWithChangeStreamSplitLargeEvent()) { - expCtx->changeStreamTokenVersion = 2; - } - - expCtx->startExpressionCounters(); - auto pipeline = Pipeline::parse(request.getPipeline(), expCtx); - expCtx->stopExpressionCounters(); + auto pipeline = parsePipelineAndRegisterQueryStats(opCtx, + origNss, + request, + ctx, + std::move(*collatorToUse), + uuid, + collatorToUseMatchesDefault, + collections, + pipelineInvolvedNamespaces, + liteParsedPipeline, + nss.isCollectionlessAggregateNS(), + resolvedView, + origRequest); + expCtx = pipeline->getContext(); + + CurOp::get(opCtx)->beginQueryPlanningTimer(); if (!request.getAllowDiskUse().value_or(true)) { allowDiskUseFalseCounter.increment(); @@ -1034,6 +1139,7 @@ Status runAggregate(OperationContext* opCtx, } }); for (auto&& exec : execs) { + // TODO SERVER-79373: Do not create a cursor if results can fit in a single batch. ClientCursorParams cursorParams( std::move(exec), origNss, @@ -1077,6 +1183,7 @@ Status runAggregate(OperationContext* opCtx, cmdObj, &bodyBuilder); } + collectQueryStatsMongod(opCtx, std::move(curOp->debug().queryStatsInfo.key)); } else { // Cursor must be specified, if explain is not. const bool keepCursor = handleCursorCommand( @@ -1089,13 +1196,15 @@ Status runAggregate(OperationContext* opCtx, PlanSummaryStats stats; planExplainer.getSummaryStats(&stats); curOp->debug().setPlanSummaryMetrics(stats); - curOp->debug().nreturned = stats.nReturned; + curOp->setEndOfOpMetrics(stats.nReturned); + + collectQueryStatsMongod(opCtx, pins[0]); - // For an optimized away pipeline, signal the cache that a query operation has completed. - // For normal pipelines this is done in DocumentSourceCursor. + // For an optimized away pipeline, signal the cache that a query operation has + // completed. For normal pipelines this is done in DocumentSourceCursor. if (ctx) { - // Due to yielding, the collection pointers saved in MultipleCollectionAccessor might - // have become invalid. We will need to refresh them here. + // Due to yielding, the collection pointers saved in MultipleCollectionAccessor + // might have become invalid. We will need to refresh them here. collections = MultipleCollectionAccessor(opCtx, &ctx->getCollection(), ctx->getNss(), @@ -1118,10 +1227,11 @@ Status runAggregate(OperationContext* opCtx, } } - // The aggregation pipeline may change the namespace of the curop and we need to set it back to - // the original namespace to correctly report command stats. One example when the namespace can - // be changed is when the pipeline contains an $out stage, which executes an internal command to - // create a temp collection, changing the curop namespace to the name of this temp collection. + // The aggregation pipeline may change the namespace of the curop and we need to set it back + // to the original namespace to correctly report command stats. One example when the + // namespace can be changed is when the pipeline contains an $out stage, which executes an + // internal command to create a temp collection, changing the curop namespace to the name of + // this temp collection. { stdx::lock_guard<Client> lk(*opCtx->getClient()); curOp->setNS_inlock(origNss.ns()); @@ -1129,5 +1239,4 @@ Status runAggregate(OperationContext* opCtx, return Status::OK(); } - } // namespace mongo diff --git a/src/mongo/db/commands/run_aggregate.h b/src/mongo/db/commands/run_aggregate.h index b61538fb93d..ba73245bdbf 100644 --- a/src/mongo/db/commands/run_aggregate.h +++ b/src/mongo/db/commands/run_aggregate.h @@ -49,25 +49,30 @@ namespace mongo { * 'privileges' contains the privileges that were required to run this aggregation, to be used later * for re-checking privileges for getMore commands. * + * If the query over a view that's already been resolved, the resolved view and the original + * user-provided request both must be provided. + * * On success, fills out 'result' with the command response. */ Status runAggregate(OperationContext* opCtx, - const NamespaceString& nss, AggregateCommandRequest& request, const LiteParsedPipeline& liteParsedPipeline, const BSONObj& cmdObj, const PrivilegeVector& privileges, - rpc::ReplyBuilderInterface* result); + rpc::ReplyBuilderInterface* result, + boost::optional<const ResolvedView&> resolvedView = boost::none, + boost::optional<const AggregateCommandRequest&> origRequest = boost::none); /** * Convenience version that internally constructs the LiteParsedPipeline. */ Status runAggregate(OperationContext* opCtx, - const NamespaceString& nss, AggregateCommandRequest& request, const BSONObj& cmdObj, const PrivilegeVector& privileges, - rpc::ReplyBuilderInterface* result); + rpc::ReplyBuilderInterface* result, + boost::optional<const ResolvedView&> resolvedView = boost::none, + boost::optional<const AggregateCommandRequest&> origRequest = boost::none); /** * Tracks explicit use of allowDiskUse:false with find and aggregate commands. diff --git a/src/mongo/db/commands/server_status_metric.h b/src/mongo/db/commands/server_status_metric.h index ff546527a8d..423c49e2f5f 100644 --- a/src/mongo/db/commands/server_status_metric.h +++ b/src/mongo/db/commands/server_status_metric.h @@ -32,8 +32,13 @@ #include <string> #include "mongo/db/jsobj.h" +#include "mongo/platform/atomic_word.h" namespace mongo { +class Atomic64Metric; + +template <> +struct BSONObjAppendFormat<Atomic64Metric> : FormatKind<NumberLong> {}; class ServerStatusMetric { public: @@ -84,4 +89,36 @@ public: private: const T* _t; }; + +/** + * Atomic wrapper for long long type for Metrics. This is for values which are set rather than + * just incremented or decremented; if you want a counter, use Counter64. + */ +class Atomic64Metric { +public: + /** Set _value to the max of the current or newMax. */ + void setIfMax(long long newMax) { + /* Note: compareAndSwap will load into val most recent value. */ + for (long long val = _value.load(); val < newMax && !_value.compareAndSwap(&val, newMax);) { + } + } + + /** store val into value. */ + void set(long long val) { + _value.storeRelaxed(val); + } + + /** Return the current value. */ + long long get() const { + return _value.loadRelaxed(); + } + + /** TODO: SERVER-73806 Avoid implicit conversion to long long */ + operator long long() const { + return get(); + } + +private: + mongo::AtomicWord<long long> _value; +}; } // namespace mongo diff --git a/src/mongo/db/commands/set_cluster_parameter_invocation.cpp b/src/mongo/db/commands/set_cluster_parameter_invocation.cpp index 6d32f73b393..b19b01ead17 100644 --- a/src/mongo/db/commands/set_cluster_parameter_invocation.cpp +++ b/src/mongo/db/commands/set_cluster_parameter_invocation.cpp @@ -126,6 +126,11 @@ StatusWith<bool> ClusterParameterDBClientService::updateParameterOnDisk( return Status(ErrorCodes::FailedToParse, errmsg); } + auto responseStatus = response.toStatus(); + if (!responseStatus.isOK()) { + return responseStatus; + } + return response.getNModified() > 0 || response.getN() > 0; } diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp index 2c74781c36e..7462b287ed8 100644 --- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp +++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp @@ -631,14 +631,6 @@ private: tenantDbName, MODE_X, [&](const CollectionPtr& collection) { - if (collection->getTimeseriesBucketsMayHaveMixedSchemaData()) { - // The catalog entry flag has already been added. This can happen if the - // upgrade process was interrupted and is being run again, or if there - // was a time-series collection created during the upgrade. The upgrade - // process cannot be aborted at this point. - return true; - } - NamespaceStringOrUUID nsOrUUID(dbName, collection->uuid()); CollMod collModCmd(collection->ns()); BSONObjBuilder unusedBuilder; diff --git a/src/mongo/db/commands/user_management_commands.cpp b/src/mongo/db/commands/user_management_commands.cpp index 13c5650bc98..ff0af0d0f0d 100644 --- a/src/mongo/db/commands/user_management_commands.cpp +++ b/src/mongo/db/commands/user_management_commands.cpp @@ -1450,7 +1450,6 @@ UsersInfoReply CmdUMCTyped<UsersInfoCommand, UMCInfoParams>::Invocation::typedRu std::move(pipeline)); // Impose no cursor privilege requirements, as cursor is drained internally uassertStatusOK(runAggregate(opCtx, - AuthorizationManager::usersCollectionNamespace, aggRequest, aggregation_request_helper::serializeToCommandObj(aggRequest), PrivilegeVector(), diff --git a/src/mongo/db/commands/validate.cpp b/src/mongo/db/commands/validate.cpp index c17184cb14a..1ca15aca557 100644 --- a/src/mongo/db/commands/validate.cpp +++ b/src/mongo/db/commands/validate.cpp @@ -31,10 +31,22 @@ #include "mongo/platform/basic.h" + +#include "mongo/base/error_codes.h" +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bson_validate_gen.h" +#include "mongo/bson/bsonelement.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/collection_validation.h" #include "mongo/db/client.h" #include "mongo/db/commands.h" +#include "mongo/db/commands/test_commands_enabled.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/query/internal_plans.h" #include "mongo/db/storage/record_store.h" @@ -331,7 +343,9 @@ public: } CollectionValidation::AdditionalOptions additionalOptions; - additionalOptions.warnOnSchemaValidation = cmdObj["warnOnSchemaValidation"].trueValue(); + additionalOptions.validationVersion = getTestCommandsEnabled() + ? (ValidationVersion)bsonTestValidationVersion + : currentValidationVersion; ValidateResults validateResults; Status status = CollectionValidation::validate(opCtx, diff --git a/src/mongo/db/commands/write_commands.cpp b/src/mongo/db/commands/write_commands.cpp index bc031c90239..7196c905f1b 100644 --- a/src/mongo/db/commands/write_commands.cpp +++ b/src/mongo/db/commands/write_commands.cpp @@ -1676,6 +1676,7 @@ public: updateRequest.setLegacyRuntimeConstants(request().getLegacyRuntimeConstants().value_or( Variables::generateRuntimeConstants(opCtx))); updateRequest.setLetParameters(request().getLet()); + updateRequest.setBypassEmptyTsReplacement(request().getBypassEmptyTsReplacement()); updateRequest.setYieldPolicy(PlanYieldPolicy::YieldPolicy::YIELD_AUTO); updateRequest.setExplain(verbosity); diff --git a/src/mongo/db/concurrency/SConscript b/src/mongo/db/concurrency/SConscript index 14d6cef061e..4b487d3d843 100644 --- a/src/mongo/db/concurrency/SConscript +++ b/src/mongo/db/concurrency/SConscript @@ -31,7 +31,7 @@ env.Library( '$BUILD_DIR/mongo/base', ], LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/curop', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/server_options_servers', '$BUILD_DIR/mongo/db/storage/recovery_unit_base', '$BUILD_DIR/mongo/idl/server_parameter', @@ -106,7 +106,7 @@ env.CppUnitTest( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/auth/authmocks', - '$BUILD_DIR/mongo/db/curop', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/service_context_d_test_fixture', '$BUILD_DIR/mongo/transport/transport_layer_common', '$BUILD_DIR/mongo/transport/transport_layer_mock', diff --git a/src/mongo/db/cst/cst_pipeline_translation_test.cpp b/src/mongo/db/cst/cst_pipeline_translation_test.cpp index 3d247af1e57..fce29285b8b 100644 --- a/src/mongo/db/cst/cst_pipeline_translation_test.cpp +++ b/src/mongo/db/cst/cst_pipeline_translation_test.cpp @@ -810,7 +810,7 @@ TEST(CstPipelineTranslationTest, TranslatesConvertExpression) { ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson( "{$convert: {input: {$const: 'true'}, to: {$const: 'bool'}, onNull: {$const: 1}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesToBoolExpression) { @@ -820,7 +820,7 @@ TEST(CstPipelineTranslationTest, TranslatesToBoolExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$convert: {input: {$const: 0}, to: {$const: 'bool'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesToDateExpression) { @@ -830,7 +830,7 @@ TEST(CstPipelineTranslationTest, TranslatesToDateExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$convert: {input: {$const: 0}, to: {$const: 'date'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesToDecimalExpression) { @@ -841,7 +841,7 @@ TEST(CstPipelineTranslationTest, TranslatesToDecimalExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$convert: {input: {$const: 2.02}, to: {$const: 'decimal'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesToDoubleExpression) { @@ -852,7 +852,7 @@ TEST(CstPipelineTranslationTest, TranslatesToDoubleExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$convert: {input: {$const: '5.5'}, to: {$const: 'double'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesToIntExpression) { @@ -862,7 +862,7 @@ TEST(CstPipelineTranslationTest, TranslatesToIntExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$convert: {input: {$const: true}, to: {$const: 'int'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesToLongExpression) { @@ -872,7 +872,7 @@ TEST(CstPipelineTranslationTest, TranslatesToLongExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$convert: {input: {$const: 1.0}, to: {$const: 'long'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesToObjectIdExpression) { @@ -883,7 +883,7 @@ TEST(CstPipelineTranslationTest, TranslatesToObjectIdExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$convert: {input: '$_id', to: {$const: 'objectId'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesToStringExpression) { @@ -894,7 +894,7 @@ TEST(CstPipelineTranslationTest, TranslatesToStringExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$convert: {input: {$const: true}, to: {$const: 'string'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesTypeExpression) { @@ -910,13 +910,13 @@ TEST(CstPipelineTranslationTest, AbsConstantTranslation) { auto expCtx = getExpCtx(); auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); - ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$abs: [{$const: -1}]}")) == - expr->serialize(false))); + ASSERT_TRUE( + ValueComparator().evaluate(Value(fromjson("{$abs: [{$const: -1}]}")) == expr->serialize())); cst = CNode{CNode::ObjectChildren{{KeyFieldname::abs, CNode{UserDouble{-1.534}}}}}; expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$abs: [{$const: -1.534}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AbsVariableTransation) { @@ -925,8 +925,8 @@ TEST(CstPipelineTranslationTest, AbsVariableTransation) { auto expCtx = getExpCtx(); auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); - ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$abs: [\"$foo\"]}")) == - expr->serialize(false))); + ASSERT_TRUE( + ValueComparator().evaluate(Value(fromjson("{$abs: [\"$foo\"]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, AbsSingletonArrayTranslation) { @@ -935,8 +935,8 @@ TEST(CstPipelineTranslationTest, AbsSingletonArrayTranslation) { auto expCtx = getExpCtx(); auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); - ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$abs: [{$const: -1}]}")) == - expr->serialize(false))); + ASSERT_TRUE( + ValueComparator().evaluate(Value(fromjson("{$abs: [{$const: -1}]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, CeilTranslationTest) { @@ -945,7 +945,7 @@ TEST(CstPipelineTranslationTest, CeilTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$ceil: [{$const: 1.578}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, DivideTranslationTest) { const auto cst = CNode{CNode::ObjectChildren{ @@ -955,7 +955,7 @@ TEST(CstPipelineTranslationTest, DivideTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$divide: [{$const: 1.5}, {$const: 1}]}")) == expr->serialize(false))); + Value(fromjson("{$divide: [{$const: 1.5}, {$const: 1}]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, ExpTranslationTest) { @@ -964,7 +964,7 @@ TEST(CstPipelineTranslationTest, ExpTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$exp: [{$const: 1.5}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, FloorTranslationTest) { @@ -973,15 +973,15 @@ TEST(CstPipelineTranslationTest, FloorTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$floor: [{$const: 1.5}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, LnTranslationTest) { const auto cst = CNode{CNode::ObjectChildren{{KeyFieldname::ln, CNode{UserDouble{1.5}}}}}; auto expCtx = getExpCtx(); auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); - ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$ln: [{$const: 1.5}]}")) == - expr->serialize(false))); + ASSERT_TRUE( + ValueComparator().evaluate(Value(fromjson("{$ln: [{$const: 1.5}]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, LogTranslationTest) { @@ -992,7 +992,7 @@ TEST(CstPipelineTranslationTest, LogTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$log: [{$const: 1.5}, {$const: 10}]}")) == expr->serialize(false))); + Value(fromjson("{$log: [{$const: 1.5}, {$const: 10}]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, LogTenTranslationTest) { @@ -1001,7 +1001,7 @@ TEST(CstPipelineTranslationTest, LogTenTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$log10: [{$const: 1.5}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, ModTranslationTest) { @@ -1012,7 +1012,7 @@ TEST(CstPipelineTranslationTest, ModTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$mod: [{$const: 15}, {$const: 10}]}")) == expr->serialize(false))); + Value(fromjson("{$mod: [{$const: 15}, {$const: 10}]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, MultiplyTranslationTest) { @@ -1025,7 +1025,7 @@ TEST(CstPipelineTranslationTest, MultiplyTranslationTest) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$multiply: [{$const: 15}, {$const: 10}, {$const: 2}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, PowTranslationTest) { @@ -1036,14 +1036,14 @@ TEST(CstPipelineTranslationTest, PowTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$pow: [{$const: 5}, {$const: 2}]}")) == - expr->serialize(false))); + expr->serialize())); cst = CNode{CNode::ObjectChildren{ {KeyFieldname::pow, CNode{CNode::ArrayChildren{CNode{UserDouble{5.846}}, CNode{UserDouble{2.846}}}}}}}; expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$pow: [{$const: 5.846}, {$const: 2.846}]}")) == expr->serialize(false))); + Value(fromjson("{$pow: [{$const: 5.846}, {$const: 2.846}]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, RoundTranslationTest) { @@ -1054,7 +1054,7 @@ TEST(CstPipelineTranslationTest, RoundTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$round: [{$const: 1.5786}, {$const: 2}]}")) == expr->serialize(false))); + Value(fromjson("{$round: [{$const: 1.5786}, {$const: 2}]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, SqrtTranslationTest) { @@ -1063,7 +1063,7 @@ TEST(CstPipelineTranslationTest, SqrtTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$sqrt: [{$const: 144}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, SubtractTranslationTest) { @@ -1074,7 +1074,7 @@ TEST(CstPipelineTranslationTest, SubtractTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$subtract: [{$const: 1.5786}, {$const: 2}]}")) == expr->serialize(false))); + Value(fromjson("{$subtract: [{$const: 1.5786}, {$const: 2}]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, TruncTranslationTest) { @@ -1085,7 +1085,7 @@ TEST(CstPipelineTranslationTest, TruncTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$trunc: [{$const: 1.5786}, {$const: 2}]}")) == expr->serialize(false))); + Value(fromjson("{$trunc: [{$const: 1.5786}, {$const: 2}]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesReplaceOneExpression) { @@ -1099,7 +1099,7 @@ TEST(CstPipelineTranslationTest, TranslatesReplaceOneExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$replaceOne: {input: {$const: 'Antonio'}, find: {$const: 'Ant'}, " - "replacement: {$const: 'T'}}}")) == expr->serialize(false))); + "replacement: {$const: 'T'}}}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesReplaceAllExpression) { @@ -1114,7 +1114,7 @@ TEST(CstPipelineTranslationTest, TranslatesReplaceAllExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$replaceAll: {input: {$const: '10gen'}, find: {$const: '10gen'}, " - "replacement: {$const: 'MongoDB'}}}")) == expr->serialize(false))); + "replacement: {$const: 'MongoDB'}}}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesTrimExpression) { @@ -1127,7 +1127,7 @@ TEST(CstPipelineTranslationTest, TranslatesTrimExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$trim: {input: {$const: ' 10gen'}, chars: {$const: 'ge'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesTrimWithoutCharsExpression) { @@ -1139,7 +1139,7 @@ TEST(CstPipelineTranslationTest, TranslatesTrimWithoutCharsExpression) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$trim: {input: {$const: ' 10gen '}}}")) == expr->serialize(false))); + Value(fromjson("{$trim: {input: {$const: ' 10gen '}}}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesLtrimExpression) { @@ -1152,7 +1152,7 @@ TEST(CstPipelineTranslationTest, TranslatesLtrimExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$ltrim: {input: {$const: ' 10gen'}, chars: {$const: 'ge'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesRtrimExpression) { @@ -1165,7 +1165,7 @@ TEST(CstPipelineTranslationTest, TranslatesRtrimExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$rtrim: {input: {$const: '10gen '}, chars: {$const: 'ge'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesConcatExpression) { @@ -1178,7 +1178,7 @@ TEST(CstPipelineTranslationTest, TranslatesConcatExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$concat: [{$const: 'abc'}, {$const: 'def'}, {$const: '1x5'}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesDateToStringExpression) { @@ -1197,8 +1197,8 @@ TEST(CstPipelineTranslationTest, TranslatesDateToStringExpression) { Value( fromjson("{$dateToString: {date: \"$date\", format: {$const: \"%Y-%m-%d\"}, timezone: " "{$const: \"America/New_York\"}, onNull: {$const: \"8/10/20\"}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesDateFromStringExpression) { @@ -1216,8 +1216,8 @@ TEST(CstPipelineTranslationTest, TranslatesDateFromStringExpression) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$dateFromString: {dateString: {$const: \"'2017-02-08T12:10:40.787'\"}, " - "format: {$const: \"America/New_York\"}}}")) == expr->serialize(false))) - << expr->serialize(false); + "format: {$const: \"America/New_York\"}}}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesIndexOfCP) { @@ -1228,9 +1228,8 @@ TEST(CstPipelineTranslationTest, TranslatesIndexOfCP) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$indexOfCP: [{$const: \"ABC\"}, {$const: \"B\"}]}")) == - expr->serialize(false))) - << expr->serialize(false); + Value(fromjson("{$indexOfCP: [{$const: \"ABC\"}, {$const: \"B\"}]}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesIndexOfBytes) { @@ -1242,8 +1241,8 @@ TEST(CstPipelineTranslationTest, TranslatesIndexOfBytes) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$indexOfBytes: [{$const: \"ABC\"}, {$const: \"B\"}]}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesSplit) { @@ -1255,8 +1254,8 @@ TEST(CstPipelineTranslationTest, TranslatesSplit) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$split: [{$const: \"sapalaiat\"}, {$const: \"a\"}]}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesStrLenBytes) { @@ -1266,8 +1265,8 @@ TEST(CstPipelineTranslationTest, TranslatesStrLenBytes) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$strLenBytes: [{$const: \"four\"}]}")) == expr->serialize(false))) - << expr->serialize(false); + Value(fromjson("{$strLenBytes: [{$const: \"four\"}]}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesStrLenCP) { @@ -1277,8 +1276,8 @@ TEST(CstPipelineTranslationTest, TranslatesStrLenCP) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$strLenCP: [{$const: \"four\"}]}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesStrCaseCmp) { @@ -1290,8 +1289,8 @@ TEST(CstPipelineTranslationTest, TranslatesStrCaseCmp) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$strcasecmp: [{$const: \"100\"}, {$const: \"2\"}]}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, DesugarsSubstrToSubstrBytes) { @@ -1304,8 +1303,8 @@ TEST(CstPipelineTranslationTest, DesugarsSubstrToSubstrBytes) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$substrBytes: [{$const: \"abc\"}, {$const: 0}, {$const: \"a\"}]}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesSubstrBytes) { @@ -1318,8 +1317,8 @@ TEST(CstPipelineTranslationTest, TranslatesSubstrBytes) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$substrBytes: [{$const: \"abc\"}, {$const: 0}, {$const: \"a\"}]}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesSubstrCP) { @@ -1332,8 +1331,8 @@ TEST(CstPipelineTranslationTest, TranslatesSubstrCP) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$substrCP: [{$const: \"abc\"}, {$const: 0}, {$const: \"a\"}]}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesToLower) { @@ -1343,8 +1342,8 @@ TEST(CstPipelineTranslationTest, TranslatesToLower) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$toLower: [{$const: \"ABC\"}]}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesToUpper) { @@ -1354,8 +1353,8 @@ TEST(CstPipelineTranslationTest, TranslatesToUpper) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$toUpper: [{$const: \"EZ as 123\"}]}")) == expr->serialize(false))) - << expr->serialize(false); + Value(fromjson("{$toUpper: [{$const: \"EZ as 123\"}]}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesRegexFind) { @@ -1371,8 +1370,8 @@ TEST(CstPipelineTranslationTest, TranslatesRegexFind) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$regexFind: {input: {$const: \"aeiou\"}, regex: {$const: /.*/i}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesRegexFindAll) { @@ -1388,8 +1387,8 @@ TEST(CstPipelineTranslationTest, TranslatesRegexFindAll) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$regexFindAll: {input: {$const: \"aeiou\"}, regex: {$const: /.*/i}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesRegexMatch) { @@ -1405,8 +1404,8 @@ TEST(CstPipelineTranslationTest, TranslatesRegexMatch) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$regexMatch: {input: {$const: \"aeiou\"}, regex: {$const: /.*/i}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesSlice) { @@ -1421,8 +1420,8 @@ TEST(CstPipelineTranslationTest, TranslatesSlice) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$slice: [[{$const: 1}, {$const: 2}, {$const: 3}], {$const: -2}]}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesMeta) { @@ -1430,9 +1429,9 @@ TEST(CstPipelineTranslationTest, TranslatesMeta) { auto expCtx = getExpCtx(); auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); - ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$meta: \"textScore\"}")) == - expr->serialize(false))) - << expr->serialize(false); + ASSERT_TRUE( + ValueComparator().evaluate(Value(fromjson("{$meta: \"textScore\"}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, RecognizesSingleDollarAsNonConst) { @@ -1444,7 +1443,7 @@ TEST(CstPipelineTranslationTest, RecognizesSingleDollarAsNonConst) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$trunc: [\"$val\", \"$places\"]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, RecognizesDoubleDollarAsNonConst) { @@ -1455,7 +1454,7 @@ TEST(CstPipelineTranslationTest, RecognizesDoubleDollarAsNonConst) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$convert: {input: \"$$NOW\", to: {$const: 'date'}}}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AllElementsTrueTest) { @@ -1466,7 +1465,7 @@ TEST(CstPipelineTranslationTest, AllElementsTrueTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$allElementsTrue: [\"$set\"]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AnyElementsTrueTest) { @@ -1477,7 +1476,7 @@ TEST(CstPipelineTranslationTest, AnyElementsTrueTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$anyElementTrue: [\"$set\"]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, SetDifferenceTest) { @@ -1489,7 +1488,7 @@ TEST(CstPipelineTranslationTest, SetDifferenceTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$setDifference: [\"$set\", \"$set2\"]}")) == expr->serialize(false))); + Value(fromjson("{$setDifference: [\"$set\", \"$set2\"]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, SetEqualsTest) { @@ -1501,7 +1500,7 @@ TEST(CstPipelineTranslationTest, SetEqualsTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$setEquals: [\"$set\", \"$set2\"]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, SetIntersectionTest) { @@ -1515,7 +1514,7 @@ TEST(CstPipelineTranslationTest, SetIntersectionTest) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$setIntersection: [\"$set\", \"$set2\", \"$set3\"]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, SetIsSubsetTest) { @@ -1527,7 +1526,7 @@ TEST(CstPipelineTranslationTest, SetIsSubsetTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$setIsSubset: [\"$set\", \"$set2\"]}")) == expr->serialize(false))); + Value(fromjson("{$setIsSubset: [\"$set\", \"$set2\"]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, SetUnionTest) { @@ -1540,8 +1539,7 @@ TEST(CstPipelineTranslationTest, SetUnionTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$setUnion: [\"$set\", \"$set2\", \"$set3\"]}")) == - expr->serialize(false))); + Value(fromjson("{$setUnion: [\"$set\", \"$set2\", \"$set3\"]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, SinTranslationTest) { @@ -1550,7 +1548,7 @@ TEST(CstPipelineTranslationTest, SinTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$sin: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, CosTranslationTest) { @@ -1559,7 +1557,7 @@ TEST(CstPipelineTranslationTest, CosTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$cos: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TanTranslationTest) { @@ -1568,7 +1566,7 @@ TEST(CstPipelineTranslationTest, TanTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$tan: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, SinhTranslationTest) { @@ -1577,7 +1575,7 @@ TEST(CstPipelineTranslationTest, SinhTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$sinh: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, CoshTranslationTest) { @@ -1586,7 +1584,7 @@ TEST(CstPipelineTranslationTest, CoshTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$cosh: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TanhTranslationTest) { @@ -1595,7 +1593,7 @@ TEST(CstPipelineTranslationTest, TanhTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$tanh: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AsinTranslationTest) { @@ -1604,7 +1602,7 @@ TEST(CstPipelineTranslationTest, AsinTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$asin: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AcosTranslationTest) { @@ -1613,7 +1611,7 @@ TEST(CstPipelineTranslationTest, AcosTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$acos: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AtanTranslationTest) { @@ -1622,7 +1620,7 @@ TEST(CstPipelineTranslationTest, AtanTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$atan: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AsinhTranslationTest) { @@ -1631,7 +1629,7 @@ TEST(CstPipelineTranslationTest, AsinhTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$asinh: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AcoshTranslationTest) { @@ -1640,7 +1638,7 @@ TEST(CstPipelineTranslationTest, AcoshTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$acosh: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AtanhTranslationTest) { @@ -1649,7 +1647,7 @@ TEST(CstPipelineTranslationTest, AtanhTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$atanh: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, DegreesToRadiansTranslationTest) { @@ -1659,7 +1657,7 @@ TEST(CstPipelineTranslationTest, DegreesToRadiansTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$degreesToRadians: [{$const: 30}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, RadiansToDegreesTranslationTest) { @@ -1672,7 +1670,7 @@ TEST(CstPipelineTranslationTest, RadiansToDegreesTranslationTest) { ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$radiansToDegrees: [{$const: " "NumberDecimal(\"0.9272952180016122324285124629224290\")}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, SinArrayTranslationTest) { @@ -1682,7 +1680,7 @@ TEST(CstPipelineTranslationTest, SinArrayTranslationTest) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$sin: [{$const: 0.927}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, TranslatesDateToPartsExpression) { @@ -1699,8 +1697,8 @@ TEST(CstPipelineTranslationTest, TranslatesDateToPartsExpression) { ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$dateToParts: {date: \"$date\", timezone: " "{$const: \"America/New_York\"}, iso8601: {$const: false}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesDateFromPartsExpressionNonIso) { @@ -1723,8 +1721,8 @@ TEST(CstPipelineTranslationTest, TranslatesDateFromPartsExpressionNonIso) { Value(fromjson("{$dateFromParts: {year: \"$year\", month: \"$month\", day: \"$day\", " "hour: \"$hour\", minute: \"$minute\", second: \"$second\"," "millisecond: \"$millisecond\", timezone: " - "{$const: \"America/New_York\"}}}")) == expr->serialize(false))) - << expr->serialize(false); + "{$const: \"America/New_York\"}}}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesDateFromPartsExpressionIso) { @@ -1750,8 +1748,8 @@ TEST(CstPipelineTranslationTest, TranslatesDateFromPartsExpressionIso) { "second: \"$second\", millisecond: \"$millisecond\"," "isoWeekYear: \"$isoWeekYear\", isoWeek: \"$isoWeek\"," "isoDayOfWeek: \"$isoDayOfWeek\", timezone: " - "{$const: \"America/New_York\"}}}")) == expr->serialize(false))) - << expr->serialize(false); + "{$const: \"America/New_York\"}}}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesDayOfMonthExpressionArgsDoc) { @@ -1766,8 +1764,8 @@ TEST(CstPipelineTranslationTest, TranslatesDayOfMonthExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$dayOfMonth: {date: \"$date\", timezone: " - "{$const: \"America/New_York\"}}}")) == expr->serialize(false))) - << expr->serialize(false); + "{$const: \"America/New_York\"}}}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesDayOfMonthExpressionArgsExpr) { @@ -1777,10 +1775,9 @@ TEST(CstPipelineTranslationTest, TranslatesDayOfMonthExpressionArgsExpr) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(BSON("$dayOfMonth" << BSON( - "date" << BSON("$const" << Date_t::fromMillisSinceEpoch(12345678))))) == - expr->serialize(false))) - << expr->serialize(false); + Value(BSON("$dayOfMonth" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesDayOfWeekExpressionArgsDoc) { @@ -1795,8 +1792,8 @@ TEST(CstPipelineTranslationTest, TranslatesDayOfWeekExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$dayOfWeek: {date: \"$date\", timezone: " - "{$const: \"America/New_York\"}}}")) == expr->serialize(false))) - << expr->serialize(false); + "{$const: \"America/New_York\"}}}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesDayOfWeekExpressionArgsExpr) { @@ -1807,8 +1804,8 @@ TEST(CstPipelineTranslationTest, TranslatesDayOfWeekExpressionArgsExpr) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(BSON("$dayOfWeek" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( - 12345678))))) == expr->serialize(false))) - << expr->serialize(false); + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesDayOfYearExpressionArgsDoc) { @@ -1823,8 +1820,8 @@ TEST(CstPipelineTranslationTest, TranslatesDayOfYearExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$dayOfYear: {date: \"$date\", timezone: " - "{$const: \"America/New_York\"}}}")) == expr->serialize(false))) - << expr->serialize(false); + "{$const: \"America/New_York\"}}}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesDayOfYearExpressionArgsExpr) { @@ -1835,8 +1832,8 @@ TEST(CstPipelineTranslationTest, TranslatesDayOfYearExpressionArgsExpr) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(BSON("$dayOfYear" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( - 12345678))))) == expr->serialize(false))) - << expr->serialize(false); + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesHourExpressionArgsDoc) { @@ -1851,8 +1848,8 @@ TEST(CstPipelineTranslationTest, TranslatesHourExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$hour: {date: \"$date\", timezone: " "{$const: \"America/New_York\"}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesHourExpressionArgsExpr) { @@ -1863,8 +1860,8 @@ TEST(CstPipelineTranslationTest, TranslatesHourExpressionArgsExpr) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(BSON("$hour" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( - 12345678))))) == expr->serialize(false))) - << expr->serialize(false); + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesIsoDayOfWeekExpressionArgsDoc) { @@ -1879,8 +1876,8 @@ TEST(CstPipelineTranslationTest, TranslatesIsoDayOfWeekExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$isoDayOfWeek: {date: \"$date\", timezone: " - "{$const: \"America/New_York\"}}}")) == expr->serialize(false))) - << expr->serialize(false); + "{$const: \"America/New_York\"}}}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesIsoDayOfWeekExpressionArgsExpr) { @@ -1890,10 +1887,9 @@ TEST(CstPipelineTranslationTest, TranslatesIsoDayOfWeekExpressionArgsExpr) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(BSON("$isoDayOfWeek" - << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch(12345678))))) == - expr->serialize(false))) - << expr->serialize(false); + Value(BSON("$isoDayOfWeek" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesIsoWeekExpressionArgsDoc) { @@ -1908,8 +1904,8 @@ TEST(CstPipelineTranslationTest, TranslatesIsoWeekExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$isoWeek: {date: \"$date\", timezone: " "{$const: \"America/New_York\"}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesIsoWeekExpressionArgsExpr) { @@ -1920,8 +1916,8 @@ TEST(CstPipelineTranslationTest, TranslatesIsoWeekExpressionArgsExpr) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(BSON("$isoWeek" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( - 12345678))))) == expr->serialize(false))) - << expr->serialize(false); + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesIsoWeekYearExpressionArgsDoc) { @@ -1936,8 +1932,8 @@ TEST(CstPipelineTranslationTest, TranslatesIsoWeekYearExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$isoWeekYear: {date: \"$date\", timezone: " - "{$const: \"America/New_York\"}}}")) == expr->serialize(false))) - << expr->serialize(false); + "{$const: \"America/New_York\"}}}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesIsoWeekYearExpressionArgsExpr) { @@ -1947,10 +1943,9 @@ TEST(CstPipelineTranslationTest, TranslatesIsoWeekYearExpressionArgsExpr) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(BSON("$isoWeekYear" << BSON( - "date" << BSON("$const" << Date_t::fromMillisSinceEpoch(12345678))))) == - expr->serialize(false))) - << expr->serialize(false); + Value(BSON("$isoWeekYear" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesMillisecondExpressionArgsDoc) { @@ -1965,8 +1960,8 @@ TEST(CstPipelineTranslationTest, TranslatesMillisecondExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$millisecond: {date: \"$date\", timezone: " - "{$const: \"America/New_York\"}}}")) == expr->serialize(false))) - << expr->serialize(false); + "{$const: \"America/New_York\"}}}")) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesMillisecondExpressionArgsExpr) { @@ -1976,10 +1971,9 @@ TEST(CstPipelineTranslationTest, TranslatesMillisecondExpressionArgsExpr) { auto expr = cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(BSON("$millisecond" << BSON( - "date" << BSON("$const" << Date_t::fromMillisSinceEpoch(12345678))))) == - expr->serialize(false))) - << expr->serialize(false); + Value(BSON("$millisecond" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesMinuteExpressionArgsDoc) { @@ -1994,8 +1988,8 @@ TEST(CstPipelineTranslationTest, TranslatesMinuteExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$minute: {date: \"$date\", timezone: " "{$const: \"America/New_York\"}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesMinuteExpressionArgsExpr) { @@ -2006,8 +2000,8 @@ TEST(CstPipelineTranslationTest, TranslatesMinuteExpressionArgsExpr) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(BSON("$minute" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( - 12345678))))) == expr->serialize(false))) - << expr->serialize(false); + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesMonthExpressionArgsDoc) { @@ -2022,8 +2016,8 @@ TEST(CstPipelineTranslationTest, TranslatesMonthExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$month: {date: \"$date\", timezone: " "{$const: \"America/New_York\"}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesMonthExpressionArgsExpr) { @@ -2034,8 +2028,8 @@ TEST(CstPipelineTranslationTest, TranslatesMonthExpressionArgsExpr) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(BSON("$month" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( - 12345678))))) == expr->serialize(false))) - << expr->serialize(false); + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesSecondExpressionArgsDoc) { @@ -2050,8 +2044,8 @@ TEST(CstPipelineTranslationTest, TranslatesSecondExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$second: {date: \"$date\", timezone: " "{$const: \"America/New_York\"}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesSecondExpressionArgsExpr) { @@ -2062,8 +2056,8 @@ TEST(CstPipelineTranslationTest, TranslatesSecondExpressionArgsExpr) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(BSON("$second" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( - 12345678))))) == expr->serialize(false))) - << expr->serialize(false); + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesWeekExpressionArgsDoc) { @@ -2078,8 +2072,8 @@ TEST(CstPipelineTranslationTest, TranslatesWeekExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$week: {date: \"$date\", timezone: " "{$const: \"America/New_York\"}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesWeekExpressionArgsExpr) { @@ -2090,8 +2084,8 @@ TEST(CstPipelineTranslationTest, TranslatesWeekExpressionArgsExpr) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(BSON("$week" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( - 12345678))))) == expr->serialize(false))) - << expr->serialize(false); + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesYearExpressionArgsDoc) { @@ -2106,8 +2100,8 @@ TEST(CstPipelineTranslationTest, TranslatesYearExpressionArgsDoc) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$year: {date: \"$date\", timezone: " "{$const: \"America/New_York\"}}}")) == - expr->serialize(false))) - << expr->serialize(false); + expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, TranslatesYearExpressionArgsExpr) { @@ -2118,8 +2112,8 @@ TEST(CstPipelineTranslationTest, TranslatesYearExpressionArgsExpr) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(BSON("$year" << BSON("date" << BSON("$const" << Date_t::fromMillisSinceEpoch( - 12345678))))) == expr->serialize(false))) - << expr->serialize(false); + 12345678))))) == expr->serialize())) + << expr->serialize(); } TEST(CstPipelineTranslationTest, ArrayElemAtTranslationTest) { @@ -2134,7 +2128,7 @@ TEST(CstPipelineTranslationTest, ArrayElemAtTranslationTest) { ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson( "{$arrayElemAt: [ [{$const: 0}, {$const: 1}, {$const: 42}], {$const: 2} ]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, ArrayToObjectTranslationTest) { @@ -2149,7 +2143,7 @@ TEST(CstPipelineTranslationTest, ArrayToObjectTranslationTest) { ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson( "{$arrayToObject: [ [{$const: 0}, {$const: 1}], [{$const: 42}, {$const: 2}] ]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, ConcatArraysTranslationTest) { @@ -2164,7 +2158,7 @@ TEST(CstPipelineTranslationTest, ConcatArraysTranslationTest) { ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson( "{$concatArrays: [ [{$const: 0}, {$const: 1}], [{$const: 42}, {$const: 2}] ]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, FilterTranslationTest) { @@ -2185,8 +2179,7 @@ TEST(CstPipelineTranslationTest, FilterTranslationTest) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$filter: { input: [ {$const: 42}, {$const: 2} ], as: \"myVar\"," - "cond: { $gt: [\"$$myVar\", {$const: 2.5}] }}}")) == - expr->serialize(false))); + "cond: { $gt: [\"$$myVar\", {$const: 2.5}] }}}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, InTranslationTest) { @@ -2200,7 +2193,7 @@ TEST(CstPipelineTranslationTest, InTranslationTest) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$in: [ {$const: 1}, [ {$const: 1}, {$const: 2} ] ]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, IndexOfArrayTranslationTest) { @@ -2214,7 +2207,7 @@ TEST(CstPipelineTranslationTest, IndexOfArrayTranslationTest) { cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$indexOfArray: [ [ {$const: 1}, {$const: 2} ], {$const: 1}]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, FirstTranslationTest) { @@ -2224,7 +2217,7 @@ TEST(CstPipelineTranslationTest, FirstTranslationTest) { auto expr = mongo::cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$first: [ {$const: 1}, {$const: 2} ]}")) == expr->serialize(false))); + Value(fromjson("{$first: [ {$const: 1}, {$const: 2} ]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, IsArrayTranslationTest) { @@ -2235,7 +2228,7 @@ TEST(CstPipelineTranslationTest, IsArrayTranslationTest) { auto expr = mongo::cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$isArray: [{$const: 1}] }")) == - expr->serialize(false))); + expr->serialize())); } { const auto cst = CNode{CNode::ObjectChildren{ @@ -2245,7 +2238,7 @@ TEST(CstPipelineTranslationTest, IsArrayTranslationTest) { auto expr = mongo::cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$isArray: [[{$const: 1}]] }")) == - expr->serialize(false))); + expr->serialize())); } { const auto cst = CNode{CNode::ObjectChildren{ @@ -2254,7 +2247,7 @@ TEST(CstPipelineTranslationTest, IsArrayTranslationTest) { auto expr = mongo::cst_pipeline_translation::translateExpression( cst, expCtx.get(), expCtx->variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$isArray: [\"$myPath\"] }")) == - expr->serialize(false))); + expr->serialize())); } } } // namespace diff --git a/src/mongo/db/cst/cst_set_operator_translation_test.cpp b/src/mongo/db/cst/cst_set_operator_translation_test.cpp index 7f5c3c4c76c..87e17c0afa8 100644 --- a/src/mongo/db/cst/cst_set_operator_translation_test.cpp +++ b/src/mongo/db/cst/cst_set_operator_translation_test.cpp @@ -59,7 +59,7 @@ TEST(CstPipelineTranslationTest, AllElementsTrueTest) { auto expr = cst_pipeline_translation::translateExpression(cst, &expCtx, expCtx.variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$allElementsTrue: [\"$set\"]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, AnyElementsTrueTest) { @@ -70,7 +70,7 @@ TEST(CstPipelineTranslationTest, AnyElementsTrueTest) { auto expr = cst_pipeline_translation::translateExpression(cst, &expCtx, expCtx.variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$anyElementTrue: [\"$set\"]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, SetDifferenceTest) { @@ -82,7 +82,7 @@ TEST(CstPipelineTranslationTest, SetDifferenceTest) { auto expr = cst_pipeline_translation::translateExpression(cst, &expCtx, expCtx.variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$setDifference: [\"$set\", \"$set2\"]}")) == expr->serialize(false))); + Value(fromjson("{$setDifference: [\"$set\", \"$set2\"]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, SetEqualsTest) { @@ -94,7 +94,7 @@ TEST(CstPipelineTranslationTest, SetEqualsTest) { auto expr = cst_pipeline_translation::translateExpression(cst, &expCtx, expCtx.variablesParseState); ASSERT_TRUE(ValueComparator().evaluate(Value(fromjson("{$setEquals: [\"$set\", \"$set2\"]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, SetIntersectionTest) { @@ -108,7 +108,7 @@ TEST(CstPipelineTranslationTest, SetIntersectionTest) { cst_pipeline_translation::translateExpression(cst, &expCtx, expCtx.variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( Value(fromjson("{$setIntersection: [\"$set\", \"$set2\", \"$set3\"]}")) == - expr->serialize(false))); + expr->serialize())); } TEST(CstPipelineTranslationTest, SetIsSubsetTest) { @@ -120,7 +120,7 @@ TEST(CstPipelineTranslationTest, SetIsSubsetTest) { auto expr = cst_pipeline_translation::translateExpression(cst, &expCtx, expCtx.variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$setIsSubset: [\"$set\", \"$set2\"]}")) == expr->serialize(false))); + Value(fromjson("{$setIsSubset: [\"$set\", \"$set2\"]}")) == expr->serialize())); } TEST(CstPipelineTranslationTest, SetUnionTest) { @@ -133,8 +133,7 @@ TEST(CstPipelineTranslationTest, SetUnionTest) { auto expr = cst_pipeline_translation::translateExpression(cst, &expCtx, expCtx.variablesParseState); ASSERT_TRUE(ValueComparator().evaluate( - Value(fromjson("{$setUnion: [\"$set\", \"$set2\", \"$set3\"]}")) == - expr->serialize(false))); + Value(fromjson("{$setUnion: [\"$set\", \"$set2\", \"$set3\"]}")) == expr->serialize())); } } // namespace diff --git a/src/mongo/db/cst/cst_sort_translation_test.cpp b/src/mongo/db/cst/cst_sort_translation_test.cpp index 3c0e69979cd..2b733e197ca 100644 --- a/src/mongo/db/cst/cst_sort_translation_test.cpp +++ b/src/mongo/db/cst/cst_sort_translation_test.cpp @@ -65,8 +65,8 @@ void assertSortPatternsEQ(SortPattern correct, SortPattern fromTest) { } if (correct[i].expression) { if (fromTest[i].expression) - ASSERT_EQ(correct[i].expression->serialize(false).toString(), - fromTest[i].expression->serialize(false).toString()); + ASSERT_EQ(correct[i].expression->serialize().toString(), + fromTest[i].expression->serialize().toString()); else { FAIL("Pattern missing expression"); } diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp index af19498253a..23fd1495598 100644 --- a/src/mongo/db/curop.cpp +++ b/src/mongo/db/curop.cpp @@ -51,6 +51,9 @@ #include "mongo/db/profile_filter.h" #include "mongo/db/query/getmore_command_gen.h" #include "mongo/db/query/plan_summary_stats.h" +#include "mongo/db/query/query_stats/query_stats.h" +#include "mongo/db/stats/timer_stats.h" +#include "mongo/db/storage/storage_engine_parameters_gen.h" #include "mongo/logv2/log.h" #include "mongo/rpc/metadata/client_metadata.h" #include "mongo/rpc/metadata/impersonated_user_metadata.h" @@ -60,7 +63,6 @@ #include "mongo/util/net/socket_utils.h" #include "mongo/util/str.h" #include "mongo/util/system_tick_source.h" -#include <mongo/db/stats/timer_stats.h> namespace mongo { @@ -322,6 +324,18 @@ void CurOp::setGenericOpRequestDetails(OperationContext* opCtx, _ns = nss.ns(); } +void CurOp::setEndOfOpMetrics(long long nreturned) { + _debug.additiveMetrics.nreturned = nreturned; + // A non-none queryStatsInfo.keyHash indicates the current query is being tracked for queryStats + // and therefore the executionTime needs to be recorded as part of that effort. executionTime is + // set with the final executionTime in completeAndLogOperation, but for query stats collection + // we want it set before incrementing cursor metrics using OpDebug's AdditiveMetrics. The value + // set here will be overwritten later in completeAndLogOperation. + if (_debug.queryStatsInfo.keyHash) { + _debug.additiveMetrics.executionTime = elapsedTimeExcludingPauses(); + } +} + void CurOp::setMessage_inlock(StringData message) { if (_progressMeter.isActive()) { LOGV2_ERROR(20527, @@ -418,9 +432,10 @@ bool CurOp::completeAndLogOperation(OperationContext* opCtx, // Obtain the total execution time of this operation. done(); - _debug.executionTime = duration_cast<Microseconds>(elapsedTimeExcludingPauses()); - - const auto executionTimeMillis = durationCount<Milliseconds>(_debug.executionTime); + _debug.additiveMetrics.executionTime = + duration_cast<Microseconds>(elapsedTimeExcludingPauses()); + const auto executionTimeMillis = + durationCount<Milliseconds>(*_debug.additiveMetrics.executionTime); if (_debug.isReplOplogGetMore) { oplogGetMoreStats.recordMillis(executionTimeMillis); @@ -822,6 +837,10 @@ void OpDebug::report(OperationContext* opCtx, pAttrs->addDeepCopy("planSummary", curop.getPlanSummary().toString()); } + if (planningTime > Microseconds::zero()) { + pAttrs->add("planningTimeMicros", durationCount<Microseconds>(planningTime)); + } + if (prepareConflictDurationMillis > Milliseconds::zero()) { pAttrs->add("prepareConflictDuration", prepareConflictDurationMillis); } @@ -860,6 +879,7 @@ void OpDebug::report(OperationContext* opCtx, pAttrs->add("replanReason", redact(*replanReason)); } OPDEBUG_TOATTR_HELP_OPTIONAL("nMatched", additiveMetrics.nMatched); + OPDEBUG_TOATTR_HELP_OPTIONAL("nBatches", additiveMetrics.nBatches); OPDEBUG_TOATTR_HELP_OPTIONAL("nModified", additiveMetrics.nModified); OPDEBUG_TOATTR_HELP_OPTIONAL("ninserted", additiveMetrics.ninserted); OPDEBUG_TOATTR_HELP_OPTIONAL("ndeleted", additiveMetrics.ndeleted); @@ -874,7 +894,7 @@ void OpDebug::report(OperationContext* opCtx, additiveMetrics.temporarilyUnavailableErrors); pAttrs->add("numYields", curop.numYields()); - OPDEBUG_TOATTR_HELP(nreturned); + OPDEBUG_TOATTR_HELP_OPTIONAL("nreturned", additiveMetrics.nreturned); if (queryHash) { pAttrs->addDeepCopy("queryHash", zeroPaddedHex(*queryHash)); @@ -965,7 +985,16 @@ void OpDebug::report(OperationContext* opCtx, pAttrs->add("remoteOpWaitMillis", durationCount<Milliseconds>(*remoteOpWaitTime)); } - pAttrs->add("durationMillis", durationCount<Milliseconds>(executionTime)); + // durationMillis should always be present for any operation + pAttrs->add( + "durationMillis", + durationCount<Milliseconds>(additiveMetrics.executionTime.value_or(Microseconds{0}))); +} + +void OpDebug::reportStorageStats(logv2::DynamicAttributes* pAttrs) const { + if (storageStats) { + pAttrs->add("storage", storageStats->toBSON()); + } } #define OPDEBUG_APPEND_NUMBER2(b, x, y) \ @@ -1026,6 +1055,7 @@ void OpDebug::append(OperationContext* opCtx, b.append("replanReason", *replanReason); } OPDEBUG_APPEND_OPTIONAL(b, "nMatched", additiveMetrics.nMatched); + OPDEBUG_APPEND_OPTIONAL(b, "nBatches", additiveMetrics.nBatches); OPDEBUG_APPEND_OPTIONAL(b, "nModified", additiveMetrics.nModified); OPDEBUG_APPEND_OPTIONAL(b, "ninserted", additiveMetrics.ninserted); OPDEBUG_APPEND_OPTIONAL(b, "ndeleted", additiveMetrics.ndeleted); @@ -1043,7 +1073,7 @@ void OpDebug::append(OperationContext* opCtx, OPDEBUG_APPEND_OPTIONAL(b, "dataThroughputAverage", dataThroughputAverage); b.appendNumber("numYield", curop.numYields()); - OPDEBUG_APPEND_NUMBER(b, nreturned); + OPDEBUG_APPEND_OPTIONAL(b, "nreturned", additiveMetrics.nreturned); if (queryHash) { b.append("queryHash", zeroPaddedHex(*queryHash)); @@ -1117,7 +1147,10 @@ void OpDebug::append(OperationContext* opCtx, b.append("remoteOpWaitMillis", durationCount<Milliseconds>(*remoteOpWaitTime)); } - b.appendNumber("millis", durationCount<Milliseconds>(executionTime)); + // millis should always be present for any operation + b.appendNumber( + "millis", + durationCount<Milliseconds>(additiveMetrics.executionTime.value_or(Microseconds{0}))); if (!curop.getPlanSummary().empty()) { b.append("planSummary", curop.getPlanSummary()); @@ -1128,6 +1161,10 @@ void OpDebug::append(OperationContext* opCtx, durationCount<Microseconds>(totalOplogSlotDurationMicros)); } + if (planningTime > Microseconds::zero()) { + b.appendNumber("planningTimeMicros", durationCount<Microseconds>(planningTime)); + } + if (!execStats.isEmpty()) { b.append("execStats", std::move(execStats)); } @@ -1279,6 +1316,9 @@ std::function<BSONObj(ProfileFilter::Args)> OpDebug::appendStaged(StringSet requ addIfNeeded("nMatched", [](auto field, auto args, auto& b) { OPDEBUG_APPEND_OPTIONAL(b, field, args.op.additiveMetrics.nMatched); }); + addIfNeeded("nBatches", [](auto field, auto args, auto& b) { + OPDEBUG_APPEND_OPTIONAL(b, field, args.op.additiveMetrics.nBatches); + }); addIfNeeded("nModified", [](auto field, auto args, auto& b) { OPDEBUG_APPEND_OPTIONAL(b, field, args.op.additiveMetrics.nModified); }); @@ -1322,7 +1362,7 @@ std::function<BSONObj(ProfileFilter::Args)> OpDebug::appendStaged(StringSet requ b.appendNumber(field, args.curop.numYields()); }); addIfNeeded("nreturned", [](auto field, auto args, auto& b) { - OPDEBUG_APPEND_NUMBER2(b, field, args.op.nreturned); + OPDEBUG_APPEND_OPTIONAL(b, field, args.op.additiveMetrics.nreturned); }); addIfNeeded("queryHash", [](auto field, auto args, auto& b) { @@ -1426,10 +1466,14 @@ std::function<BSONObj(ProfileFilter::Args)> OpDebug::appendStaged(StringSet requ // the profiler (OpDebug::append) and the log file (OpDebug::report), so for the profile filter // we support both names. addIfNeeded("millis", [](auto field, auto args, auto& b) { - b.appendNumber(field, durationCount<Milliseconds>(args.op.executionTime)); + b.appendNumber(field, + durationCount<Milliseconds>( + args.op.additiveMetrics.executionTime.value_or(Microseconds{0}))); }); addIfNeeded("durationMillis", [](auto field, auto args, auto& b) { - b.appendNumber(field, durationCount<Milliseconds>(args.op.executionTime)); + b.appendNumber(field, + durationCount<Milliseconds>( + args.op.additiveMetrics.executionTime.value_or(Microseconds{0}))); }); addIfNeeded("planSummary", [](auto field, auto args, auto& b) { @@ -1445,6 +1489,10 @@ std::function<BSONObj(ProfileFilter::Args)> OpDebug::appendStaged(StringSet requ } }); + addIfNeeded("planningTimeMicros", [](auto field, auto args, auto& b) { + b.appendNumber(field, durationCount<Microseconds>(args.op.planningTime)); + }); + addIfNeeded("execStats", [](auto field, auto args, auto& b) { if (!args.op.execStats.isEmpty()) { b.append(field, args.op.execStats); @@ -1572,12 +1620,12 @@ void OpDebug::appendResolvedViewsInfo(BSONObjBuilder& builder) const { namespace { /** - * Adds two boost::optional long longs together. Returns boost::none if both 'lhs' and 'rhs' are - * uninitialized, or the sum of 'lhs' and 'rhs' if they are both initialized. Returns 'lhs' if only - * 'rhs' is uninitialized and vice-versa. + * Adds two boost::optionals of the same type with an operator+() together. Returns boost::none if + * both 'lhs' and 'rhs' are uninitialized, or the sum of 'lhs' and 'rhs' if they are both + * initialized. Returns 'lhs' if only 'rhs' is uninitialized and vice-versa. */ -boost::optional<long long> addOptionalLongs(const boost::optional<long long>& lhs, - const boost::optional<long long>& rhs) { +template <typename T> +boost::optional<T> addOptionals(const boost::optional<T>& lhs, const boost::optional<T>& rhs) { if (!rhs) { return lhs; } @@ -1586,24 +1634,29 @@ boost::optional<long long> addOptionalLongs(const boost::optional<long long>& lh } // namespace void OpDebug::AdditiveMetrics::add(const AdditiveMetrics& otherMetrics) { - keysExamined = addOptionalLongs(keysExamined, otherMetrics.keysExamined); - docsExamined = addOptionalLongs(docsExamined, otherMetrics.docsExamined); - nMatched = addOptionalLongs(nMatched, otherMetrics.nMatched); - nModified = addOptionalLongs(nModified, otherMetrics.nModified); - ninserted = addOptionalLongs(ninserted, otherMetrics.ninserted); - ndeleted = addOptionalLongs(ndeleted, otherMetrics.ndeleted); - nUpserted = addOptionalLongs(nUpserted, otherMetrics.nUpserted); - keysInserted = addOptionalLongs(keysInserted, otherMetrics.keysInserted); - keysDeleted = addOptionalLongs(keysDeleted, otherMetrics.keysDeleted); + keysExamined = addOptionals(keysExamined, otherMetrics.keysExamined); + docsExamined = addOptionals(docsExamined, otherMetrics.docsExamined); + nMatched = addOptionals(nMatched, otherMetrics.nMatched); + nreturned = addOptionals(nreturned, otherMetrics.nreturned); + nBatches = addOptionals(nBatches, otherMetrics.nBatches); + nModified = addOptionals(nModified, otherMetrics.nModified); + ninserted = addOptionals(ninserted, otherMetrics.ninserted); + ndeleted = addOptionals(ndeleted, otherMetrics.ndeleted); + nUpserted = addOptionals(nUpserted, otherMetrics.nUpserted); + keysInserted = addOptionals(keysInserted, otherMetrics.keysInserted); + keysDeleted = addOptionals(keysDeleted, otherMetrics.keysDeleted); prepareReadConflicts.fetchAndAdd(otherMetrics.prepareReadConflicts.load()); writeConflicts.fetchAndAdd(otherMetrics.writeConflicts.load()); temporarilyUnavailableErrors.fetchAndAdd(otherMetrics.temporarilyUnavailableErrors.load()); + executionTime = addOptionals(executionTime, otherMetrics.executionTime); } void OpDebug::AdditiveMetrics::reset() { keysExamined = boost::none; docsExamined = boost::none; nMatched = boost::none; + nreturned = boost::none; + nBatches = boost::none; nModified = boost::none; ninserted = boost::none; ndeleted = boost::none; @@ -1613,17 +1666,20 @@ void OpDebug::AdditiveMetrics::reset() { prepareReadConflicts.store(0); writeConflicts.store(0); temporarilyUnavailableErrors.store(0); + executionTime = boost::none; } bool OpDebug::AdditiveMetrics::equals(const AdditiveMetrics& otherMetrics) const { return keysExamined == otherMetrics.keysExamined && docsExamined == otherMetrics.docsExamined && - nMatched == otherMetrics.nMatched && nModified == otherMetrics.nModified && + nMatched == otherMetrics.nMatched && nreturned == otherMetrics.nreturned && + nBatches == otherMetrics.nBatches && nModified == otherMetrics.nModified && ninserted == otherMetrics.ninserted && ndeleted == otherMetrics.ndeleted && nUpserted == otherMetrics.nUpserted && keysInserted == otherMetrics.keysInserted && keysDeleted == otherMetrics.keysDeleted && prepareReadConflicts.load() == otherMetrics.prepareReadConflicts.load() && writeConflicts.load() == otherMetrics.writeConflicts.load() && - temporarilyUnavailableErrors.load() == otherMetrics.temporarilyUnavailableErrors.load(); + temporarilyUnavailableErrors.load() == otherMetrics.temporarilyUnavailableErrors.load() && + executionTime == otherMetrics.executionTime; } void OpDebug::AdditiveMetrics::incrementWriteConflicts(long long n) { @@ -1648,6 +1704,20 @@ void OpDebug::AdditiveMetrics::incrementKeysDeleted(long long n) { *keysDeleted += n; } +void OpDebug::AdditiveMetrics::incrementNreturned(long long n) { + if (!nreturned) { + nreturned = 0; + } + *nreturned += n; +} + +void OpDebug::AdditiveMetrics::incrementNBatches() { + if (!nBatches) { + nBatches = 0; + } + ++(*nBatches); +} + void OpDebug::AdditiveMetrics::incrementNinserted(long long n) { if (!ninserted) { ninserted = 0; @@ -1662,6 +1732,13 @@ void OpDebug::AdditiveMetrics::incrementNUpserted(long long n) { *nUpserted += n; } +void OpDebug::AdditiveMetrics::incrementExecutionTime(Microseconds n) { + if (!executionTime) { + executionTime = Microseconds{0}; + } + *executionTime += n; +} + void OpDebug::AdditiveMetrics::incrementPrepareReadConflicts(long long n) { prepareReadConflicts.fetchAndAdd(n); } @@ -1672,6 +1749,8 @@ string OpDebug::AdditiveMetrics::report() const { OPDEBUG_TOSTRING_HELP_OPTIONAL("keysExamined", keysExamined); OPDEBUG_TOSTRING_HELP_OPTIONAL("docsExamined", docsExamined); OPDEBUG_TOSTRING_HELP_OPTIONAL("nMatched", nMatched); + OPDEBUG_TOSTRING_HELP_OPTIONAL("nreturned", nreturned); + OPDEBUG_TOSTRING_HELP_OPTIONAL("nBatches", nBatches); OPDEBUG_TOSTRING_HELP_OPTIONAL("nModified", nModified); OPDEBUG_TOSTRING_HELP_OPTIONAL("ninserted", ninserted); OPDEBUG_TOSTRING_HELP_OPTIONAL("ndeleted", ndeleted); @@ -1681,6 +1760,9 @@ string OpDebug::AdditiveMetrics::report() const { OPDEBUG_TOSTRING_HELP_ATOMIC("prepareReadConflicts", prepareReadConflicts); OPDEBUG_TOSTRING_HELP_ATOMIC("writeConflicts", writeConflicts); OPDEBUG_TOSTRING_HELP_ATOMIC("temporarilyUnavailableErrors", temporarilyUnavailableErrors); + if (executionTime) { + s << " durationMillis:" << durationCount<Milliseconds>(*executionTime); + } return s.str(); } @@ -1689,6 +1771,8 @@ void OpDebug::AdditiveMetrics::report(logv2::DynamicAttributes* pAttrs) const { OPDEBUG_TOATTR_HELP_OPTIONAL("keysExamined", keysExamined); OPDEBUG_TOATTR_HELP_OPTIONAL("docsExamined", docsExamined); OPDEBUG_TOATTR_HELP_OPTIONAL("nMatched", nMatched); + OPDEBUG_TOATTR_HELP_OPTIONAL("nreturned", nreturned); + OPDEBUG_TOATTR_HELP_OPTIONAL("nBatches", nBatches); OPDEBUG_TOATTR_HELP_OPTIONAL("nModified", nModified); OPDEBUG_TOATTR_HELP_OPTIONAL("ninserted", ninserted); OPDEBUG_TOATTR_HELP_OPTIONAL("ndeleted", ndeleted); @@ -1698,6 +1782,9 @@ void OpDebug::AdditiveMetrics::report(logv2::DynamicAttributes* pAttrs) const { OPDEBUG_TOATTR_HELP_ATOMIC("prepareReadConflicts", prepareReadConflicts); OPDEBUG_TOATTR_HELP_ATOMIC("writeConflicts", writeConflicts); OPDEBUG_TOATTR_HELP_ATOMIC("temporarilyUnavailableErrors", temporarilyUnavailableErrors); + if (executionTime) { + pAttrs->add("durationMillis", durationCount<Milliseconds>(*executionTime)); + } } BSONObj OpDebug::AdditiveMetrics::reportBSON() const { @@ -1705,6 +1792,8 @@ BSONObj OpDebug::AdditiveMetrics::reportBSON() const { OPDEBUG_APPEND_OPTIONAL(b, "keysExamined", keysExamined); OPDEBUG_APPEND_OPTIONAL(b, "docsExamined", docsExamined); OPDEBUG_APPEND_OPTIONAL(b, "nMatched", nMatched); + OPDEBUG_APPEND_OPTIONAL(b, "nreturned", nreturned); + OPDEBUG_APPEND_OPTIONAL(b, "nBatches", nBatches); OPDEBUG_APPEND_OPTIONAL(b, "nModified", nModified); OPDEBUG_APPEND_OPTIONAL(b, "ninserted", ninserted); OPDEBUG_APPEND_OPTIONAL(b, "ndeleted", ndeleted); @@ -1714,6 +1803,9 @@ BSONObj OpDebug::AdditiveMetrics::reportBSON() const { OPDEBUG_APPEND_ATOMIC(b, "prepareReadConflicts", prepareReadConflicts); OPDEBUG_APPEND_ATOMIC(b, "writeConflicts", writeConflicts); OPDEBUG_APPEND_ATOMIC(b, "temporarilyUnavailableErrors", temporarilyUnavailableErrors); + if (executionTime) { + b.appendNumber("durationMillis", durationCount<Milliseconds>(*executionTime)); + } return b.obj(); } diff --git a/src/mongo/db/curop.h b/src/mongo/db/curop.h index af982844d35..ba1545130ff 100644 --- a/src/mongo/db/curop.h +++ b/src/mongo/db/curop.h @@ -30,18 +30,18 @@ #pragma once +#include "mongo/util/duration.h" #include <memory> #include "mongo/config.h" #include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/user_acquisition_stats.h" #include "mongo/db/catalog/collection_catalog.h" -#include "mongo/db/clientcursor.h" #include "mongo/db/commands.h" #include "mongo/db/cursor_id.h" #include "mongo/db/operation_context.h" #include "mongo/db/profile_filter.h" -#include "mongo/db/server_options.h" +#include "mongo/db/query/query_stats/key.h" #include "mongo/db/stats/resource_consumption_metrics.h" #include "mongo/db/write_concern_options.h" #include "mongo/logv2/attribute_storage.h" @@ -66,8 +66,10 @@ struct PlanSummaryStats; class OpDebug { public: /** - * Holds counters for execution statistics that are meaningful both for multi-statement - * transactions and for individual operations outside of a transaction. + * Holds counters for execution statistics that can be accumulated by one or more operations. + * They're accumulated as we go for a single operation, but are also extracted and stored + * externally if they need to be accumulated across multiple operations (which have multiple + * CurOps), including for cursors and multi-statement transactions. */ class AdditiveMetrics { public: @@ -120,6 +122,16 @@ public: void incrementKeysDeleted(long long n); /** + * Increments nreturned by n. + */ + void incrementNreturned(long long n); + + /** + * Increments nBatches by 1. + */ + void incrementNBatches(); + + /** * Increments ninserted by n. */ void incrementNinserted(long long n); @@ -135,6 +147,11 @@ public: void incrementPrepareReadConflicts(long long n); /** + * Increments executionTime by n. + */ + void incrementExecutionTime(Microseconds n); + + /** * Generates a string showing all non-empty fields. For every non-empty field field1, * field2, ..., with corresponding values value1, value2, ..., we will output a string in * the format: "<field1>:<value1> <field2>:<value2> ...". @@ -149,6 +166,10 @@ public: // Number of records that match the query. boost::optional<long long> nMatched; + // Number of records returned so far. + boost::optional<long long> nreturned; + // Number of batches returned so far. + boost::optional<long long> nBatches; // Number of records written (no no-ops). boost::optional<long long> nModified; boost::optional<long long> ninserted; @@ -169,6 +190,9 @@ public: AtomicWord<long long> prepareReadConflicts{0}; AtomicWord<long long> writeConflicts{0}; AtomicWord<long long> temporarilyUnavailableErrors{0}; + + // Amount of time spent executing a query. + boost::optional<Microseconds> executionTime; }; OpDebug() = default; @@ -178,6 +202,8 @@ public: const ResourceConsumption::OperationMetrics* operationMetrics, logv2::DynamicAttributes* pAttrs) const; + void reportStorageStats(logv2::DynamicAttributes* pAttrs) const; + /** * Appends information about the current operation to "builder" * @@ -263,6 +289,51 @@ public: // The hash of the query's "stable" key. This represents the query's shape. boost::optional<uint32_t> queryHash; + /* The QueryStatsInfo struct was created to bundle all the queryStats related fields of CurOp & + * OpDebug together (SERVER-83280). + * + * ClusterClientCursorImpl and ClientCursor also contain _queryStatsKey and _queryStatsKeyHash + * members but NOT a wasRateLimited member. Variable names & accesses would be more consistent + * across the code if ClusterClientCursorImpl and ClientCursor each also had a QueryStatsInfo + * struct, but we considered and rejected two different potential implementations of this: + * - Option 1: + * Declare a QueryStatsInfo struct in each .h file. Every struct would have key and keyHash + * fields, and a wasRateLimited field would be added only to CurOp. But, it seemed confusing + * to have slightly different structs with the same name declared three different times. + * - Option 2: + * Create a query_stats_info.h that declares QueryStatsInfo--identical to the version defined + * in this file. CurOp/OpDebug, ClientCursor, and ClusterClientCursorImpl would then all + * have their own QueryStatsInfo instances, potentially as a unique_ptr or boost::optional. A + * benefit to this would be the ability to to just move the entire QueryStatsInfo struct from + * Op to the Cursor, instead of copying it over field by field (the current method). But: + * - The current code moves ownership of the key, but copies the keyHash. So, for workflows + * that require multiple cursors, like sharding, one cursor would own the key, but all + * cursors would have copies of the keyHash. The problem with trying to move around the + * struct in its entirety is that access to the *entire* struct would be lost on the + * move, meaning there's no way to retain the keyHash (that doesn't largely nullify the + * benefits of having the struct). + * - It seemed odd to have ClientCursor and ClusterClientCursorImpl using the struct but + * never needing the wasRateLimited field. + */ + + // Note that the only case when the three fields of the below struct are null, none, and false + // is if the query stats feature flag is turned off. + struct QueryStatsInfo { + // Uniquely identifies one query stats entry. + // nullptr if `wasRateLimited` is true. + std::unique_ptr<query_stats::Key> key; + // A cached value of `absl::HashOf(key)`. + // Always populated if `key` is non-null. boost::none if `wasRateLimited` is true. + boost::optional<std::size_t> keyHash; + // True if the request was rate limited and stats should not be collected. + bool wasRateLimited = false; + // True if the request was a change stream request. + // TODO SERVER-89058 will make it true for all tailable cursors. + bool willNeverExhaust = false; + }; + + QueryStatsInfo queryStatsInfo; + // Has a value if this operation is a query. True if the execution tree for the find part of the // query was built exclusively using the classic query engine, false if any part was built using // SBE. @@ -278,9 +349,10 @@ public: // Details of any error (whether from an exception or a command returning failure). Status errInfo = Status::OK(); - // response info - Microseconds executionTime{0}; - long long nreturned{-1}; + // Amount of time spent planning the query. Begins after parsing and ends + // after optimizations. + Microseconds planningTime{0}; + int responseLength{-1}; // Shard targeting info. @@ -305,7 +377,9 @@ public: // Used to track the amount of time spent waiting for a response from remote operations. boost::optional<Microseconds> remoteOpWaitTime; - // Stores additive metrics. + // Stores the current operation's count of these metrics. If they are needed to be accumulated + // elsewhere, they should be extracted by another aggregator (like the ClientCursor) to ensure + // these only ever reflect just this CurOp's consumption. AdditiveMetrics additiveMetrics; // Stores storage statistics. @@ -395,6 +469,13 @@ public: NetworkOp op); /** + * Sets metrics collected at the end of an operation onto curOp's OpDebug instance. Note that + * this is used in tandem with OpDebug::setPlanSummaryMetrics so should not repeat any metrics + * collected there. + */ + void setEndOfOpMetrics(long long nreturned); + + /** * Marks the operation end time, records the length of the client response if a valid response * exists, and then - subject to the current values of slowMs and sampleRate - logs this CurOp * to file under the given LogComponent. Returns 'true' if, in addition to being logged, this @@ -688,6 +769,37 @@ public: return computeElapsedTimeTotal(start, _end.load()) - _totalPausedDuration; } + /** + * The planningTimeMicros metric, reported in the system profiler and in queryStats, is measured + * using the Curop instance's _tickSource. Currently, _tickSource is only paused in places where + logical work is being done. If this were to change, and _tickSource + were to be paused during query planning for reasons unrelated to the work of + planning/optimization, it would break the planning time measurement below. + * + */ + void beginQueryPlanningTimer() { + // This is an inner executor/cursor, the metrics for which don't get tracked by + // OpDebug::planningTime. + if (_queryPlanningStart.load() != 0) { + return; + } + _queryPlanningStart = _tickSource->getTicks(); + } + + void stopQueryPlanningTimer() { + // The planningTime metric is defined as being done once PrepareExecutionHelper::prepare() + // is hit, which calls this function to stop the timer. As certain queries like $lookup + // require inner cursors/executors that will follow this same codepath, it is important to + // make sure the metric exclusively captures the time associated with the outermost cursor. + // This is done by making sure planningTime has not already been set and that start has been + // marked (as inner executors are prepared outside of the codepath that begins the planning + // timer). + auto start = _queryPlanningStart.load(); + if (debug().planningTime == Microseconds{0} && start != 0) { + _queryPlanningEnd = _tickSource->getTicks(); + debug().planningTime = computeElapsedTimeTotal(start, _queryPlanningEnd.load()); + } + } /** * Starts the waitForWriteConcern timer. @@ -839,16 +951,6 @@ public: _tickSource = tickSource; } - /** - * Merge match counters from the current operation into the global map and stop counting. - */ - void stopMatchExprCounter(); - - /** - * Increment the counter for the match expression with given name in the current operation. - */ - void incrementMatchExprCounter(StringData name); - private: class CurOpStack; @@ -923,6 +1025,10 @@ private: // These values are used to calculate the amount of time spent waiting for write concern. std::atomic<TickSource::Tick> _waitForWriteConcernStart{0}; // NOLINT std::atomic<TickSource::Tick> _waitForWriteConcernEnd{0}; // NOLINT + + // These values are used to calculate the amount of time spent planning a query. + std::atomic<TickSource::Tick> _queryPlanningStart{0}; // NOLINT + std::atomic<TickSource::Tick> _queryPlanningEnd{0}; // NOLINT }; } // namespace mongo diff --git a/src/mongo/db/curop_metrics.cpp b/src/mongo/db/curop_metrics.cpp index 9bc2763cccc..e433e4a9dfb 100644 --- a/src/mongo/db/curop_metrics.cpp +++ b/src/mongo/db/curop_metrics.cpp @@ -64,8 +64,8 @@ ServerStatusMetricField<Counter64> displayWriteConflicts("operation.writeConflic void recordCurOpMetrics(OperationContext* opCtx) { const OpDebug& debug = CurOp::get(opCtx)->debug(); - if (debug.nreturned > 0) - returnedCounter.increment(debug.nreturned); + if (debug.additiveMetrics.nreturned) + returnedCounter.increment(*debug.additiveMetrics.nreturned); if (debug.additiveMetrics.ninserted) insertedCounter.increment(*debug.additiveMetrics.ninserted); if (debug.additiveMetrics.nMatched) diff --git a/src/mongo/db/curop_test.cpp b/src/mongo/db/curop_test.cpp index 332e7d9042e..af327f05482 100644 --- a/src/mongo/db/curop_test.cpp +++ b/src/mongo/db/curop_test.cpp @@ -67,6 +67,10 @@ TEST(CurOpTest, AddingAdditiveMetricsObjectsTogetherShouldAddFieldsTogether) { additiveMetricsToAdd.docsExamined = 2; currentAdditiveMetrics.nMatched = 5; additiveMetricsToAdd.nMatched = 5; + currentAdditiveMetrics.nreturned = 10; + additiveMetricsToAdd.nreturned = 5; + currentAdditiveMetrics.nBatches = 2; + additiveMetricsToAdd.nBatches = 1; currentAdditiveMetrics.nModified = 3; additiveMetricsToAdd.nModified = 1; currentAdditiveMetrics.ninserted = 4; @@ -79,6 +83,8 @@ TEST(CurOpTest, AddingAdditiveMetricsObjectsTogetherShouldAddFieldsTogether) { additiveMetricsToAdd.keysInserted = 5; currentAdditiveMetrics.keysDeleted = 4; additiveMetricsToAdd.keysDeleted = 2; + currentAdditiveMetrics.executionTime = Microseconds{200}; + additiveMetricsToAdd.executionTime = Microseconds{80}; currentAdditiveMetrics.prepareReadConflicts.store(1); additiveMetricsToAdd.prepareReadConflicts.store(5); currentAdditiveMetrics.writeConflicts.store(7); @@ -96,6 +102,10 @@ TEST(CurOpTest, AddingAdditiveMetricsObjectsTogetherShouldAddFieldsTogether) { *additiveMetricsBeforeAdd.docsExamined + *additiveMetricsToAdd.docsExamined); ASSERT_EQ(*currentAdditiveMetrics.nMatched, *additiveMetricsBeforeAdd.nMatched + *additiveMetricsToAdd.nMatched); + ASSERT_EQ(*currentAdditiveMetrics.nreturned, + *additiveMetricsBeforeAdd.nreturned + *additiveMetricsToAdd.nreturned); + ASSERT_EQ(*currentAdditiveMetrics.nBatches, + *additiveMetricsBeforeAdd.nBatches + *additiveMetricsToAdd.nBatches); ASSERT_EQ(*currentAdditiveMetrics.nModified, *additiveMetricsBeforeAdd.nModified + *additiveMetricsToAdd.nModified); ASSERT_EQ(*currentAdditiveMetrics.ninserted, @@ -108,6 +118,8 @@ TEST(CurOpTest, AddingAdditiveMetricsObjectsTogetherShouldAddFieldsTogether) { *additiveMetricsBeforeAdd.keysInserted + *additiveMetricsToAdd.keysInserted); ASSERT_EQ(*currentAdditiveMetrics.keysDeleted, *additiveMetricsBeforeAdd.keysDeleted + *additiveMetricsToAdd.keysDeleted); + ASSERT_EQ(*currentAdditiveMetrics.executionTime, + *additiveMetricsBeforeAdd.executionTime + *additiveMetricsToAdd.executionTime); ASSERT_EQ(currentAdditiveMetrics.prepareReadConflicts.load(), additiveMetricsBeforeAdd.prepareReadConflicts.load() + additiveMetricsToAdd.prepareReadConflicts.load()); @@ -123,6 +135,8 @@ TEST(CurOpTest, AddingUninitializedAdditiveMetricsFieldsShouldBeTreatedAsZero) { // Initialize field values for both AdditiveMetrics objects. additiveMetricsToAdd.keysExamined = 5; currentAdditiveMetrics.docsExamined = 4; + currentAdditiveMetrics.nreturned = 2; + additiveMetricsToAdd.nBatches = 1; currentAdditiveMetrics.nModified = 3; additiveMetricsToAdd.ninserted = 0; currentAdditiveMetrics.keysInserted = 6; @@ -147,6 +161,14 @@ TEST(CurOpTest, AddingUninitializedAdditiveMetricsFieldsShouldBeTreatedAsZero) { // should be treated as zero. ASSERT_EQ(*currentAdditiveMetrics.docsExamined, *additiveMetricsBeforeAdd.docsExamined); + // The 'nreturned' field for the AdditiveMetrics object to add was not initialized, so it + // should be treated as zero. + ASSERT_EQ(*currentAdditiveMetrics.nreturned, *additiveMetricsBeforeAdd.nreturned); + + // The 'nBatches' field for the current AdditiveMetrics object was not initialized, so it + // should be treated as zero. + ASSERT_EQ(*currentAdditiveMetrics.nBatches, *additiveMetricsToAdd.nBatches); + // The 'nMatched' field for both the current AdditiveMetrics object and the AdditiveMetrics // object to add were not initialized, so nMatched should still be uninitialized after the add. ASSERT_EQ(currentAdditiveMetrics.nMatched, boost::none); @@ -155,6 +177,11 @@ TEST(CurOpTest, AddingUninitializedAdditiveMetricsFieldsShouldBeTreatedAsZero) { // object to add were not initialized, so nUpserted should still be uninitialized after the add. ASSERT_EQ(currentAdditiveMetrics.nUpserted, boost::none); + // The 'executionTime' field for both the current AdditiveMetrics object and the AdditiveMetrics + // object to add were not initialized, so executionTime should still be uninitialized after the + // add. + ASSERT_EQ(currentAdditiveMetrics.executionTime, boost::none); + // The following field values should have changed after adding. ASSERT_EQ(*currentAdditiveMetrics.keysInserted, *additiveMetricsBeforeAdd.keysInserted + *additiveMetricsToAdd.keysInserted); @@ -175,6 +202,8 @@ TEST(CurOpTest, AdditiveMetricsFieldsShouldIncrementByN) { additiveMetrics.writeConflicts.store(1); additiveMetrics.keysInserted = 2; additiveMetrics.prepareReadConflicts.store(6); + additiveMetrics.nreturned = 3; + additiveMetrics.executionTime = Microseconds{160}; // Increment the fields. additiveMetrics.incrementWriteConflicts(1); @@ -183,6 +212,9 @@ TEST(CurOpTest, AdditiveMetricsFieldsShouldIncrementByN) { additiveMetrics.incrementNinserted(3); additiveMetrics.incrementNUpserted(6); additiveMetrics.incrementPrepareReadConflicts(2); + additiveMetrics.incrementNreturned(2); + additiveMetrics.incrementNBatches(); + additiveMetrics.incrementExecutionTime(Microseconds{120}); ASSERT_EQ(additiveMetrics.writeConflicts.load(), 2); ASSERT_EQ(*additiveMetrics.keysInserted, 7); @@ -190,6 +222,9 @@ TEST(CurOpTest, AdditiveMetricsFieldsShouldIncrementByN) { ASSERT_EQ(*additiveMetrics.ninserted, 3); ASSERT_EQ(*additiveMetrics.nUpserted, 6); ASSERT_EQ(additiveMetrics.prepareReadConflicts.load(), 8); + ASSERT_EQ(*additiveMetrics.nreturned, 5); + ASSERT_EQ(*additiveMetrics.nBatches, 1); + ASSERT_EQ(*additiveMetrics.executionTime, Microseconds{280}); } TEST(CurOpTest, OptionalAdditiveMetricsNotDisplayedIfUninitialized) { diff --git a/src/mongo/db/cursor_manager.cpp b/src/mongo/db/cursor_manager.cpp index f26f77ab50d..f57c67a6c75 100644 --- a/src/mongo/db/cursor_manager.cpp +++ b/src/mongo/db/cursor_manager.cpp @@ -53,6 +53,7 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/query/plan_executor.h" +#include "mongo/db/query/query_feature_flags_gen.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/db/service_context.h" #include "mongo/logv2/log.h" @@ -61,29 +62,6 @@ namespace mongo { -static Counter64 cursorStatsLifespanLessThan1Second; -static Counter64 cursorStatsLifespanLessThan5Seconds; -static Counter64 cursorStatsLifespanLessThan15Seconds; -static Counter64 cursorStatsLifespanLessThan30Seconds; -static Counter64 cursorStatsLifespanLessThan1Minute; -static Counter64 cursorStatsLifespanLessThan10Minutes; -static Counter64 cursorStatsLifespanGreaterThanOrEqual10Minutes; - -static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan1Second( - "cursor.lifespan.lessThan1Second", &cursorStatsLifespanLessThan1Second); -static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan5Seconds( - "cursor.lifespan.lessThan5Seconds", &cursorStatsLifespanLessThan5Seconds); -static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan15Seconds( - "cursor.lifespan.lessThan15Seconds", &cursorStatsLifespanLessThan15Seconds); -static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan30Seconds( - "cursor.lifespan.lessThan30Seconds", &cursorStatsLifespanLessThan30Seconds); -static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan1Minute( - "cursor.lifespan.lessThan1Minute", &cursorStatsLifespanLessThan1Minute); -static ServerStatusMetricField<Counter64> dCursorStatsLifespanLessThan10Minutes( - "cursor.lifespan.lessThan10Minutes", &cursorStatsLifespanLessThan10Minutes); -static ServerStatusMetricField<Counter64> dCursorStatsLifespanGreaterThanOrEqual10Minutes( - "cursor.lifespan.greaterThanOrEqual10Minutes", &cursorStatsLifespanGreaterThanOrEqual10Minutes); - constexpr int CursorManager::kNumPartitions; namespace { @@ -96,26 +74,6 @@ ServiceContext::ConstructorActionRegisterer cursorManagerRegisterer{ auto cursorManager = std::make_unique<CursorManager>(svcCtx->getPreciseClockSource()); CursorManager::set(svcCtx, std::move(cursorManager)); }}; - -void incrementCursorLifespanMetric(Date_t birth, Date_t death) { - auto elapsed = death - birth; - - if (elapsed < Seconds(1)) { - cursorStatsLifespanLessThan1Second.increment(); - } else if (elapsed < Seconds(5)) { - cursorStatsLifespanLessThan5Seconds.increment(); - } else if (elapsed < Seconds(15)) { - cursorStatsLifespanLessThan15Seconds.increment(); - } else if (elapsed < Seconds(30)) { - cursorStatsLifespanLessThan30Seconds.increment(); - } else if (elapsed < Minutes(1)) { - cursorStatsLifespanLessThan1Minute.increment(); - } else if (elapsed < Minutes(10)) { - cursorStatsLifespanLessThan10Minutes.increment(); - } else { - cursorStatsLifespanGreaterThanOrEqual10Minutes.increment(); - } -} } // namespace CursorManager* CursorManager::get(ServiceContext* svcCtx) { @@ -161,7 +119,7 @@ CursorManager::~CursorManager() { for (auto&& cursor : *partition) { // Callers must ensure that no cursors are in use. invariant(!cursor.second->_operationUsingCursor); - cursor.second->dispose(nullptr); + cursor.second->dispose(nullptr, boost::none); delete cursor.second; } } @@ -201,7 +159,7 @@ std::size_t CursorManager::timeoutCursors(OperationContext* opCtx, Date_t now) { "Cursor timed out", "cursorId"_attr = cursor->cursorid(), "idleSince"_attr = cursor->getLastUseDate()); - cursor->dispose(opCtx); + cursor->dispose(opCtx, boost::none); } return toDisposeWithoutMutex.size(); } @@ -263,6 +221,11 @@ StatusWith<ClientCursorPin> CursorManager::pinCursor( CurOp::get(opCtx)->debug().queryHash = cursor->_queryHash; CurOp::get(opCtx)->debug().planCacheKey = cursor->_planCacheKey; + // Pass along queryStats context so it is retrievable after query execution for storing metrics. + CurOp::get(opCtx)->debug().queryStatsInfo.keyHash = cursor->_queryStatsKeyHash; + CurOp::get(opCtx)->debug().queryStatsInfo.willNeverExhaust = + cursor->_queryStatsWillNeverExhaust; + cursor->_operationUsingCursor = opCtx; // We use pinning of a cursor as a proxy for active, user-initiated use of a cursor. Therefore, @@ -376,8 +339,11 @@ stdx::unordered_set<CursorId> CursorManager::getCursorsForOpKeys( stdx::lock_guard<Latch> lk(_opKeyMutex); for (auto opKey : opKeys) { - if (auto it = _opKeyMap.find(opKey); it != _opKeyMap.end()) - cursors.insert(it->second); + if (auto it = _opKeyMap.find(opKey); it != _opKeyMap.end()) { + for (auto cursor : it->second) { + cursors.insert(cursor); + } + } } return cursors; } @@ -424,7 +390,12 @@ ClientCursorPin CursorManager::registerCursor(OperationContext* opCtx, // If set, store the mapping of OperationKey to the generated CursorID. if (auto opKey = opCtx->getOperationKey()) { stdx::lock_guard<Latch> lk(_opKeyMutex); - _opKeyMap.emplace(*opKey, cursorId); + auto it = _opKeyMap.find(*opKey); + if (it != _opKeyMap.end()) { + it->second.insert(cursorId); + } else { + _opKeyMap.emplace(*opKey, std::set<CursorId>{cursorId}); + } } // Restores the maxTimeMS provided in the cursor generating command in the case it used @@ -435,26 +406,41 @@ ClientCursorPin CursorManager::registerCursor(OperationContext* opCtx, return ClientCursorPin(opCtx, unownedCursor, this); } -void CursorManager::deregisterCursor(ClientCursor* cursor) { - removeCursorFromMap(_cursorMap, cursor); - incrementCursorLifespanMetric(cursor->_createdDate, _preciseClockSource->now()); +// Note the following subleties of the implementations of deregisterAndDestroyCursor: +// - We must make sure the cursor is unpinned (by clearing the '_operationUsingCursor' field) before +// destruction, since it is an error to delete a pinned cursor. +// - In addition, we must deregister the cursor from the manager's map before clearing the +// '_operationUsingCursor' field, since it is an error to unpin a registered cursor without +// holidng the appropriate cursor manager mutex. By first deregistering the cursor, we ensure that +// no other thread can access '_cursor', meaning that it is safe for us to write to +// '_operationUsingCursor' without holding the CursorManager mutex. +void CursorManager::deregisterAndDestroyCursor( + OperationContext* opCtx, std::unique_ptr<ClientCursor, ClientCursor::Deleter> cursor) { + removeCursorFromMap(_cursorMap, cursor.get()); + _destroyCursor(opCtx, std::move(cursor)); } void CursorManager::deregisterAndDestroyCursor( Partitioned<stdx::unordered_map<CursorId, ClientCursor*>>::OnePartition&& lk, OperationContext* opCtx, std::unique_ptr<ClientCursor, ClientCursor::Deleter> cursor) { + // Restrict the scope of the lock so we can destroy the cursor without holding any cursor + // manager mutexes. { auto lockWithRestrictedScope = std::move(lk); removeCursorFromMap(lockWithRestrictedScope, cursor.get()); } + _destroyCursor(opCtx, std::move(cursor)); +} - incrementCursorLifespanMetric(cursor->_createdDate, _preciseClockSource->now()); +void CursorManager::_destroyCursor(OperationContext* opCtx, + std::unique_ptr<ClientCursor, ClientCursor::Deleter> cursor) { // Dispose of the cursor without holding any cursor manager mutexes. Disposal of a cursor can // require taking lock manager locks, which we want to avoid while holding a mutex. If we did // so, any caller of a CursorManager method which already held a lock manager lock could induce // a deadlock when trying to acquire a CursorManager lock. - cursor->dispose(opCtx); + cursor->dispose(opCtx, _preciseClockSource->now()); + cursor->_operationUsingCursor = nullptr; } Status CursorManager::killCursor(OperationContext* opCtx, CursorId id) { diff --git a/src/mongo/db/cursor_manager.h b/src/mongo/db/cursor_manager.h index 05854c7e17d..3111898557d 100644 --- a/src/mongo/db/cursor_manager.h +++ b/src/mongo/db/cursor_manager.h @@ -142,10 +142,7 @@ public: * operation next checks for interruption. * Case (2) will only occur if the cursor is pinned. * - * Returns ErrorCodes::CursorNotFound if the cursor id is not owned by this manager. Returns - * ErrorCodes::OperationFailed if attempting to erase a pinned cursor. - * - * If 'shouldAudit' is true, will perform audit logging. + * Returns ErrorCodes::CursorNotFound if the cursor id is not owned by this manager. */ Status killCursor(OperationContext* opCtx, CursorId id); @@ -215,14 +212,17 @@ private: static constexpr int kNumPartitions = 16; friend class ClientCursorPin; - ClientCursorPin _registerCursor( - OperationContext* opCtx, std::unique_ptr<ClientCursor, ClientCursor::Deleter> clientCursor); - - void deregisterCursor(ClientCursor* cursor); + // deregisterAndDestroyCursor deregisters the cursor from the manager's cursorMap, then safely + // destroys the cursor. The first overload requires having acquired the cursor manager partition + // lock already. void deregisterAndDestroyCursor( - Partitioned<stdx::unordered_map<CursorId, ClientCursor*>>::OnePartition&&, + Partitioned<stdx::unordered_map<CursorId, ClientCursor*>>::OnePartition&& lk, OperationContext* opCtx, std::unique_ptr<ClientCursor, ClientCursor::Deleter> cursor); + void deregisterAndDestroyCursor(OperationContext* opCtx, + std::unique_ptr<ClientCursor, ClientCursor::Deleter> cursor); + void _destroyCursor(OperationContext* opCtx, + std::unique_ptr<ClientCursor, ClientCursor::Deleter> cursor); void unpin(OperationContext* opCtx, std::unique_ptr<ClientCursor, ClientCursor::Deleter> cursor); @@ -235,7 +235,13 @@ private: // 'cursor'. if (auto opKey = cursor->getOperationKey()) { stdx::lock_guard<Latch> lk(_opKeyMutex); - _opKeyMap.erase(*opKey); + auto it = _opKeyMap.find(*opKey); + if (it != _opKeyMap.end()) { + it->second.erase(cursor->cursorid()); + if (it->second.empty()) { + _opKeyMap.erase(*opKey); + } + } } map->erase(cursor->cursorid()); } @@ -263,7 +269,7 @@ private: // cursors in the map above are not present in this map, since OperationKey is not required when // registering a cursor. mutable Mutex _opKeyMutex = MONGO_MAKE_LATCH("CursorManager::_opKeyMutex"); - stdx::unordered_map<OperationKey, CursorId, UUID::Hash> _opKeyMap; + stdx::unordered_map<OperationKey, std::set<CursorId>, UUID::Hash> _opKeyMap; ClockSource* _preciseClockSource; }; diff --git a/src/mongo/db/db_raii.cpp b/src/mongo/db/db_raii.cpp index 581f038afc8..bd906e36f15 100644 --- a/src/mongo/db/db_raii.cpp +++ b/src/mongo/db/db_raii.cpp @@ -936,6 +936,18 @@ const NamespaceString& AutoGetCollectionForReadCommandMaybeLockFree::getNss() co } } +query_shape::CollectionType AutoGetCollectionForReadCommandMaybeLockFree::getCollectionType() + const { + if (auto&& view = getView()) { + return view->timeseries() ? query_shape::CollectionType::kTimeseries + : query_shape::CollectionType::kView; + } + auto&& collection = getCollection(); + return collection ? query_shape::CollectionType::kCollection + : query_shape::CollectionType::kNonExistent; +} + + bool AutoGetCollectionForReadCommandMaybeLockFree::isAnySecondaryNamespaceAViewOrSharded() const { return _autoGet ? _autoGet->isAnySecondaryNamespaceAViewOrSharded() : _autoGetLockFree->isAnySecondaryNamespaceAViewOrSharded(); diff --git a/src/mongo/db/db_raii.h b/src/mongo/db/db_raii.h index 55e96aea833..e1d9be6371b 100644 --- a/src/mongo/db/db_raii.h +++ b/src/mongo/db/db_raii.h @@ -32,6 +32,7 @@ #include <string> #include "mongo/db/catalog_raii.h" +#include "mongo/db/collection_type.h" #include "mongo/db/stats/top.h" #include "mongo/util/timer.h" @@ -469,6 +470,7 @@ public: return getCollection(); } const CollectionPtr& getCollection() const; + query_shape::CollectionType getCollectionType() const; const ViewDefinition* getView() const; const NamespaceString& getNss() const; bool isAnySecondaryNamespaceAViewOrSharded() const; diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index 64412e5a2fd..605f26c8ef9 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -298,6 +298,7 @@ UpdateResult Helpers::upsert(OperationContext* opCtx, request.setUpsert(); if (fromMigrate) { request.setSource(OperationSource::kFromMigrate); + request.setBypassEmptyTsReplacement(true); } request.setYieldPolicy(PlanYieldPolicy::YieldPolicy::NO_YIELD); @@ -319,6 +320,7 @@ void Helpers::update(OperationContext* opCtx, request.setUpdateModification(write_ops::UpdateModification::parseFromClassicUpdate(updateMod)); if (fromMigrate) { request.setSource(OperationSource::kFromMigrate); + request.setBypassEmptyTsReplacement(true); } request.setYieldPolicy(PlanYieldPolicy::YieldPolicy::NO_YIELD); diff --git a/src/mongo/db/dbmessage.cpp b/src/mongo/db/dbmessage.cpp index bba632c1ff1..04d8e756916 100644 --- a/src/mongo/db/dbmessage.cpp +++ b/src/mongo/db/dbmessage.cpp @@ -148,7 +148,7 @@ Message makeUnsupportedOpInsertMessage(StringData ns, reservedFlags |= InsertOption_ContinueOnError; b.appendNum(reservedFlags); - b.appendStr(ns); + b.appendCStr(ns); for (size_t i = 0; i < count; i++) { objs[i].appendSelfToBufBuilder(b); diff --git a/src/mongo/db/dbmessage_test.cpp b/src/mongo/db/dbmessage_test.cpp index 73354253738..cf310947265 100644 --- a/src/mongo/db/dbmessage_test.cpp +++ b/src/mongo/db/dbmessage_test.cpp @@ -101,7 +101,7 @@ TEST(DBMessage1, GoodInsert) { string ns("test"); b.appendNum(static_cast<int>(1)); - b.appendStr(ns); + b.appendCStr(ns); b.appendNum(static_cast<int>(3)); b.appendNum(static_cast<int>(39)); @@ -120,7 +120,7 @@ TEST(DBMessage1, GoodInsert2) { string ns("test"); b.appendNum(static_cast<int>(1)); - b.appendStr(ns); + b.appendCStr(ns); b.appendNum(static_cast<int>(3)); b.appendNum(static_cast<int>(39)); diff --git a/src/mongo/db/exec/SConscript b/src/mongo/db/exec/SConscript index eac1c6a3105..fb803f2cb53 100644 --- a/src/mongo/db/exec/SConscript +++ b/src/mongo/db/exec/SConscript @@ -130,6 +130,7 @@ env.CppUnitTest( "find_projection_executor_test.cpp", "inclusion_projection_executor_test.cpp", "projection_executor_builder_test.cpp", + "projection_executor_redaction_test.cpp", "projection_executor_test.cpp", "projection_executor_utils_test.cpp", "projection_executor_wildcard_access_test.cpp", diff --git a/src/mongo/db/exec/add_fields_projection_executor.h b/src/mongo/db/exec/add_fields_projection_executor.h index 12f7bbfe19b..b1349d715d6 100644 --- a/src/mongo/db/exec/add_fields_projection_executor.h +++ b/src/mongo/db/exec/add_fields_projection_executor.h @@ -94,9 +94,9 @@ public: */ void parse(const BSONObj& spec); - Document serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const final { - return _root->serialize(explain); + Document serializeTransformation(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options = {}) const final { + return _root->serialize(explain, options); } /** diff --git a/src/mongo/db/exec/bucket_unpacker.cpp b/src/mongo/db/exec/bucket_unpacker.cpp index db14d8b08cc..8bba2da9e4d 100644 --- a/src/mongo/db/exec/bucket_unpacker.cpp +++ b/src/mongo/db/exec/bucket_unpacker.cpp @@ -986,9 +986,9 @@ BSONObj BucketSpec::pushdownPredicate( BSONObjBuilder result; if (metaOnlyPredicate) - metaOnlyPredicate->serialize(&result); + metaOnlyPredicate->serialize(&result, {}); if (bucketMetricPredicate) - bucketMetricPredicate->serialize(&result); + bucketMetricPredicate->serialize(&result, {}); return result.obj(); } @@ -1546,7 +1546,9 @@ BSONObj BucketUnpacker::getNextBson() { // Add computed meta projections. for (auto&& name : _spec.computedMetaProjFields()) { - builder.appendAs(_computedMetaProjections[name], name); + if (_computedMetaProjections[name]) { + builder.appendAs(_computedMetaProjections[name], name); + } } return builder.obj(); @@ -1757,12 +1759,18 @@ void BucketUnpacker::eraseMetaFromFieldSetAndDetermineIncludeMeta() { } } -void BucketUnpacker::eraseExcludedComputedMetaProjFields() { - if (_spec.behavior() == BucketSpec::Behavior::kExclude) { - for (const auto& field : _spec.fieldSet()) { - _spec.eraseFromComputedMetaProjFields(field); - } - } +void BucketUnpacker::eraseUnneededComputedMetaProjFields() { + // If this is an inclusion spec and the current computed field is not part of in the include + // fields, it means the computed field should not be available after the current unpack stage. + // Similarly, for exclusion spec, if the current computed field is part of the exclude fields, + // the computed fields should not be available after the current unpack stage. This can happen + // if there was a $project stage after a $addFields stage. + bool removeIfInFieldSet = _spec.behavior() == BucketSpec::Behavior::kExclude; + auto conditionToErase = [&](const std::string& computedField) { + bool inFieldSet = _spec.fieldSet().find(computedField) != _spec.fieldSet().end(); + return inFieldSet == removeIfInFieldSet; + }; + _spec.eraseIfPredTrueFromComputedMetaProjFields(conditionToErase); } void BucketUnpacker::setBucketSpec(BucketSpec&& bucketSpec) { @@ -1770,7 +1778,7 @@ void BucketUnpacker::setBucketSpec(BucketSpec&& bucketSpec) { eraseMetaFromFieldSetAndDetermineIncludeMeta(); determineIncludeTimeField(); - eraseExcludedComputedMetaProjFields(); + eraseUnneededComputedMetaProjFields(); _includeMinTimeAsMetadata = _spec.includeMinTimeAsMetadata; _includeMaxTimeAsMetadata = _spec.includeMaxTimeAsMetadata; diff --git a/src/mongo/db/exec/bucket_unpacker.h b/src/mongo/db/exec/bucket_unpacker.h index 29f2f1f30d2..3c63d6abe49 100644 --- a/src/mongo/db/exec/bucket_unpacker.h +++ b/src/mongo/db/exec/bucket_unpacker.h @@ -115,8 +115,16 @@ public: return _computedMetaProjFields; } - void eraseFromComputedMetaProjFields(const std::string& field) { - _computedMetaProjFields.erase(field); + // Remove fields that the predicate function evaluates to true for. + void eraseIfPredTrueFromComputedMetaProjFields(const std::function<bool(std::string)> pred) { + auto it = _computedMetaProjFields.begin(); + while (it != _computedMetaProjFields.end()) { + if (pred(*it)) { + it = _computedMetaProjFields.erase(it); + } else { + ++it; + } + } } void setUsesExtendedRange(bool usesExtendedRange) { @@ -374,8 +382,9 @@ private: // included in the materialized measurements. void eraseMetaFromFieldSetAndDetermineIncludeMeta(); - // Erase computed meta projection fields if they are present in the exclusion field set. - void eraseExcludedComputedMetaProjFields(); + // Erase computed meta projection fields if they are present in the exclusion field set or if + // they are not present in the inclusion set. + void eraseUnneededComputedMetaProjFields(); BucketSpec _spec; diff --git a/src/mongo/db/exec/bucket_unpacker_test.cpp b/src/mongo/db/exec/bucket_unpacker_test.cpp index 9eecea624c5..4a638467bea 100644 --- a/src/mongo/db/exec/bucket_unpacker_test.cpp +++ b/src/mongo/db/exec/bucket_unpacker_test.cpp @@ -170,6 +170,11 @@ public: } return root.obj(); } + + bool computedMetaProjFieldsContainsField(std::set<std::string>& computedMetaProjFields, + std::string field) { + return computedMetaProjFields.find(field) != computedMetaProjFields.end(); + } }; TEST_F(BucketUnpackerTest, UnpackBasicIncludeAllMeasurementFields) { @@ -178,7 +183,7 @@ TEST_F(BucketUnpackerTest, UnpackBasicIncludeAllMeasurementFields) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"); auto unpacker = makeBucketUnpacker(std::move(fields), @@ -188,11 +193,12 @@ TEST_F(BucketUnpackerTest, UnpackBasicIncludeAllMeasurementFields) { ASSERT_TRUE(unpacker.hasNext()); assertGetNext(unpacker, - Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}); + Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, - Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}); + assertGetNext( + unpacker, + Document{fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}); ASSERT_FALSE(unpacker.hasNext()); } @@ -201,7 +207,7 @@ TEST_F(BucketUnpackerTest, ExcludeASingleField) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"); auto test = [&](BSONObj bucket) { @@ -211,12 +217,14 @@ TEST_F(BucketUnpackerTest, ExcludeASingleField) { kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, - Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}); + assertGetNext( + unpacker, + Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, - Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2}")}); + assertGetNext( + unpacker, + Document{fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -229,7 +237,7 @@ TEST_F(BucketUnpackerTest, EmptyIncludeGetsEmptyMeasurements) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"); auto test = [&](BSONObj bucket) { @@ -255,7 +263,7 @@ TEST_F(BucketUnpackerTest, EmptyExcludeMaterializesAllFields) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"); auto test = [&](BSONObj bucket) { @@ -264,13 +272,14 @@ TEST_F(BucketUnpackerTest, EmptyExcludeMaterializesAllFields) { std::move(bucket), kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, - Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}); + assertGetNext( + unpacker, + Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}); ASSERT_TRUE(unpacker.hasNext()); assertGetNext( unpacker, - Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}); + Document{fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -283,7 +292,7 @@ TEST_F(BucketUnpackerTest, SparseColumnsWhereOneColumnIsExhaustedBeforeTheOther) auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1}, b:{'1':1}}}"); auto test = [&](BSONObj bucket) { @@ -292,11 +301,13 @@ TEST_F(BucketUnpackerTest, SparseColumnsWhereOneColumnIsExhaustedBeforeTheOther) std::move(bucket), kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, - Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}); + assertGetNext( + unpacker, + Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, - Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, b: 1}")}); + assertGetNext( + unpacker, + Document{fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, b: 1}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -310,7 +321,7 @@ TEST_F(BucketUnpackerTest, UnpackBasicIncludeWithDollarPrefix) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "$a:{'0':1, '1':2}, b:{'1':1}}}"); auto test = [&](BSONObj bucket) { @@ -319,13 +330,14 @@ TEST_F(BucketUnpackerTest, UnpackBasicIncludeWithDollarPrefix) { std::move(bucket), kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, - Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, $a: 1}")}); - - ASSERT_TRUE(unpacker.hasNext()); assertGetNext( unpacker, - Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, $a: 2, b: 1}")}); + Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, $a: 1}")}); + + ASSERT_TRUE(unpacker.hasNext()); + assertGetNext(unpacker, + Document{fromjson( + "{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, $a: 2, b: 1}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -338,7 +350,7 @@ TEST_F(BucketUnpackerTest, BucketsWithMetadataOnly) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}}}"); + "time: {'0':Date(1), '1':Date(2)}}}"); auto test = [&](BSONObj bucket) { auto unpacker = makeBucketUnpacker(fields, @@ -347,10 +359,10 @@ TEST_F(BucketUnpackerTest, BucketsWithMetadataOnly) { kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); assertGetNext(unpacker, - Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}")}); + Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}")}); ASSERT_TRUE(unpacker.hasNext()); assertGetNext(unpacker, - Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}")}); + Document{fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -363,8 +375,8 @@ TEST_F(BucketUnpackerTest, UnorderedRowKeysDoesntAffectMaterialization) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'1':1, '0':2, '2': " - "3}, time: {'1':1, '0': 2, " - "'2': 3}}}"); + "3}, time: {'1':Date(1), '0': Date(2), " + "'2': Date(3)}}}"); auto test = [&](BSONObj bucket) { auto unpacker = makeBucketUnpacker(fields, @@ -373,15 +385,15 @@ TEST_F(BucketUnpackerTest, UnorderedRowKeysDoesntAffectMaterialization) { kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); assertGetNext(unpacker, - Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}")}); + Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}")}); ASSERT_TRUE(unpacker.hasNext()); assertGetNext(unpacker, - Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}")}); + Document{fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}")}); ASSERT_TRUE(unpacker.hasNext()); assertGetNext(unpacker, - Document{fromjson("{time: 3, myMeta: {m1: 999, m2: 9999}, _id: 3}")}); + Document{fromjson("{time: Date(3), myMeta: {m1: 999, m2: 9999}, _id: 3}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -393,8 +405,9 @@ TEST_F(BucketUnpackerTest, MissingMetaFieldDoesntMaterializeMetadata) { std::set<std::string> fields{}; auto bucket = fromjson( - "{control: {'version': 1}, data: {_id: {'0':1, '1':2, '2': 3}, time: {'0':1, '1': 2, '2': " - "3}}}"); + "{control: {'version': 1}, data: {_id: {'0':1, '1':2, '2': 3}, time: {'0':Date(1), '1': " + "Date(2), '2': " + "Date(3)}}}"); auto test = [&](BSONObj bucket) { auto unpacker = makeBucketUnpacker(fields, @@ -402,13 +415,13 @@ TEST_F(BucketUnpackerTest, MissingMetaFieldDoesntMaterializeMetadata) { std::move(bucket), kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 1, _id: 1}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(1), _id: 1}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 2, _id: 2}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(2), _id: 2}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 3, _id: 3}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(3), _id: 3}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -420,8 +433,9 @@ TEST_F(BucketUnpackerTest, MissingMetaFieldDoesntMaterializeMetadataUnorderedKey std::set<std::string> fields{}; auto bucket = fromjson( - "{control: {'version': 1}, data: {_id: {'1':1, '0':2, '2': 3}, time: {'1':1, '0': 2, '2': " - "3}}}"); + "{control: {'version': 1}, data: {_id: {'1':1, '0':2, '2': 3}, time: {'1':Date(1), '0': " + "Date(2), '2': " + "Date(3)}}}"); auto test = [&](BSONObj bucket) { auto unpacker = makeBucketUnpacker(fields, @@ -429,13 +443,13 @@ TEST_F(BucketUnpackerTest, MissingMetaFieldDoesntMaterializeMetadataUnorderedKey std::move(bucket), kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 1, _id: 1}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(1), _id: 1}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 2, _id: 2}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(2), _id: 2}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 3, _id: 3}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(3), _id: 3}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -448,8 +462,8 @@ TEST_F(BucketUnpackerTest, ExcludedMetaFieldDoesntMaterializeMetadataWhenBucketH auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2, '2': " - "3}, time: {'0':1, '1': 2, " - "'2': 3}}}"); + "3}, time: {'0':Date(1), '1': Date(2), " + "'2': Date(3)}}}"); auto test = [&](BSONObj bucket) { auto unpacker = makeBucketUnpacker(fields, @@ -457,13 +471,13 @@ TEST_F(BucketUnpackerTest, ExcludedMetaFieldDoesntMaterializeMetadataWhenBucketH std::move(bucket), kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 1, _id: 1}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(1), _id: 1}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 2, _id: 2}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(2), _id: 2}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 3, _id: 3}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(3), _id: 3}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -476,7 +490,7 @@ TEST_F(BucketUnpackerTest, UnpackerResetThrowsOnUndefinedMeta) { auto bucket = fromjson( "{control: {'version': 1}, meta: undefined, data: {_id: {'0':1, '1':2, '2': 3}, time: " - "{'0':1, '1': 2, '2': 3}}}"); + "{'0':Date(1), '1': Date(2), '2': Date(3)}}}"); auto test = [&](BSONObj bucket) { assertUnpackerThrowsCode(fields, @@ -495,8 +509,8 @@ TEST_F(BucketUnpackerTest, UnpackerResetThrowsOnUnexpectedMeta) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2, '2': " - "3}, time: {'0':1, '1': 2, " - "'2': 3}}}"); + "3}, time: {'0':Date(1), '1': Date(2), " + "'2': Date(3)}}}"); auto test = [&](BSONObj bucket) { assertUnpackerThrowsCode(fields, @@ -514,8 +528,9 @@ TEST_F(BucketUnpackerTest, NullMetaInBucketMaterializesAsNull) { std::set<std::string> fields{}; auto bucket = fromjson( - "{control: {'version': 1}, meta: null, data: {_id: {'0':4, '1':5, '2':6}, time: {'0':4, " - "'1': 5, '2': 6}}}"); + "{control: {'version': 1}, meta: null, data: {_id: {'0':4, '1':5, '2':6}, time: " + "{'0':Date(4), " + "'1': Date(5), '2': Date(6)}}}"); auto test = [&](BSONObj bucket) { auto unpacker = makeBucketUnpacker(fields, @@ -523,13 +538,13 @@ TEST_F(BucketUnpackerTest, NullMetaInBucketMaterializesAsNull) { std::move(bucket), kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 4, myMeta: null, _id: 4}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(4), myMeta: null, _id: 4}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 5, myMeta: null, _id: 5}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(5), myMeta: null, _id: 5}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 6, myMeta: null, _id: 6}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(6), myMeta: null, _id: 6}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -545,7 +560,7 @@ TEST_F(BucketUnpackerTest, GetNextHandlesMissingMetaInBucket) { control: {version: 1}, data: { _id: {'0':4, '1':5, '2':6}, - time: {'0':4, '1': 5, '2': 6} + time: {'0':Date(4), '1': Date(5), '2': Date(6)} } })"); @@ -555,13 +570,13 @@ TEST_F(BucketUnpackerTest, GetNextHandlesMissingMetaInBucket) { std::move(bucket), kUserDefinedMetaName.toString()); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 4, _id: 4}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(4), _id: 4}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 5, _id: 5}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(5), _id: 5}")}); ASSERT_TRUE(unpacker.hasNext()); - assertGetNext(unpacker, Document{fromjson("{time: 6, _id: 6}")}); + assertGetNext(unpacker, Document{fromjson("{time: Date(6), _id: 6}")}); ASSERT_FALSE(unpacker.hasNext()); }; @@ -606,7 +621,7 @@ TEST_F(BucketUnpackerTest, EraseMetaFromFieldSetAndDetermineIncludeMeta) { control: {version: 1}, data: { _id: {'0':4, '1':5, '2':6}, - time: {'0':4, '1': 5, '2': 6} + time: {'0':Date(4), '1': Date(5), '2': Date(6)} } })"); auto unpacker = makeBucketUnpacker(empFields, @@ -659,13 +674,83 @@ TEST_F(BucketUnpackerTest, EraseMetaFromFieldSetAndDetermineIncludeMeta) { ASSERT_FALSE(unpacker.includeMetaField()); } +TEST_F(BucketUnpackerTest, EraseUnneededComputedMetaProjFieldsWithInclusiveProject) { + auto bucket = fromjson(R"( +{ + control: {version: 1}, + data: { + _id: {'0':4, '1':5, '2':6}, + time: {'0':Date(4), '1': Date(5), '2': Date(6)} + } +})"); + std::set<std::string> unpackerFields{kUserDefinedTimeName.toString()}; + auto unpacker = makeBucketUnpacker(unpackerFields, + BucketSpec::Behavior::kInclude, + std::move(bucket), + kUserDefinedMetaName.toString()); + + // Add fields to '_computedMetaProjFields'. + unpacker.addComputedMetaProjFields({"hello"_sd, "bye"_sd}); + auto computedMetaProjFields = unpacker.bucketSpec().computedMetaProjFields(); + ASSERT_TRUE(computedMetaProjFieldsContainsField(computedMetaProjFields, "hello")); + ASSERT_TRUE(computedMetaProjFieldsContainsField(computedMetaProjFields, "bye")); + + auto spec = unpacker.bucketSpec(); + std::set<std::string> includeFields{kUserDefinedTimeName.toString(), "bye"}; + spec.setFieldSet(includeFields); + spec.setBehavior(BucketSpec::Behavior::kInclude); + + // This calls eraseUnneededComputedMetaProjFields(). + unpacker.setBucketSpec(std::move(spec)); + computedMetaProjFields = unpacker.bucketSpec().computedMetaProjFields(); + // As "hello" was not in the includes, it should be removed. + ASSERT_FALSE(computedMetaProjFieldsContainsField(computedMetaProjFields, "hello")); + // As "bye" was in the includes, it should still be in '_computedMetaProjFields'. + ASSERT_TRUE(computedMetaProjFieldsContainsField(computedMetaProjFields, "bye")); +} + +TEST_F(BucketUnpackerTest, EraseUnneededComputedMetaProjFieldsWithExclusiveProject) { + auto bucket = fromjson(R"( +{ + control: {version: 1}, + data: { + _id: {'0':4, '1':5, '2':6}, + time: {'0':Date(4), '1': Date(5), '2': Date(6)} + } +})"); + std::set<std::string> unpackerFields{kUserDefinedTimeName.toString()}; + auto unpacker = makeBucketUnpacker(unpackerFields, + BucketSpec::Behavior::kInclude, + std::move(bucket), + kUserDefinedMetaName.toString()); + + // Add fields to '_computedMetaProjFields'. + unpacker.addComputedMetaProjFields({"hello"_sd, "bye"_sd}); + auto computedMetaProjFields = unpacker.bucketSpec().computedMetaProjFields(); + ASSERT_TRUE(computedMetaProjFieldsContainsField(computedMetaProjFields, "hello")); + ASSERT_TRUE(computedMetaProjFieldsContainsField(computedMetaProjFields, "bye")); + + auto spec = unpacker.bucketSpec(); + std::set<std::string> excludeFields{kUserDefinedTimeName.toString(), "bye"}; + spec.setFieldSet(excludeFields); + spec.setBehavior(BucketSpec::Behavior::kExclude); + + // This calls eraseUnneededComputedMetaProjFields(). + unpacker.setBucketSpec(std::move(spec)); + computedMetaProjFields = unpacker.bucketSpec().computedMetaProjFields(); + // As "hello" was not excluded, it should still exist. + ASSERT_TRUE(computedMetaProjFieldsContainsField(computedMetaProjFields, "hello")); + // As "bye" was in the excludes, it should be removed from '_computedMetaProjFields'. + ASSERT_FALSE(computedMetaProjFieldsContainsField(computedMetaProjFields, "bye")); +} + TEST_F(BucketUnpackerTest, DetermineIncludeTimeField) { auto bucket = fromjson(R"( { control: {version: 1}, data: { _id: {'0':4, '1':5, '2':6}, - time: {'0':4, '1': 5, '2': 6} + time: {'0':Date(4), '1': Date(5), '2': Date(6)} } })"); std::set<std::string> unpackerFields{kUserDefinedTimeName.toString()}; @@ -900,7 +985,7 @@ TEST_F(BucketUnpackerTest, TamperedCompressedCountLess) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"); auto compressedBucket = @@ -913,8 +998,9 @@ TEST_F(BucketUnpackerTest, TamperedCompressedCountLess) { std::move(modifiedCompressedBucket), kUserDefinedMetaName.toString()); - auto doc0 = Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; - auto doc1 = Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}; + auto doc0 = Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; + auto doc1 = + Document{fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}; // 1 is reported when asking for numberOfMeasurements() ASSERT_EQ(unpacker.numberOfMeasurements(), 1); @@ -935,7 +1021,7 @@ TEST_F(BucketUnpackerTest, TamperedCompressedCountMore) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"); auto compressedBucket = @@ -948,8 +1034,9 @@ TEST_F(BucketUnpackerTest, TamperedCompressedCountMore) { std::move(modifiedCompressedBucket), kUserDefinedMetaName.toString()); - auto doc0 = Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; - auto doc1 = Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}; + auto doc0 = Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; + auto doc1 = + Document{fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}; ASSERT_EQ(unpacker.numberOfMeasurements(), 3); ASSERT_DOCUMENT_EQ(unpacker.extractSingleMeasurement(0), doc0); @@ -970,7 +1057,7 @@ TEST_F(BucketUnpackerTest, TamperedCompressedCountMissing) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"); auto compressedBucket = @@ -983,8 +1070,9 @@ TEST_F(BucketUnpackerTest, TamperedCompressedCountMissing) { std::move(modifiedCompressedBucket), kUserDefinedMetaName.toString()); - auto doc0 = Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; - auto doc1 = Document{fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}; + auto doc0 = Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; + auto doc1 = + Document{fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a :2, b: 1}")}; // Missing count field will make the unpacker measure the number of time fields for an accurate // count @@ -1006,7 +1094,7 @@ TEST_F(BucketUnpackerTest, TamperedCompressedElementMismatchDataField) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"); auto compressedBucket = @@ -1020,7 +1108,7 @@ TEST_F(BucketUnpackerTest, TamperedCompressedElementMismatchDataField) { std::move(modifiedCompressedBucket), kUserDefinedMetaName.toString()); - auto doc0 = Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; + auto doc0 = Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; ASSERT_EQ(unpacker.numberOfMeasurements(), 2); ASSERT_DOCUMENT_EQ(unpacker.extractSingleMeasurement(0), doc0); @@ -1041,7 +1129,7 @@ TEST_F(BucketUnpackerTest, TamperedCompressedElementMismatchTimeField) { auto bucket = fromjson( "{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"); auto compressedBucket = @@ -1055,7 +1143,7 @@ TEST_F(BucketUnpackerTest, TamperedCompressedElementMismatchTimeField) { std::move(modifiedCompressedBucket), kUserDefinedMetaName.toString()); - auto doc0 = Document{fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; + auto doc0 = Document{fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}")}; ASSERT_EQ(unpacker.numberOfMeasurements(), 2); ASSERT_DOCUMENT_EQ(unpacker.extractSingleMeasurement(0), doc0); diff --git a/src/mongo/db/exec/collection_scan.cpp b/src/mongo/db/exec/collection_scan.cpp index f8550dbe81d..877603a0965 100644 --- a/src/mongo/db/exec/collection_scan.cpp +++ b/src/mongo/db/exec/collection_scan.cpp @@ -275,19 +275,36 @@ void CollectionScan::setLatestOplogEntryTimestamp(const Record& record) { } void CollectionScan::assertTsHasNotFallenOffOplog(const Record& record) { - // If the first entry we see in the oplog is the replset initialization, then it doesn't matter - // if its timestamp is later than the timestamp that should not have fallen off the oplog; no - // events earlier can have fallen off this oplog. Otherwise, verify that the timestamp of the - // first observed oplog entry is earlier than or equal to timestamp that should not have fallen - // off the oplog. - auto oplogEntry = uassertStatusOK(repl::OplogEntry::parse(record.data.toBson())); + const auto oplogEntry = record.data.toBson(); + const repl::OplogEntryParserNonStrict oplogEntryParser{oplogEntry}; invariant(_specificStats.docsTested == 0); - const bool isNewRS = - oplogEntry.getObject().binaryEqual(BSON("msg" << repl::kInitiatingSetMsg)) && - oplogEntry.getOpType() == repl::OpTypeEnum::kNoop; + + // Indicates that 'oplogEntry' means initialization of a replica set. + bool isNewRS{false}; + + // Indicates that the timestamp of the observed oplog entry 'oplogEntry' is earlier than or + // equal to timestamp that should not have fallen off the oplog. + bool tsHasNotFallenOff{false}; + try { + tsHasNotFallenOff = + oplogEntryParser.getOpTime().getTimestamp() <= *_params.assertTsHasNotFallenOffOplog; + + // If the first entry we see in the oplog is the replset initialization, then it doesn't + // matter if its timestamp is later than the timestamp that should not have fallen off the + // oplog; no events earlier can have fallen off this oplog. + // NOTE: A change collection can be created at any moment as such it might not have replset + // initialization message, as such this case is not fully applicable for the change + // collection. + isNewRS = oplogEntryParser.getOpType() == repl::OpTypeEnum::kNoop && + oplogEntryParser.getObject().binaryEqual(BSON("msg" << repl::kInitiatingSetMsg)); + } catch (const AssertionException& exception) { + uasserted(8881102, + str::stream() << "Failed to parse the oldest oplog entry" << causedBy(exception)); + } uassert(ErrorCodes::OplogQueryMinTsMissing, "Specified timestamp has already fallen off the oplog", - isNewRS || oplogEntry.getTimestamp() <= *_params.assertTsHasNotFallenOffOplog); + isNewRS || tsHasNotFallenOff); + // We don't need to check this assertion again after we've confirmed the first oplog event. _params.assertTsHasNotFallenOffOplog = boost::none; } @@ -431,9 +448,7 @@ void CollectionScan::doReattachToOperationContext() { unique_ptr<PlanStageStats> CollectionScan::getStats() { // Add a BSON representation of the filter to the stats tree, if there is one. if (nullptr != _filter) { - BSONObjBuilder bob; - _filter->serialize(&bob); - _commonStats.filter = bob.obj(); + _commonStats.filter = _filter->serialize(); } unique_ptr<PlanStageStats> ret = std::make_unique<PlanStageStats>(_commonStats, STAGE_COLLSCAN); diff --git a/src/mongo/db/exec/collection_scan_common.h b/src/mongo/db/exec/collection_scan_common.h index 5770943229c..8aecb6750fc 100644 --- a/src/mongo/db/exec/collection_scan_common.h +++ b/src/mongo/db/exec/collection_scan_common.h @@ -42,12 +42,16 @@ struct CollectionScanParams { }; enum class ScanBoundInclusion { - kExcludeBothStartAndEndRecords, - kIncludeStartRecordOnly, - kIncludeEndRecordOnly, - kIncludeBothStartAndEndRecords, + kExcludeBothStartAndEndRecords = 0b00, + kIncludeStartRecordOnly = 0b01, + kIncludeEndRecordOnly = 0b10, + kIncludeBothStartAndEndRecords = 0b11, }; + static ScanBoundInclusion makeInclusion(bool startInclusive, bool endInclusive) { + return ScanBoundInclusion(int(startInclusive) | (int(endInclusive) << 1)); + } + // If present, this parameter sets the start point of a forward scan or the end point of a // reverse scan. A forward scan will start scanning at the document with the lowest RecordId // greater than or equal to minRecord. A reverse scan will stop and return EOF on the first diff --git a/src/mongo/db/exec/document_value/document.cpp b/src/mongo/db/exec/document_value/document.cpp index 7172370e4c3..2bcb637d17b 100644 --- a/src/mongo/db/exec/document_value/document.cpp +++ b/src/mongo/db/exec/document_value/document.cpp @@ -255,7 +255,8 @@ Value& DocumentStorage::appendField(T field, ValueElement::Kind kind) { append(nextCollision); append(nameSize); append(kind); - field.copyTo(dest, true); + dest += field.copy(dest, field.size()); + *dest++ = '\0'; // Like std::string, there is both an explicit size and final NUL byte. // Padding for alignment handled above #undef append @@ -432,6 +433,12 @@ Document DocumentStorage::shred() const { return md.freeze(); } +void DocumentStorage::loadIntoCache() const { + for (DocumentStorageIterator it = iterator(); !it.atEnd(); it.advance()) { + it.get(); + } +} + void DocumentStorage::loadLazyMetadata() const { if (_haveLazyLoadedMetadata) { return; @@ -819,7 +826,7 @@ void Document::serializeForSorter(BufBuilder& buf) const { buf.appendNum(static_cast<int>(numElems)); for (DocumentStorageIterator it = storage().iterator(); !it.atEnd(); it.advance()) { - buf.appendStr(it->nameSD(), /*NUL byte*/ true); + buf.appendCStr(it->nameSD()); it->val.serializeForSorter(buf); } diff --git a/src/mongo/db/exec/document_value/document.h b/src/mongo/db/exec/document_value/document.h index 6114aee792d..4ff1522f3a9 100644 --- a/src/mongo/db/exec/document_value/document.h +++ b/src/mongo/db/exec/document_value/document.h @@ -254,6 +254,13 @@ public: return storage().shred(); } + /** + * Loads the whole document into cache. + */ + void loadIntoCache() const { + return storage().loadIntoCache(); + } + /** Calculate a hash value. * * Meant to be used to create composite hashes suitable for diff --git a/src/mongo/db/exec/document_value/document_internal.h b/src/mongo/db/exec/document_value/document_internal.h index d7fb3b337d6..372130c3e68 100644 --- a/src/mongo/db/exec/document_value/document_internal.h +++ b/src/mongo/db/exec/document_value/document_internal.h @@ -296,8 +296,8 @@ public: return _sd.size(); } - inline void copyTo(char* dest, bool includeEndingNull) const { - return _sd.copyTo(dest, includeEndingNull); + inline size_t copy(char* dest, size_t len) const { + return _sd.copy(dest, len); } constexpr const char* rawData() const noexcept { @@ -385,6 +385,11 @@ public: */ Document shred() const; + /** + * Loads the whole document into cache. + */ + void loadIntoCache() const; + static const DocumentStorage& emptyDoc() { return kEmptyDoc; } diff --git a/src/mongo/db/exec/document_value/document_value_test.cpp b/src/mongo/db/exec/document_value/document_value_test.cpp index 80441c17f25..ca1123a7555 100644 --- a/src/mongo/db/exec/document_value/document_value_test.cpp +++ b/src/mongo/db/exec/document_value/document_value_test.cpp @@ -46,16 +46,6 @@ #include "mongo/dbtests/dbtests.h" #include "mongo/logv2/log.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - ::mongo::str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString() \ - << " Exception occured in: " << #EXPRESSION; \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace DocumentTests { using std::numeric_limits; diff --git a/src/mongo/db/exec/document_value/document_value_test_util.h b/src/mongo/db/exec/document_value/document_value_test_util.h index b6959a7d17f..7b88c9688fa 100644 --- a/src/mongo/db/exec/document_value/document_value_test_util.h +++ b/src/mongo/db/exec/document_value/document_value_test_util.h @@ -59,6 +59,13 @@ #define _ASSERT_DOCVAL_COMPARISON(NAME, a, b) \ ::mongo::unittest::assertComparison_##NAME(__FILE__, __LINE__, #a, #b, a, b) +// TODO SERVER-87736 make these not say "AUTO". +// These are backport-special macros, adapted from the "AUTO" version on more recent branches. The +// automatic functionality doesn't exist on this branch. But the assertions should still pass. +#define ASSERT_VALUE_EQ_AUTO(expected, val) ASSERT_EQ(expected, val.toString()) +#define ASSERT_DOCUMENT_EQ_AUTO(expected, actual) \ + ASSERT_BSONOBJ_EQ(fromjson(expected), actual.toBson()) + namespace mongo { namespace unittest { diff --git a/src/mongo/db/exec/document_value/value.cpp b/src/mongo/db/exec/document_value/value.cpp index 616efb2128a..a4f47313b49 100644 --- a/src/mongo/db/exec/document_value/value.cpp +++ b/src/mongo/db/exec/document_value/value.cpp @@ -117,7 +117,7 @@ void ValueStorage::putString(StringData s) { if (sizeNoNUL <= sizeof(shortStrStorage)) { shortStr = true; shortStrSize = s.size(); - s.copyTo(shortStrStorage, false); // no NUL + s.copy(shortStrStorage, s.size()); // All memory is zeroed before this is called, so we know that // the nulTerminator field will definitely contain a NUL byte. @@ -148,8 +148,9 @@ void ValueStorage::putRegEx(const BSONRegEx& re) { // Need to copy since putString doesn't support scatter-gather. std::unique_ptr<char[]> buf(new char[totalLen]); - re.pattern.copyTo(buf.get(), true); - re.flags.copyTo(buf.get() + patternLen + 1, false); // no NUL + auto dest = buf.get(); + dest = str::copyAsCString(dest, re.pattern); + re.flags.copy(dest, re.flags.size()); // NUL added automatically by putString() putString(StringData(buf.get(), totalLen)); } @@ -1310,7 +1311,7 @@ void Value::serializeForSorter(BufBuilder& buf) const { case Code: { StringData str = getRawData(); buf.appendNum(int(str.size())); - buf.appendStr(str, /*NUL byte*/ false); + buf.appendStrBytes(str); break; } @@ -1318,13 +1319,13 @@ void Value::serializeForSorter(BufBuilder& buf) const { StringData str = getRawData(); buf.appendChar(_storage.binDataType()); buf.appendNum(int(str.size())); - buf.appendStr(str, /*NUL byte*/ false); + buf.appendStrBytes(str); break; } case RegEx: - buf.appendStr(getRegex(), /*NUL byte*/ true); - buf.appendStr(getRegexFlags(), /*NUL byte*/ true); + buf.appendCStr(getRegex()); + buf.appendCStr(getRegexFlags()); break; case Object: @@ -1333,13 +1334,13 @@ void Value::serializeForSorter(BufBuilder& buf) const { case DBRef: buf.appendStruct(_storage.getDBRef()->oid); - buf.appendStr(_storage.getDBRef()->ns, /*NUL byte*/ true); + buf.appendCStr(_storage.getDBRef()->ns); break; case CodeWScope: { intrusive_ptr<const RCCodeWScope> cws = _storage.getCodeWScope(); buf.appendNum(int(cws->code.size())); - buf.appendStr(cws->code, /*NUL byte*/ false); + buf.appendStrBytes(cws->code); cws->scope.serializeForSorter(buf); break; } diff --git a/src/mongo/db/exec/document_value/value.h b/src/mongo/db/exec/document_value/value.h index a69494f1995..62a31f25727 100644 --- a/src/mongo/db/exec/document_value/value.h +++ b/src/mongo/db/exec/document_value/value.h @@ -430,13 +430,16 @@ public: ImplicitValue(T&& arg) : Value(std::forward<T>(arg)) {} ImplicitValue(std::initializer_list<ImplicitValue> values) : Value(convertToValues(values)) {} + ImplicitValue(std::vector<ImplicitValue> values) : Value(convertToValues(values)) {} - ImplicitValue(std::vector<int> values) : Value(convertToValues(values)) {} + template <typename T> + ImplicitValue(std::vector<T> values) : Value(convertToValues(values)) {} - static std::vector<Value> convertToValues(const std::vector<int>& vec) { + template <typename T> + static std::vector<Value> convertToValues(const std::vector<T>& vec) { std::vector<Value> values; values.reserve(vec.size()); - for_each(vec.begin(), vec.end(), ([&](const int& val) { values.emplace_back(val); })); + for_each(vec.begin(), vec.end(), ([&](const T& val) { values.emplace_back(val); })); return values; } diff --git a/src/mongo/db/exec/exclusion_projection_executor.cpp b/src/mongo/db/exec/exclusion_projection_executor.cpp index 9823ed1b125..2061bf4fe93 100644 --- a/src/mongo/db/exec/exclusion_projection_executor.cpp +++ b/src/mongo/db/exec/exclusion_projection_executor.cpp @@ -38,14 +38,15 @@ std::pair<BSONObj, bool> ExclusionNode::extractProjectOnFieldAndRename(const Str BSONObjBuilder extractedExclusion; // Check for a projection directly on 'oldName'. For example, {oldName: 0}. - if (auto it = _projectedFields.find(oldName); it != _projectedFields.end()) { + if (auto it = _projectedFieldsSet.find(oldName); it != _projectedFieldsSet.end()) { extractedExclusion.append(newName, false); - _projectedFields.erase(it); + _projectedFieldsSet.erase(it); + _projectedFields.remove(std::string(oldName)); } // Check for a projection on subfields of 'oldName'. For example, {oldName: {a: 0, b: 0}}. if (auto it = _children.find(oldName); it != _children.end()) { - extractedExclusion.append(newName, it->second->serialize(boost::none).toBson()); + extractedExclusion.append(newName, it->second->serialize(boost::none, {}).toBson()); _children.erase(it); } diff --git a/src/mongo/db/exec/exclusion_projection_executor.h b/src/mongo/db/exec/exclusion_projection_executor.h index d7a441e3b5c..37623206723 100644 --- a/src/mongo/db/exec/exclusion_projection_executor.h +++ b/src/mongo/db/exec/exclusion_projection_executor.h @@ -99,10 +99,12 @@ protected: */ class ExclusionProjectionExecutor : public ProjectionExecutor { public: - ExclusionProjectionExecutor(const boost::intrusive_ptr<ExpressionContext>& expCtx, - ProjectionPolicies policies, - bool allowFastPath = false) - : ProjectionExecutor(expCtx, policies), _root(new ExclusionNode(_policies)) {} + ExclusionProjectionExecutor( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + ProjectionPolicies policies, + bool allowFastPath = false, + boost::optional<projection_ast::ProjectionPathASTNode> proj = boost::none) + : ProjectionExecutor(expCtx, policies, proj), _root(new ExclusionNode(_policies)) {} TransformerType getType() const final { return TransformerType::kExclusionProjection; @@ -116,16 +118,17 @@ public: return _root.get(); } - Document serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const final { + Document serializeTransformation(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options = {}) const final { MutableDocument output; // The ExclusionNode tree in '_root' will always have a top-level _id node if _id is to be // excluded. If the _id node is not present, then explicitly set {_id: true} to avoid // ambiguity in the expected behavior of the serialized projection. - _root->serialize(explain, &output); - if (output.peek()["_id"].missing()) { - output.addField("_id", Value{true}); + _root->serialize(explain, &output, options); + auto idFieldName = options.serializeFieldPath("_id"); + if (output.peek()[idFieldName].missing()) { + output.addField(idFieldName, Value{true}); } return output.freeze(); } diff --git a/src/mongo/db/exec/fetch.cpp b/src/mongo/db/exec/fetch.cpp index 9d51dad5ca4..2059faf7d79 100644 --- a/src/mongo/db/exec/fetch.cpp +++ b/src/mongo/db/exec/fetch.cpp @@ -181,10 +181,8 @@ unique_ptr<PlanStageStats> FetchStage::getStats() { _commonStats.isEOF = isEOF(); // Add a BSON representation of the filter to the stats tree, if there is one. - if (nullptr != _filter) { - BSONObjBuilder bob; - _filter->serialize(&bob); - _commonStats.filter = bob.obj(); + if (_filter) { + _commonStats.filter = _filter->serialize(); } unique_ptr<PlanStageStats> ret = std::make_unique<PlanStageStats>(_commonStats, STAGE_FETCH); diff --git a/src/mongo/db/exec/inclusion_projection_executor.cpp b/src/mongo/db/exec/inclusion_projection_executor.cpp index 35828091f16..d06cedf61b6 100644 --- a/src/mongo/db/exec/inclusion_projection_executor.cpp +++ b/src/mongo/db/exec/inclusion_projection_executor.cpp @@ -67,7 +67,7 @@ void FastPathEligibleInclusionNode::_applyProjections(BSONObj bson, BSONObjBuild const auto bsonElement{it.next()}; const auto fieldName{bsonElement.fieldNameStringData()}; - if (_projectedFields.find(fieldName) != _projectedFields.end()) { + if (_projectedFieldsSet.find(fieldName) != _projectedFieldsSet.end()) { bob->append(bsonElement); --nFieldsNeeded; } else if (auto childIt = _children.find(fieldName); childIt != _children.end()) { @@ -219,11 +219,12 @@ std::pair<BSONObj, bool> InclusionNode::extractComputedProjectionsInProject( for (const auto& expressionSpec : addFieldsExpressions) { auto&& fieldName = std::get<0>(expressionSpec).toString(); auto oldExpr = std::get<1>(expressionSpec); - oldExpr->serialize(false).addToBsonObj(&bb, fieldName); + oldExpr->serialize().addToBsonObj(&bb, fieldName); if (std::get<2>(expressionSpec)) { // Replace the expression with an inclusion projected field. - _projectedFields.insert(fieldName); + auto it = _projectedFields.insert(_projectedFields.end(), fieldName); + _projectedFieldsSet.insert(StringData(*it)); _expressions.erase(fieldName); // Only computed projections at the beginning of the list were marked to become // projected fields. The new projected field is at the beginning of the @@ -302,7 +303,7 @@ std::pair<BSONObj, bool> InclusionNode::extractComputedProjectionsInAddFields( for (const auto& expressionSpec : addFieldsExpressions) { auto&& fieldName = expressionSpec.first.toString(); auto expr = expressionSpec.second; - expr->serialize(false).addToBsonObj(&bb, fieldName); + expr->serialize().addToBsonObj(&bb, fieldName); // Remove the expression from this inclusion node. _expressions.erase(fieldName); diff --git a/src/mongo/db/exec/inclusion_projection_executor.h b/src/mongo/db/exec/inclusion_projection_executor.h index aef4c07b434..7f505f54797 100644 --- a/src/mongo/db/exec/inclusion_projection_executor.h +++ b/src/mongo/db/exec/inclusion_projection_executor.h @@ -173,19 +173,24 @@ private: */ class InclusionProjectionExecutor : public ProjectionExecutor { public: - InclusionProjectionExecutor(const boost::intrusive_ptr<ExpressionContext>& expCtx, - ProjectionPolicies policies, - std::unique_ptr<InclusionNode> root) - : ProjectionExecutor(expCtx, policies), _root(std::move(root)) {} - - InclusionProjectionExecutor(const boost::intrusive_ptr<ExpressionContext>& expCtx, - ProjectionPolicies policies, - bool allowFastPath = false) + InclusionProjectionExecutor( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + ProjectionPolicies policies, + std::unique_ptr<InclusionNode> root, + boost::optional<projection_ast::ProjectionPathASTNode> proj = boost::none) + : ProjectionExecutor(expCtx, policies, proj), _root(std::move(root)) {} + + InclusionProjectionExecutor( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + ProjectionPolicies policies, + bool allowFastPath = false, + boost::optional<projection_ast::ProjectionPathASTNode> proj = boost::none) : InclusionProjectionExecutor( expCtx, policies, allowFastPath ? std::make_unique<FastPathEligibleInclusionNode>(policies) - : std::make_unique<InclusionNode>(policies)) {} + : std::make_unique<InclusionNode>(policies), + proj) {} TransformerType getType() const final { return TransformerType::kInclusionProjection; @@ -202,16 +207,17 @@ public: /** * Serialize the projection. */ - Document serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const final { + Document serializeTransformation(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options = {}) const final { MutableDocument output; // The InclusionNode tree in '_root' will always have a top-level _id node if _id is to be // included. If the _id node is not present, then explicitly set {_id: false} to avoid // ambiguity in the expected behavior of the serialized projection. - _root->serialize(explain, &output); - if (output.peek()["_id"].missing()) { - output.addField("_id", Value{false}); + _root->serialize(explain, &output, options); + auto idFieldName = options.serializeFieldPath("_id"); + if (output.peek()[idFieldName].missing()) { + output.addField(idFieldName, Value{false}); } return output.freeze(); diff --git a/src/mongo/db/exec/index_scan.cpp b/src/mongo/db/exec/index_scan.cpp index 06399c4e33b..c61fc452ec7 100644 --- a/src/mongo/db/exec/index_scan.cpp +++ b/src/mongo/db/exec/index_scan.cpp @@ -280,9 +280,7 @@ std::unique_ptr<PlanStageStats> IndexScan::getStats() { // Add a BSON representation of the filter to the stats tree, if there is one. if (nullptr != _filter) { - BSONObjBuilder bob; - _filter->serialize(&bob); - _commonStats.filter = bob.obj(); + _commonStats.filter = _filter->serialize(); } // These specific stats fields never change. diff --git a/src/mongo/db/exec/or.cpp b/src/mongo/db/exec/or.cpp index ec0d680ac37..078765ffc84 100644 --- a/src/mongo/db/exec/or.cpp +++ b/src/mongo/db/exec/or.cpp @@ -122,10 +122,8 @@ unique_ptr<PlanStageStats> OrStage::getStats() { _commonStats.isEOF = isEOF(); // Add a BSON representation of the filter to the stats tree, if there is one. - if (nullptr != _filter) { - BSONObjBuilder bob; - _filter->serialize(&bob); - _commonStats.filter = bob.obj(); + if (_filter) { + _commonStats.filter = _filter->serialize(); } unique_ptr<PlanStageStats> ret = std::make_unique<PlanStageStats>(_commonStats, STAGE_OR); diff --git a/src/mongo/db/exec/projection.h b/src/mongo/db/exec/projection.h index 00e7fb33dbc..236792ce64a 100644 --- a/src/mongo/db/exec/projection.h +++ b/src/mongo/db/exec/projection.h @@ -33,6 +33,7 @@ #include "mongo/db/exec/projection_executor.h" #include "mongo/db/jsobj.h" #include "mongo/db/matcher/expression.h" +#include "mongo/db/query/projection.h" #include "mongo/db/query/projection_ast.h" #include "mongo/db/record_id.h" diff --git a/src/mongo/db/exec/projection_executor.h b/src/mongo/db/exec/projection_executor.h index ca8e0d3990c..449ef8c3a7b 100644 --- a/src/mongo/db/exec/projection_executor.h +++ b/src/mongo/db/exec/projection_executor.h @@ -38,6 +38,7 @@ #include "mongo/db/pipeline/expression_context.h" #include "mongo/db/pipeline/field_path.h" #include "mongo/db/pipeline/transformer_interface.h" +#include "mongo/db/query/projection_ast.h" #include "mongo/db/query/projection_policies.h" namespace mongo::projection_executor { @@ -97,10 +98,20 @@ public: */ virtual boost::optional<std::set<FieldRef>> extractExhaustivePaths() const = 0; + /** + * The query shape is made by serializing the first parsed representation of the query, which in + * the case of $project queries is a projection_ast::Projection. The ProjectionExecutor, holds + * onto the root node of the AST for only $project queries, so that the first parsed + * representation is accessible at serialization. + */ + boost::optional<projection_ast::ProjectionPathASTNode> projection = boost::none; + protected: ProjectionExecutor(const boost::intrusive_ptr<ExpressionContext>& expCtx, - ProjectionPolicies policies) - : _expCtx(expCtx), + ProjectionPolicies policies, + boost::optional<projection_ast::ProjectionPathASTNode> proj = boost::none) + : projection(proj), + _expCtx(expCtx), _policies(policies), _projectionPostImageVarId{ _expCtx->variablesParseState.defineVariable(kProjectionPostImageVarName)} {} diff --git a/src/mongo/db/exec/projection_executor_builder.cpp b/src/mongo/db/exec/projection_executor_builder.cpp index 1b712685ea9..f5e94be3f78 100644 --- a/src/mongo/db/exec/projection_executor_builder.cpp +++ b/src/mongo/db/exec/projection_executor_builder.cpp @@ -251,7 +251,7 @@ auto buildProjectionExecutor(boost::intrusive_ptr<ExpressionContext> expCtx, const ProjectionPolicies policies, const BuilderParamsBitSet params) { ProjectionExecutorVisitorContext<Executor> context{ - {std::make_unique<Executor>(expCtx, policies, params[kAllowFastPath]), expCtx}}; + {std::make_unique<Executor>(expCtx, policies, params[kAllowFastPath], *root), expCtx}}; ProjectionExecutorVisitor<Executor> executorVisitor{&context}; projection_ast::PathTrackingWalker walker{&context, {&executorVisitor}, {}}; tree_walker::walk<true, projection_ast::ASTNode>(root, &walker); diff --git a/src/mongo/db/exec/projection_executor_builder.h b/src/mongo/db/exec/projection_executor_builder.h index 476f2b63a25..12f683a1cba 100644 --- a/src/mongo/db/exec/projection_executor_builder.h +++ b/src/mongo/db/exec/projection_executor_builder.h @@ -32,6 +32,7 @@ #include <bitset> #include "mongo/db/exec/projection_executor.h" +#include "mongo/db/query/projection.h" #include "mongo/db/query/projection_ast.h" namespace mongo::projection_executor { diff --git a/src/mongo/db/exec/projection_executor_redaction_test.cpp b/src/mongo/db/exec/projection_executor_redaction_test.cpp new file mode 100644 index 00000000000..70eb59855bb --- /dev/null +++ b/src/mongo/db/exec/projection_executor_redaction_test.cpp @@ -0,0 +1,208 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "document_value/document_value_test_util.h" +#include "mongo/db/exec/projection_executor.h" +#include "mongo/db/exec/projection_executor_builder.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/projection_ast_util.h" +#include "mongo/db/query/projection_parser.h" +#include "mongo/db/query/projection_policies.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { +std::unique_ptr<projection_executor::ProjectionExecutor> compileProjection(BSONObj proj) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto policies = ProjectionPolicies::findProjectionPolicies(); + auto ast = projection_ast::parseAndAnalyze(expCtx, proj, policies); + return projection_executor::buildProjectionExecutor( + expCtx, &ast, policies, projection_executor::kDefaultBuilderParams); +} +std::unique_ptr<projection_executor::ProjectionExecutor> compileProjection(BSONObj proj, + BSONObj query) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto match = uassertStatusOK(MatchExpressionParser::parse(query, expCtx)); + auto policies = ProjectionPolicies::findProjectionPolicies(); + auto ast = projection_ast::parseAndAnalyze(expCtx, proj, match.get(), query, policies); + auto exec = projection_executor::buildProjectionExecutor( + expCtx, &ast, policies, projection_executor::kDefaultBuilderParams); + return exec; +} + +TEST(Redaction, ProjectionTest) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto redactProj = [&](std::string obj) { + return compileProjection(fromjson(obj))->serializeTransformation(boost::none, options); + }; + + /// Inclusion projections + + // Simple single inclusion + auto actual = redactProj("{a: 1}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<a>":true})", + actual); + + actual = redactProj("{a: true}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<a>":true})", + actual); + + // Dotted path + actual = redactProj("{\"a.b\": 1}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<a>":{"HASH<b>":true}})", + actual); + + // Two fields + actual = redactProj("{a: 1, b: 1}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<a>":true,"HASH<b>":true})", + actual); + + // Explicit _id: 1 + actual = redactProj("{b: 1, _id: 1}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<b>":true})", + actual); + + // Two nested fields + actual = redactProj("{\"b.d\": 1, \"b.c\": 1}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<b>":{"HASH<d>":true,"HASH<c>":true}})", + actual); + + actual = redactProj("{\"b.d\": 1, a: 1, \"b.c\": 1}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "HASH<_id>": true, + "HASH<a>": true, + "HASH<b>": { + "HASH<d>": true, + "HASH<c>": true + } + })", + actual); + + /// Exclusion projections + + // Simple single exclusion + actual = redactProj("{a: 0}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<a>":false,"HASH<_id>":true})", + actual); + + // Dotted path + actual = redactProj("{\"a.b\": 0}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<a>":{"HASH<b>":false},"HASH<_id>":true})", + actual); + + // Two fields + actual = redactProj("{a: 0, b: 0}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<a>":false,"HASH<b>":false,"HASH<_id>":true})", + actual); + + // Explicit _id: 0 + actual = redactProj("{b: 0, _id: 0}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":false,"HASH<b>":false})", + actual); + + // Two nested fields + actual = redactProj("{\"b.d\": 0, \"b.c\": 0}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<b>":{"HASH<d>":false,"HASH<c>":false},"HASH<_id>":true})", + actual); + + actual = redactProj("{\"b.d\": 0, a: 0, \"b.c\": 0}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "HASH<a>": false, + "HASH<b>": { + "HASH<d>": false, + "HASH<c>": false + }, + "HASH<_id>": true + })", + actual); + + /// Add fields projection + actual = redactProj("{a: \"hi\"}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<a>":"?string"})", + actual); + + actual = redactProj("{a: '$field'}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<a>":"$HASH<field>"})", + actual); + + // Dotted path + actual = redactProj("{\"a.b\": \"hi\"}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<a>":{"HASH<b>":"?string"}})", + actual); + + // Two fields + actual = redactProj("{a: \"hi\", b: \"hello\"}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<a>":"?string","HASH<b>":"?string"})", + actual); + + // Explicit _id: 0 + actual = redactProj("{b: \"hi\", _id: \"hey\"}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<b>":"?string","HASH<_id>":"?string"})", + actual); + + // Two nested fields + actual = redactProj("{\"b.d\": \"hello\", \"b.c\": \"world\"}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<_id>":true,"HASH<b>":{"HASH<d>":"?string","HASH<c>":"?string"}})", + actual); + + actual = redactProj("{\"b.d\": \"hello\", a: \"world\", \"b.c\": \"mongodb\"}"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "HASH<_id>": true, + "HASH<b>": { + "HASH<d>": "?string", + "HASH<c>": "?string" + }, + "HASH<a>": "?string" + })", + actual); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/exec/projection_node.cpp b/src/mongo/db/exec/projection_node.cpp index 773730c33bc..6b053c4593f 100644 --- a/src/mongo/db/exec/projection_node.cpp +++ b/src/mongo/db/exec/projection_node.cpp @@ -48,7 +48,8 @@ void ProjectionNode::addProjectionForPath(const FieldPath& path) { void ProjectionNode::_addProjectionForPath(const FieldPath& path) { makeOptimizationsStale(); if (path.getPathLength() == 1) { - _projectedFields.insert(path.fullPath()); + auto it = _projectedFields.insert(_projectedFields.end(), path.fullPath()); + _projectedFieldsSet.insert(StringData(*it)); return; } // FieldPath can't be empty, so it is safe to obtain the first path component here. @@ -143,7 +144,7 @@ void ProjectionNode::applyProjections(const Document& inputDoc, MutableDocument* while (it.more()) { auto fieldName = it.fieldName(); - if (_projectedFields.find(fieldName) != _projectedFields.end()) { + if (_projectedFieldsSet.find(fieldName) != _projectedFieldsSet.end()) { if (isIncl) { outputProjectedField(fieldName, it.next().second, outputDoc); } else { @@ -274,25 +275,27 @@ void ProjectionNode::optimize() { _maxFieldsToProject = maxFieldsToProject(); } -Document ProjectionNode::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Document ProjectionNode::serialize(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options) const { MutableDocument outputDoc; - serialize(explain, &outputDoc); + serialize(explain, &outputDoc, options); return outputDoc.freeze(); } void ProjectionNode::serialize(boost::optional<ExplainOptions::Verbosity> explain, - MutableDocument* output) const { + MutableDocument* output, + const SerializationOptions& options) const { // Determine the boolean value for projected fields in the explain output. const bool projVal = isIncluded(); // Always put "_id" first if it was projected (implicitly or explicitly). - if (_projectedFields.find("_id") != _projectedFields.end()) { - output->addField("_id", Value(projVal)); + if (_projectedFieldsSet.find("_id") != _projectedFieldsSet.end()) { + output->addField(options.serializeFieldPath("_id"), Value(projVal)); } for (auto&& projectedField : _projectedFields) { if (projectedField != "_id") { - output->addField(projectedField, Value(projVal)); + output->addField(options.serializeFieldPathFromString(projectedField), Value(projVal)); } } @@ -300,13 +303,14 @@ void ProjectionNode::serialize(boost::optional<ExplainOptions::Verbosity> explai auto childIt = _children.find(field); if (childIt != _children.end()) { MutableDocument subDoc; - childIt->second->serialize(explain, &subDoc); - output->addField(field, subDoc.freezeToValue()); + childIt->second->serialize(explain, &subDoc, options); + output->addField(options.serializeFieldPathFromString(field), subDoc.freezeToValue()); } else { invariant(_policies.computedFieldsPolicy == ComputedFieldsPolicy::kAllowComputedFields); auto expressionIt = _expressions.find(field); invariant(expressionIt != _expressions.end()); - output->addField(field, expressionIt->second->serialize(static_cast<bool>(explain))); + output->addField(options.serializeFieldPathFromString(field), + expressionIt->second->serialize(options)); } } } diff --git a/src/mongo/db/exec/projection_node.h b/src/mongo/db/exec/projection_node.h index 073fe14ae4e..cd744ef4cac 100644 --- a/src/mongo/db/exec/projection_node.h +++ b/src/mongo/db/exec/projection_node.h @@ -29,8 +29,9 @@ #pragma once -#include "mongo/db/exec/projection_executor.h" +#include <list> +#include "mongo/db/exec/projection_executor.h" #include "mongo/db/query/projection_policies.h" namespace mongo::projection_executor { @@ -128,10 +129,12 @@ public: void optimize(); - Document serialize(boost::optional<ExplainOptions::Verbosity> explain) const; + Document serialize(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options) const; void serialize(boost::optional<ExplainOptions::Verbosity> explain, - MutableDocument* output) const; + MutableDocument* output, + const SerializationOptions& options) const; protected: /** @@ -165,7 +168,14 @@ protected: StringMap<std::unique_ptr<ProjectionNode>> _children; StringMap<boost::intrusive_ptr<Expression>> _expressions; - StringSet _projectedFields; + + // List of the projected fields in the order in which they were specified. + std::list<std::string> _projectedFields; + + // Set of projected fields. Note that the _projectedFields list actually owns the strings, and + // this StringDataSet simply holds views of those strings. + StringDataSet _projectedFieldsSet; + ProjectionPolicies _policies; std::string _pathToNode; diff --git a/src/mongo/db/exec/sbe/stages/hash_agg.cpp b/src/mongo/db/exec/sbe/stages/hash_agg.cpp index 99bcc9f11c3..2514dedf1b0 100644 --- a/src/mongo/db/exec/sbe/stages/hash_agg.cpp +++ b/src/mongo/db/exec/sbe/stages/hash_agg.cpp @@ -103,18 +103,26 @@ std::unique_ptr<PlanStage> HashAggStage::clone() const { void HashAggStage::doSaveState(bool relinquishCursor) { if (relinquishCursor) { if (_rsCursor) { - _rsCursor->save(); + _recordStore->saveCursor(_opCtx, _rsCursor); } } if (_rsCursor) { _rsCursor->setSaveStorageCursorOnDetachFromOperationContext(!relinquishCursor); } + + if (_recordStore) { + _recordStore->saveState(); + } } void HashAggStage::doRestoreState(bool relinquishCursor) { invariant(_opCtx); + if (_recordStore) { + _recordStore->restoreState(); + } + if (_rsCursor && relinquishCursor) { - auto couldRestore = _rsCursor->restore(); + auto couldRestore = _recordStore->restoreCursor(_opCtx, _rsCursor); uassert(6196500, "HashAggStage could not restore cursor", couldRestore); } } @@ -262,8 +270,7 @@ void HashAggStage::makeTemporaryRecordStore() { "No storage engine so HashAggStage cannot spill to disk", _opCtx->getServiceContext()->getStorageEngine()); assertIgnorePrepareConflictsBehavior(_opCtx); - _recordStore = _opCtx->getServiceContext()->getStorageEngine()->makeTemporaryRecordStore( - _opCtx, KeyFormat::String); + _recordStore = std::make_unique<SpillingStore>(_opCtx); _specificStats.usedDisk = true; } @@ -291,10 +298,10 @@ void HashAggStage::spillRowToDisk(const value::MaterializedRow& key, if (collator) { // The keystring cannot always be deserialized back to the original keys when a collation is // in use, so we also store the unmodified key in the data part of the spilled record. - upsertToRecordStore(_opCtx, _recordStore->rs(), rid, key, val, false /*update*/); + _recordStore->upsertToRecordStore(_opCtx, rid, key, val, false /*update*/); } else { auto typeBits = kb.getTypeBits(); - upsertToRecordStore(_opCtx, _recordStore->rs(), rid, val, typeBits, false /*update*/); + _recordStore->upsertToRecordStore(_opCtx, rid, val, typeBits, false /*update*/); } _specificStats.spilledRecords++; @@ -414,7 +421,9 @@ void HashAggStage::open(bool reOpen) { for (auto&& accessor : _outAggAccessors) { accessor->setIndex(0); } - _rsCursor.reset(); + if (_recordStore) { + _recordStore->resetCursor(_opCtx, _rsCursor); + } _recordStore.reset(); _outKeyRowRecordStore = {0}; _outAggRowRecordStore = {0}; @@ -490,7 +499,7 @@ void HashAggStage::open(bool reOpen) { _specificStats.spilledDataStorageSize = _recordStore->rs()->storageSize(_opCtx); // Establish a cursor, positioned at the beginning of the record store. - _rsCursor = _recordStore->rs()->getCursor(_opCtx); + _rsCursor = _recordStore->getCursor(_opCtx); // Callers will be obtaining the results from the spill table, so set the // 'SwitchAccessors' so that they refer to the rows recovered from the record store @@ -675,6 +684,9 @@ void HashAggStage::close() { trackClose(); _ht = boost::none; + if (_recordStore && _opCtx) { + _recordStore->resetCursor(_opCtx, _rsCursor); + } _rsCursor.reset(); _recordStore.reset(); _outKeyRowRecordStore = {0}; diff --git a/src/mongo/db/exec/sbe/stages/hash_agg.h b/src/mongo/db/exec/sbe/stages/hash_agg.h index 2f77e445883..91f91051363 100644 --- a/src/mongo/db/exec/sbe/stages/hash_agg.h +++ b/src/mongo/db/exec/sbe/stages/hash_agg.h @@ -31,6 +31,7 @@ #include "mongo/db/exec/sbe/expressions/expression.h" #include "mongo/db/exec/sbe/stages/stages.h" +#include "mongo/db/exec/sbe/util/spilling.h" #include "mongo/db/exec/sbe/vm/vm.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/db/storage/temporary_record_store.h" @@ -277,7 +278,7 @@ private: internalQuerySBEAggApproxMemoryUseInBytesBeforeSpill.load(); // A record store which is instantiated and written to in the case of spilling. - std::unique_ptr<TemporaryRecordStore> _recordStore; + std::unique_ptr<SpillingStore> _recordStore; std::unique_ptr<SeekableRecordCursor> _rsCursor; // A monotically increasing counter used to ensure uniqueness of 'RecordId' values. When diff --git a/src/mongo/db/exec/sbe/stages/hash_lookup.cpp b/src/mongo/db/exec/sbe/stages/hash_lookup.cpp index fbc8ff73058..02e95307c4b 100644 --- a/src/mongo/db/exec/sbe/stages/hash_lookup.cpp +++ b/src/mongo/db/exec/sbe/stages/hash_lookup.cpp @@ -187,6 +187,22 @@ value::SlotAccessor* HashLookupStage::getAccessor(CompileCtx& ctx, value::SlotId return outerChild()->getAccessor(ctx, slot); } } +void HashLookupStage::doSaveState(bool relinquishCursor) { + if (_recordStoreHt) { + _recordStoreHt->saveState(); + } + if (_recordStoreBuf) { + _recordStoreBuf->saveState(); + } +} +void HashLookupStage::doRestoreState(bool relinquishCursor) { + if (_recordStoreHt) { + _recordStoreHt->restoreState(); + } + if (_recordStoreBuf) { + _recordStoreBuf->restoreState(); + } +} void HashLookupStage::reset() { _ht = boost::none; @@ -259,7 +275,7 @@ void HashLookupStage::addHashTableEntry(value::SlotAccessor* keyAccessor, size_t auto val = std::vector<size_t>{valueIndex}; auto [tagKey, valKey] = keyAccessor->getViewOfValue(); - spillIndicesToRecordStore(_recordStoreHt->rs(), tagKey, valKey, val); + spillIndicesToRecordStore(_recordStoreHt.get(), tagKey, valKey, val); } } else { // The key is already present in '_ht' so the memory will only grow by one size_t. If we @@ -281,7 +297,7 @@ void HashLookupStage::addHashTableEntry(value::SlotAccessor* keyAccessor, size_t // Evict the hash table value. _computedTotalMemUsage -= htIt->second.size() * sizeof(size_t); htIt->second.push_back(valueIndex); - spillIndicesToRecordStore(_recordStoreHt->rs(), tagKeyView, valKeyView, htIt->second); + spillIndicesToRecordStore(_recordStoreHt.get(), tagKeyView, valKeyView, htIt->second); _ht->erase(htIt); } } @@ -297,17 +313,15 @@ void HashLookupStage::makeTemporaryRecordStore() { _opCtx->getServiceContext()->getStorageEngine()); assertIgnorePrepareConflictsBehavior(_opCtx); - _recordStoreBuf = _opCtx->getServiceContext()->getStorageEngine()->makeTemporaryRecordStore( - _opCtx, KeyFormat::Long); + _recordStoreBuf = std::make_unique<SpillingStore>(_opCtx, KeyFormat::Long); - _recordStoreHt = _opCtx->getServiceContext()->getStorageEngine()->makeTemporaryRecordStore( - _opCtx, KeyFormat::String); + _recordStoreHt = std::make_unique<SpillingStore>(_opCtx, KeyFormat::String); _specificStats.usedDisk = true; } void HashLookupStage::spillBufferedValueToDisk(OperationContext* opCtx, - RecordStore* rs, + SpillingStore* rs, size_t bufferIdx, const value::MaterializedRow& val) { auto rid = getValueRecordId(bufferIdx); @@ -315,15 +329,7 @@ void HashLookupStage::spillBufferedValueToDisk(OperationContext* opCtx, BufBuilder buf; val.serializeForSorter(buf); - assertIgnorePrepareConflictsBehavior(opCtx); - WriteUnitOfWork wuow(opCtx); - - auto status = rs->insertRecord(opCtx, rid, buf.buf(), buf.len(), Timestamp{}); - wuow.commit(); - - tassert(6373906, - str::stream() << "Failed to write to disk because " << status.getStatus().reason(), - status.isOK()); + rs->upsertToRecordStore(opCtx, rid, buf, false); _specificStats.spilledBuffRecords++; // Add size of record ID + size of buffer. @@ -341,7 +347,7 @@ size_t HashLookupStage::bufferValueOrSpill(value::MaterializedRow& value) { if (!hasSpilledBufToDisk()) { makeTemporaryRecordStore(); } - spillBufferedValueToDisk(_opCtx, _recordStoreBuf->rs(), bufferIndex, value); + spillBufferedValueToDisk(_opCtx, _recordStoreBuf.get(), bufferIndex, value); } _valueId++; return bufferIndex; @@ -427,7 +433,7 @@ void HashLookupStage::accumulateFromValueIndices(const C& bufferIndices) { // We must shift the '_bufferIt' index by one when using it as a RecordId because a // RecordId of 0 is invalid. auto rid = getValueRecordId(_bufferIt); - auto rsValue = readFromRecordStore(_opCtx, _recordStoreBuf->rs(), rid); + auto rsValue = _recordStoreBuf->readFromRecordStore(_opCtx, rid); if (!rsValue) { tasserted(6373900, "bufferIdx not found in record store"); } @@ -443,7 +449,7 @@ void HashLookupStage::accumulateFromValueIndices(const C& bufferIndices) { } } -void HashLookupStage::writeIndicesToRecordStore(RecordStore* rs, +void HashLookupStage::writeIndicesToRecordStore(SpillingStore* rs, value::TypeTags tagKey, value::Value valKey, const std::vector<size_t>& value, @@ -458,7 +464,7 @@ void HashLookupStage::writeIndicesToRecordStore(RecordStore* rs, key.reset(0, false, tagKey, valKey); auto [rid, typeBits] = serializeKeyForRecordStore(key); - upsertToRecordStore(_opCtx, rs, rid, buf, typeBits, update); + rs->upsertToRecordStore(_opCtx, rid, buf, typeBits, update); if (!update) { _specificStats.spilledHtRecords++; // Add the size of key (which comprises of the memory usage for the key + its type bits), @@ -471,7 +477,7 @@ void HashLookupStage::writeIndicesToRecordStore(RecordStore* rs, } boost::optional<std::vector<size_t>> HashLookupStage::readIndicesFromRecordStore( - RecordStore* rs, value::TypeTags tagKey, value::Value valKey) { + SpillingStore* rs, value::TypeTags tagKey, value::Value valKey) { _probeKey.reset(0, false, tagKey, valKey); auto [rid, _] = serializeKeyForRecordStore(_probeKey); @@ -490,7 +496,7 @@ boost::optional<std::vector<size_t>> HashLookupStage::readIndicesFromRecordStore return boost::none; } -void HashLookupStage::spillIndicesToRecordStore(RecordStore* rs, +void HashLookupStage::spillIndicesToRecordStore(SpillingStore* rs, value::TypeTags tagKey, value::Value valKey, const std::vector<size_t>& value) { @@ -545,7 +551,7 @@ PlanState HashLookupStage::getNext() { normalizeStringIfCollator(tagElemView, valElemView); auto indicesFromRS = readIndicesFromRecordStore( - _recordStoreHt->rs(), tagElemCollView, valElemCollView); + _recordStoreHt.get(), tagElemCollView, valElemCollView); if (indicesFromRS) { indices.insert(indicesFromRS->begin(), indicesFromRS->end()); } @@ -567,7 +573,7 @@ PlanState HashLookupStage::getNext() { normalizeStringIfCollator(tagKeyView, valKeyView); auto indicesFromRS = readIndicesFromRecordStore( - _recordStoreHt->rs(), tagKeyCollView, valKeyCollView); + _recordStoreHt.get(), tagKeyCollView, valKeyCollView); if (indicesFromRS) { accumulateFromValueIndices(*indicesFromRS); } diff --git a/src/mongo/db/exec/sbe/stages/hash_lookup.h b/src/mongo/db/exec/sbe/stages/hash_lookup.h index 2e3f0b34816..b312e0a68f4 100644 --- a/src/mongo/db/exec/sbe/stages/hash_lookup.h +++ b/src/mongo/db/exec/sbe/stages/hash_lookup.h @@ -33,6 +33,7 @@ #include "mongo/db/exec/sbe/expressions/expression.h" #include "mongo/db/exec/sbe/stages/stages.h" +#include "mongo/db/exec/sbe/util/spilling.h" #include "mongo/db/exec/sbe/vm/vm.h" #include "mongo/db/query/query_knobs_gen.h" @@ -101,6 +102,10 @@ public: std::vector<DebugPrinter::Block> debugPrint() const final; size_t estimateCompileTimeSize() const final; +protected: + void doSaveState(bool relinquishCursor) override; + void doRestoreState(bool relinquishCursor) override; + private: using HashTableType = std::unordered_map<value::MaterializedRow, // NOLINT std::vector<size_t>, @@ -119,23 +124,23 @@ private: // Spilling helpers. void addHashTableEntry(value::SlotAccessor* keyAccessor, size_t valueIndex); void spillBufferedValueToDisk(OperationContext* opCtx, - RecordStore* rs, + SpillingStore* rs, size_t bufferIdx, const value::MaterializedRow&); size_t bufferValueOrSpill(value::MaterializedRow& value); void setInnerProjectSwitchAccessor(int idx); - boost::optional<std::vector<size_t>> readIndicesFromRecordStore(RecordStore* rs, + boost::optional<std::vector<size_t>> readIndicesFromRecordStore(SpillingStore* rs, value::TypeTags tagKey, value::Value valKey); - void writeIndicesToRecordStore(RecordStore* rs, + void writeIndicesToRecordStore(SpillingStore* rs, value::TypeTags tagKey, value::Value valKey, const std::vector<size_t>& value, bool update); - void spillIndicesToRecordStore(RecordStore* rs, + void spillIndicesToRecordStore(SpillingStore* rs, value::TypeTags tagKey, value::Value valKey, const std::vector<size_t>& value); @@ -229,8 +234,8 @@ private: // rows in '_buffer'. long long _computedTotalMemUsage = 0; - std::unique_ptr<TemporaryRecordStore> _recordStoreHt; - std::unique_ptr<TemporaryRecordStore> _recordStoreBuf; + std::unique_ptr<SpillingStore> _recordStoreHt; + std::unique_ptr<SpillingStore> _recordStoreBuf; HashLookupStats _specificStats; }; diff --git a/src/mongo/db/exec/sbe/util/spilling.cpp b/src/mongo/db/exec/sbe/util/spilling.cpp index 0f0cbb93d94..7675fad6846 100644 --- a/src/mongo/db/exec/sbe/util/spilling.cpp +++ b/src/mongo/db/exec/sbe/util/spilling.cpp @@ -29,6 +29,18 @@ #include "mongo/db/exec/sbe/util/spilling.h" +#include "mongo/base/status.h" +#include "mongo/base/status_with.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/timestamp.h" +#include "mongo/db/query/query_knobs_gen.h" +#include "mongo/db/storage/record_data.h" +#include "mongo/db/storage/recovery_unit.h" +#include "mongo/db/storage/write_unit_of_work.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/bufreader.h" +#include "mongo/util/str.h" + namespace mongo { namespace sbe { @@ -57,32 +69,76 @@ KeyString::Value decodeKeyString(const RecordId& rid, KeyString::TypeBits typeBi return kb.getValueCopy(); } -boost::optional<value::MaterializedRow> readFromRecordStore(OperationContext* opCtx, - RecordStore* rs, - const RecordId& rid) { - RecordData record; - if (rs->findRecord(opCtx, rid, &record)) { - auto valueReader = BufReader(record.data(), record.size()); - return value::MaterializedRow::deserializeForSorter(valueReader, {}); - } - return boost::none; +SpillingStore::SpillingStore(OperationContext* opCtx, KeyFormat format) { + _recordStore = + opCtx->getServiceContext()->getStorageEngine()->makeTemporaryRecordStore(opCtx, format); + + _spillingUnit = std::unique_ptr<RecoveryUnit>( + opCtx->getServiceContext()->getStorageEngine()->newRecoveryUnit()); + _spillingUnit->setCacheMaxWaitTimeout(Milliseconds(internalQuerySpillingMaxWaitTimeout.load())); + _spillingState = WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork; } -static int upsertToRecordStore( - OperationContext* opCtx, RecordStore* rs, const RecordId& key, BufBuilder& buf, bool update) { +SpillingStore::~SpillingStore() {} +int SpillingStore::upsertToRecordStore(OperationContext* opCtx, + const RecordId& recordKey, + const value::MaterializedRow& key, + const value::MaterializedRow& val, + bool update) { + BufBuilder buf; + key.serializeForSorter(buf); + val.serializeForSorter(buf); + return upsertToRecordStore(opCtx, recordKey, buf, update); +} + +int SpillingStore::upsertToRecordStore( + OperationContext* opCtx, + const RecordId& key, + const value::MaterializedRow& val, + const KeyString::TypeBits& typeBits, // recover type of value. + bool update) { + BufBuilder bufValue; + val.serializeForSorter(bufValue); + // Append the 'typeBits' to the end of the val's buffer so the 'key' can be reconstructed when + // draining HashAgg. + bufValue.appendBuf(typeBits.getBuffer(), typeBits.getSize()); + + return upsertToRecordStore(opCtx, key, bufValue, update); +} + +int SpillingStore::upsertToRecordStore( + OperationContext* opCtx, + const RecordId& key, + BufBuilder& buf, + const KeyString::TypeBits& typeBits, // recover type of value. + bool update) { + // Append the 'typeBits' to the end of the val's buffer so the 'key' can be reconstructed when + // draining HashAgg. + buf.appendBuf(typeBits.getBuffer(), typeBits.getSize()); + + return upsertToRecordStore(opCtx, key, buf, update); +} + +int SpillingStore::upsertToRecordStore(OperationContext* opCtx, + const RecordId& key, + BufBuilder& buf, + bool update) { assertIgnorePrepareConflictsBehavior(opCtx); + switchToSpilling(opCtx); + ON_BLOCK_EXIT([&] { switchToOriginal(opCtx); }); WriteUnitOfWork wuow(opCtx); auto result = mongo::Status::OK(); if (update) { - result = rs->updateRecord(opCtx, key, buf.buf(), buf.len()); + result = rs()->updateRecord(opCtx, key, buf.buf(), buf.len()); } else { - auto status = rs->insertRecord(opCtx, key, buf.buf(), buf.len(), Timestamp{}); + auto status = rs()->insertRecord(opCtx, key, buf.buf(), buf.len(), Timestamp{}); result = status.getStatus(); } wuow.commit(); + if (!result.isOK()) { tasserted(5843600, str::stream() << "Failed to write to disk because " << result.reason()); return 0; @@ -90,42 +146,59 @@ static int upsertToRecordStore( return buf.len(); } -int upsertToRecordStore(OperationContext* opCtx, - RecordStore* rs, - const RecordId& key, - const value::MaterializedRow& val, - const KeyString::TypeBits& typeBits, // recover type of value. - bool update) { - BufBuilder buf; - val.serializeForSorter(buf); - // Append the 'typeBits' to the end of the val's buffer so the 'key' can be reconstructed when - // draining HashAgg. - buf.appendBuf(typeBits.getBuffer(), typeBits.getSize()); - return upsertToRecordStore(opCtx, rs, key, buf, update); +Status SpillingStore::insertRecords(OperationContext* opCtx, + std::vector<Record>* inOutRecords, + const std::vector<Timestamp>& timestamps) { + assertIgnorePrepareConflictsBehavior(opCtx); + + switchToSpilling(opCtx); + ON_BLOCK_EXIT([&] { switchToOriginal(opCtx); }); + WriteUnitOfWork wuow(opCtx); + auto status = rs()->insertRecords(opCtx, inOutRecords, timestamps); + wuow.commit(); + + return status; } -int upsertToRecordStore(OperationContext* opCtx, - RecordStore* rs, - const RecordId& recordKey, - const value::MaterializedRow& key, - const value::MaterializedRow& val, - bool update) { - BufBuilder buf; - key.serializeForSorter(buf); - val.serializeForSorter(buf); - return upsertToRecordStore(opCtx, rs, recordKey, buf, update); +boost::optional<value::MaterializedRow> SpillingStore::readFromRecordStore(OperationContext* opCtx, + const RecordId& rid) { + switchToSpilling(opCtx); + ON_BLOCK_EXIT([&] { switchToOriginal(opCtx); }); + + RecordData record; + if (rs()->findRecord(opCtx, rid, &record)) { + auto valueReader = BufReader(record.data(), record.size()); + return value::MaterializedRow::deserializeForSorter(valueReader, {}); + } + return boost::none; } -int upsertToRecordStore(OperationContext* opCtx, - RecordStore* rs, - const RecordId& key, - BufBuilder& buf, - const KeyString::TypeBits& typeBits, // recover type of value. - bool update) { - // Append the 'typeBits' to the end of the val's buffer so the 'key' can be reconstructed when - // draining HashAgg. - buf.appendBuf(typeBits.getBuffer(), typeBits.getSize()); - return upsertToRecordStore(opCtx, rs, key, buf, update); +bool SpillingStore::findRecord(OperationContext* opCtx, const RecordId& loc, RecordData* out) { + switchToSpilling(opCtx); + ON_BLOCK_EXIT([&] { switchToOriginal(opCtx); }); + + return rs()->findRecord(opCtx, loc, out); } + +void SpillingStore::switchToSpilling(OperationContext* opCtx) { + invariant(!_originalUnit); + _originalUnit = opCtx->releaseRecoveryUnit(); + _originalState = opCtx->setRecoveryUnit(std::move(_spillingUnit), _spillingState); +} +void SpillingStore::switchToOriginal(OperationContext* opCtx) { + invariant(!_spillingUnit); + _spillingUnit = opCtx->releaseRecoveryUnit(); + _spillingState = opCtx->setRecoveryUnit(std::move(_originalUnit), _originalState); + invariant(!(_spillingUnit->getState() == RecoveryUnit::State::kInactiveInUnitOfWork || + _spillingUnit->getState() == RecoveryUnit::State::kActive)); +} + +void SpillingStore::saveState() { + _spillingUnit->abandonSnapshot(); +} +void SpillingStore::restoreState() { + // We do not have to do anything. +} + } // namespace sbe } // namespace mongo diff --git a/src/mongo/db/exec/sbe/util/spilling.h b/src/mongo/db/exec/sbe/util/spilling.h index 2d0eb98ec88..205d6f1a031 100644 --- a/src/mongo/db/exec/sbe/util/spilling.h +++ b/src/mongo/db/exec/sbe/util/spilling.h @@ -29,9 +29,14 @@ #pragma once -#include "mongo/platform/basic.h" +#include <boost/optional/optional.hpp> +#include <utility> +#include "mongo/bson/util/builder.h" #include "mongo/db/exec/sbe/values/slot.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/record_id.h" +#include "mongo/db/storage/record_store.h" #include "mongo/db/storage/temporary_record_store.h" namespace mongo { @@ -50,40 +55,104 @@ std::pair<RecordId, KeyString::TypeBits> encodeKeyString(KeyString::Builder&, // Reconstructs the KeyString carried in RecordId using 'typeBits'. KeyString::Value decodeKeyString(const RecordId& rid, KeyString::TypeBits typeBits); -// Reads a materialized row from the record store. -boost::optional<value::MaterializedRow> readFromRecordStore(OperationContext* opCtx, - RecordStore* rs, - const RecordId& rid); - -/** - * Inserts or updates a key/value into 'rs'. The 'update' flag controls whether or not an update - * will be performed. If a key/value pair is inserted into the 'rs' that already exists and - * 'update' is false, this function will tassert. - * - * Returns the size of the new record in bytes, including the record id and value portions. - */ -int upsertToRecordStore(OperationContext* opCtx, - RecordStore* rs, - const RecordId& key, - const value::MaterializedRow& val, - const KeyString::TypeBits& typeBits, - bool update); /** - * When a collator is provided, the key is encoded using the collator before being converted to a - * record id. In this case, it is not possible to recover the key from the record id, thus we need - * to store the original value of the key as well. + * SpillingStore is a wrapper around a temporary record store than maintains its own transaction as + * we do not want to intermingle operations running in the main query with spill reads and writes. */ -int upsertToRecordStore(OperationContext* opCtx, - RecordStore* rs, - const RecordId& recordKey, - const value::MaterializedRow& key, - const value::MaterializedRow& val, - bool update); -int upsertToRecordStore(OperationContext* opCtx, - RecordStore* rs, - const RecordId& key, - BufBuilder& buf, - const KeyString::TypeBits& typeBits, // recover type of value. - bool update); +class SpillingStore { +public: + SpillingStore(OperationContext* opCtx, KeyFormat format = KeyFormat::String); + ~SpillingStore(); + + /** + * When a collator is provided, the key is encoded using the collator before being converted to + * a record id. In this case, it is not possible to recover the key from the record id, thus we + * need to store the original value of the key as well. + */ + int upsertToRecordStore(OperationContext* opCtx, + const RecordId& recordKey, + const value::MaterializedRow& key, + const value::MaterializedRow& val, + bool update); + /** + * Inserts or updates a key/value into 'rs'. The 'update' flag controls whether or not an update + * will be performed. If a key/value pair is inserted into the 'rs' that already exists and + * 'update' is false, this function will tassert. + * + * Returns the size of the new record in bytes, including the record id and value portions. + */ + int upsertToRecordStore(OperationContext* opCtx, + const RecordId& key, + const value::MaterializedRow& val, + const KeyString::TypeBits& typeBits, + bool update); + int upsertToRecordStore(OperationContext* opCtx, + const RecordId& key, + BufBuilder& buf, + const KeyString::TypeBits& typeBits, // recover type of value. + bool update); + int upsertToRecordStore(OperationContext* opCtx, + const RecordId& key, + BufBuilder& buf, + bool update); + + + Status insertRecords(OperationContext* opCtx, + std::vector<Record>* inOutRecords, + const std::vector<Timestamp>& timestamps); + + // Reads a materialized row from the record store. + boost::optional<value::MaterializedRow> readFromRecordStore(OperationContext* opCtx, + const RecordId& rid); + + bool findRecord(OperationContext* opCtx, const RecordId& loc, RecordData* out); + + auto rs() { + return _recordStore->rs(); + } + + auto getCursor(OperationContext* opCtx) { + switchToSpilling(opCtx); + ON_BLOCK_EXIT([&] { switchToOriginal(opCtx); }); + return rs()->getCursor(opCtx); + } + + void resetCursor(OperationContext* opCtx, std::unique_ptr<SeekableRecordCursor>& cursor) { + switchToSpilling(opCtx); + ON_BLOCK_EXIT([&] { switchToOriginal(opCtx); }); + cursor.reset(); + } + + auto saveCursor(OperationContext* opCtx, std::unique_ptr<SeekableRecordCursor>& cursor) { + switchToSpilling(opCtx); + ON_BLOCK_EXIT([&] { switchToOriginal(opCtx); }); + + return cursor->save(); + } + + auto restoreCursor(OperationContext* opCtx, std::unique_ptr<SeekableRecordCursor>& cursor) { + switchToSpilling(opCtx); + ON_BLOCK_EXIT([&] { switchToOriginal(opCtx); }); + + return cursor->restore(); + } + + void saveState(); + void restoreState(); + +private: + void switchToSpilling(OperationContext* opCtx); + void switchToOriginal(OperationContext* opCtx); + + std::unique_ptr<TemporaryRecordStore> _recordStore; + + std::unique_ptr<RecoveryUnit> _originalUnit; + WriteUnitOfWork::RecoveryUnitState _originalState; + + std::unique_ptr<RecoveryUnit> _spillingUnit; + WriteUnitOfWork::RecoveryUnitState _spillingState; + + size_t _counter{0}; +}; } // namespace sbe } // namespace mongo diff --git a/src/mongo/db/exec/sbe/values/slot.cpp b/src/mongo/db/exec/sbe/values/slot.cpp index 2dd622fcecb..cb53de849ac 100644 --- a/src/mongo/db/exec/sbe/values/slot.cpp +++ b/src/mongo/db/exec/sbe/values/slot.cpp @@ -271,7 +271,7 @@ static void serializeValue(BufBuilder& buf, TypeTags tag, Value val) { case TypeTags::StringSmall: { // Small strings cannot contain null bytes, so it is safe to serialize them as plain // C-strings with a null terminator. - buf.appendStr(getStringView(tag, val), true /* includeEndingNull */); + buf.appendCStr(getStringView(tag, val)); break; } case TypeTags::StringBig: @@ -279,7 +279,7 @@ static void serializeValue(BufBuilder& buf, TypeTags tag, Value val) { case TypeTags::bsonSymbol: { auto sv = getStringOrSymbolView(tag, val); buf.appendNum(static_cast<uint32_t>(sv.size())); - buf.appendStr(sv, false /* includeEndingNull */); + buf.appendStrBytes(sv); break; } case TypeTags::Array: { @@ -309,7 +309,7 @@ static void serializeValue(BufBuilder& buf, TypeTags tag, Value val) { auto obj = getObjectView(val); buf.appendNum(obj->size()); for (size_t idx = 0; idx < obj->size(); ++idx) { - buf.appendStr(obj->field(idx), true /* includeEndingNull */); + buf.appendCStr(obj->field(idx)); auto [tag, val] = obj->getAt(idx); serializeValue(buf, tag, val); } @@ -352,27 +352,27 @@ static void serializeValue(BufBuilder& buf, TypeTags tag, Value val) { } case TypeTags::bsonRegex: { auto regex = getBsonRegexView(val); - buf.appendStr(regex.pattern, true /* includeEndingNull */); - buf.appendStr(regex.flags, true /* includeEndingNull */); + buf.appendCStr(regex.pattern); + buf.appendCStr(regex.flags); break; } case TypeTags::bsonJavascript: { auto javascriptCode = getBsonJavascriptView(val); buf.appendNum(static_cast<uint32_t>(javascriptCode.size())); - buf.appendStr(javascriptCode, false /* includeEndingNull */); + buf.appendStrBytes(javascriptCode); break; } case TypeTags::bsonDBPointer: { auto dbptr = getBsonDBPointerView(val); buf.appendNum(static_cast<uint32_t>(dbptr.ns.size())); - buf.appendStr(dbptr.ns, false /* includeEndingNull */); + buf.appendStrBytes(dbptr.ns); buf.appendBuf(dbptr.id, sizeof(ObjectIdType)); break; } case TypeTags::bsonCodeWScope: { auto cws = getBsonCodeWScopeView(val); buf.appendNum(static_cast<uint32_t>(cws.code.size())); - buf.appendStr(cws.code, false /* includeEndingNull */); + buf.appendStrBytes(cws.code); auto scopeLen = ConstDataView(cws.scope).read<LittleEndian<uint32_t>>(); buf.appendBuf(cws.scope, scopeLen); break; @@ -507,9 +507,10 @@ static void serializeValueIntoKeyString(KeyString::Builder& buf, } break; } + case TypeTags::bsonObjectId: case TypeTags::ObjectId: { buf.appendBool(true); - buf.appendBytes(getObjectIdView(val), sizeof(ObjectIdType)); + buf.appendOID(OID::from(getRawPointerView(val))); break; } case TypeTags::bsonObject: { @@ -532,11 +533,6 @@ static void serializeValueIntoKeyString(KeyString::Builder& buf, } break; } - case TypeTags::bsonObjectId: { - buf.appendBool(true); - buf.appendOID(OID::from(getRawPointerView(val))); - break; - } case TypeTags::bsonBinData: { BufBuilder innerBinDataBuf; innerBinDataBuf.appendUChar(static_cast<uint8_t>(tag)); diff --git a/src/mongo/db/exec/sbe/values/value_builder.h b/src/mongo/db/exec/sbe/values/value_builder.h index 00333e9f824..53748519723 100644 --- a/src/mongo/db/exec/sbe/values/value_builder.h +++ b/src/mongo/db/exec/sbe/values/value_builder.h @@ -112,21 +112,21 @@ public: } else { appendValueBufferOffset(TypeTags::StringBig); _valueBufferBuilder->appendNum(static_cast<int32_t>(in.size() + 1)); - _valueBufferBuilder->appendStr(in, true /* includeEndingNull */); + _valueBufferBuilder->appendStrBytesAndNul(in); } } void append(const BSONSymbol& in) { appendValueBufferOffset(TypeTags::bsonSymbol); _valueBufferBuilder->appendNum(static_cast<int32_t>(in.symbol.size() + 1)); - _valueBufferBuilder->appendStr(in.symbol, true /* includeEndingNull */); + _valueBufferBuilder->appendStrBytesAndNul(in.symbol); } void append(const BSONCode& in) { appendValueBufferOffset(TypeTags::bsonJavascript); // Add one to account null byte at the end. _valueBufferBuilder->appendNum(static_cast<uint32_t>(in.code.size() + 1)); - _valueBufferBuilder->appendStr(in.code, true /* includeEndingNull */); + _valueBufferBuilder->appendStrBytesAndNul(in.code); } void append(const BSONCodeWScope& in) { @@ -134,7 +134,7 @@ public: _valueBufferBuilder->appendNum( static_cast<uint32_t>(4 + in.code.size() + 1 + in.scope.objsize())); _valueBufferBuilder->appendNum(static_cast<int32_t>(in.code.size() + 1)); - _valueBufferBuilder->appendStr(in.code, true /* includeEndingNull */); + _valueBufferBuilder->appendStrBytesAndNul(in.code); _valueBufferBuilder->appendBuf(in.scope.objdata(), in.scope.objsize()); } @@ -147,14 +147,14 @@ public: void append(const BSONRegEx& in) { appendValueBufferOffset(TypeTags::bsonRegex); - _valueBufferBuilder->appendStr(in.pattern, true /* includeEndingNull */); - _valueBufferBuilder->appendStr(in.flags, true /* includeEndingNull */); + _valueBufferBuilder->appendCStr(in.pattern); + _valueBufferBuilder->appendCStr(in.flags); } void append(const BSONDBRef& in) { appendValueBufferOffset(TypeTags::bsonDBPointer); _valueBufferBuilder->appendNum(static_cast<int32_t>(in.ns.size() + 1)); - _valueBufferBuilder->appendStr(in.ns, true /* includeEndingNull */); + _valueBufferBuilder->appendStrBytesAndNul(in.ns); _valueBufferBuilder->appendBuf(in.oid.view().view(), OID::kOIDSize); } diff --git a/src/mongo/db/exec/sbe/values/value_serialization_test.cpp b/src/mongo/db/exec/sbe/values/value_serialization_test.cpp index 3be3212c627..ceacf610a4a 100644 --- a/src/mongo/db/exec/sbe/values/value_serialization_test.cpp +++ b/src/mongo/db/exec/sbe/values/value_serialization_test.cpp @@ -467,4 +467,17 @@ TEST_F(ValueSerializeForKeyString, RoundtripWideRow) { } runTest(row); } + +// Test that roundtripping through KeyString works for ObjectIdType: ObjectId; bsonObjectId. +TEST_F(ValueSerializeForKeyString, RoundtripObjectIdType) { + auto [objectIdTag, objectIdVal] = value::makeNewObjectId(); + + auto oid = OID::gen(); + auto obj = BSON("" << oid); + auto oidStorage = obj.firstElement().value(); + + sbe::value::ValueGuard testDataGuard{objectIdTag, objectIdVal}; + runTest({{objectIdTag, objectIdVal}, + {value::TypeTags::bsonObjectId, value::bitcastFrom<const char*>(oidStorage)}}); +} } // namespace mongo::sbe diff --git a/src/mongo/db/exec/sbe/vm/arith.cpp b/src/mongo/db/exec/sbe/vm/arith.cpp index 1d41ff59ce1..e6c9d380ffd 100644 --- a/src/mongo/db/exec/sbe/vm/arith.cpp +++ b/src/mongo/db/exec/sbe/vm/arith.cpp @@ -1027,7 +1027,7 @@ std::tuple<bool, value::TypeTags, value::Value> ByteCode::genericLn(value::TypeT if (!operand.isGreater(Decimal128::kNormalizedZero) && !operand.isNaN()) { return {false, value::TypeTags::Nothing, 0}; } - auto operandLn = operand.logarithm(); + auto operandLn = operand.naturalLogarithm(); auto [tag, value] = value::makeCopyDecimal(operandLn); return {true, tag, value}; diff --git a/src/mongo/db/exec/skip.cpp b/src/mongo/db/exec/skip.cpp index d3d0fc48afd..755e9278fe0 100644 --- a/src/mongo/db/exec/skip.cpp +++ b/src/mongo/db/exec/skip.cpp @@ -47,7 +47,7 @@ SkipStage::SkipStage(ExpressionContext* expCtx, long long toSkip, WorkingSet* ws, std::unique_ptr<PlanStage> child) - : PlanStage(kStageType, expCtx), _ws(ws), _toSkip(toSkip) { + : PlanStage(kStageType, expCtx), _ws(ws), _leftToSkip(toSkip), _skipAmount(toSkip) { _children.emplace_back(std::move(child)); } @@ -63,9 +63,9 @@ PlanStage::StageState SkipStage::doWork(WorkingSetID* out) { if (PlanStage::ADVANCED == status) { // If we're still skipping results... - if (_toSkip > 0) { + if (_leftToSkip > 0) { // ...drop the result. - --_toSkip; + --_leftToSkip; _ws->free(id); return PlanStage::NEED_TIME; } @@ -82,7 +82,7 @@ PlanStage::StageState SkipStage::doWork(WorkingSetID* out) { unique_ptr<PlanStageStats> SkipStage::getStats() { _commonStats.isEOF = isEOF(); - _specificStats.skip = _toSkip; + _specificStats.skip = _skipAmount; unique_ptr<PlanStageStats> ret = std::make_unique<PlanStageStats>(_commonStats, STAGE_SKIP); ret->specific = std::make_unique<SkipStats>(_specificStats); ret->children.emplace_back(child()->getStats()); diff --git a/src/mongo/db/exec/skip.h b/src/mongo/db/exec/skip.h index 24937662d02..5d0a764bd8a 100644 --- a/src/mongo/db/exec/skip.h +++ b/src/mongo/db/exec/skip.h @@ -66,8 +66,13 @@ public: private: WorkingSet* _ws; - // We drop the first _toSkip results that we would have returned. - long long _toSkip; + // The number of results left to skip. This number is decremented during query execution as we + // successfully skip a document. + long long _leftToSkip; + + // Represents the number of results to skip. Unlike '_leftToSkip', this remains constant and + // is used when gathering statistics in explain. + const long long _skipAmount; // Stats SkipStats _specificStats; diff --git a/src/mongo/db/exec/text_or.cpp b/src/mongo/db/exec/text_or.cpp index a0ba3eb347e..f9e0745ca16 100644 --- a/src/mongo/db/exec/text_or.cpp +++ b/src/mongo/db/exec/text_or.cpp @@ -104,9 +104,7 @@ std::unique_ptr<PlanStageStats> TextOrStage::getStats() { _commonStats.isEOF = isEOF(); if (_filter) { - BSONObjBuilder bob; - _filter->serialize(&bob); - _commonStats.filter = bob.obj(); + _commonStats.filter = _filter->serialize(); } unique_ptr<PlanStageStats> ret = std::make_unique<PlanStageStats>(_commonStats, STAGE_TEXT_OR); diff --git a/src/mongo/db/exec/upsert_stage.cpp b/src/mongo/db/exec/upsert_stage.cpp index 212880f6af3..dfb54a99082 100644 --- a/src/mongo/db/exec/upsert_stage.cpp +++ b/src/mongo/db/exec/upsert_stage.cpp @@ -278,6 +278,8 @@ void UpsertStage::_generateNewDocumentFromSuppliedDoc(const FieldRefSet& immutab suppliedDoc, write_ops::UpdateModification::ClassicTag{}, true /* isReplacement */), {}); replacementDriver.setLogOp(false); + replacementDriver.setBypassEmptyTsReplacement( + static_cast<bool>(_params.request->getBypassEmptyTsReplacement())); // We do not validate for storage, as we will validate the full document before inserting. // However, we ensure that no immutable fields are modified. diff --git a/src/mongo/db/ftdc/SConscript b/src/mongo/db/ftdc/SConscript index 72a96523195..6a21cf9efdc 100644 --- a/src/mongo/db/ftdc/SConscript +++ b/src/mongo/db/ftdc/SConscript @@ -25,6 +25,7 @@ ftdcEnv.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/bson/util/bson_extract', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/service_context', diff --git a/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp index 121a51e3489..3f13d8bc080 100644 --- a/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp +++ b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp @@ -33,6 +33,7 @@ #include <memory> #include <string> +#include <sys/resource.h> #include <vector> #include "mongo/base/status.h" @@ -40,6 +41,8 @@ #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/ftdc/collector.h" #include "mongo/db/ftdc/controller.h" +#include "mongo/util/errno_util.h" +#include "mongo/util/functional.h" #include "mongo/util/processinfo.h" #include "mongo/util/procparser.h" @@ -181,10 +184,66 @@ private: std::vector<StringData> _disksStringData; }; +class SimpleFunctionCollector final : public FTDCCollectorInterface { +public: + SimpleFunctionCollector(StringData name, + unique_function<void(OperationContext*, BSONObjBuilder&)> collectFn) + : _name(name.toString()), _collectFn(std::move(collectFn)) {} + + void collect(OperationContext* opCtx, BSONObjBuilder& builder) override { + _collectFn(opCtx, builder); + } + + std::string name() const override { + return _name; + } + +private: + std::string _name; + unique_function<void(OperationContext*, BSONObjBuilder&)> _collectFn; +}; + + +void collectUlimit(int resource, StringData resourceName, BSONObjBuilder& builder) { + + struct rlimit rlim; + + BSONObjBuilder subObjBuilder(builder.subobjStart(resourceName)); + + if (!getrlimit(resource, &rlim)) { + subObjBuilder.append("soft", static_cast<int64_t>(rlim.rlim_cur)); + subObjBuilder.append("hard", static_cast<int64_t>(rlim.rlim_max)); + } else { + auto ec = lastSystemError(); + + subObjBuilder.append("error", errorMessage(ec)); + } +} + +void collectUlimits(OperationContext*, BSONObjBuilder& builder) { + collectUlimit(RLIMIT_CPU, "cpuTime_secs"_sd, builder); + collectUlimit(RLIMIT_FSIZE, "fileSize_blocks"_sd, builder); + collectUlimit(RLIMIT_DATA, "dataSegSize_kb"_sd, builder); + collectUlimit(RLIMIT_STACK, "stackSize_kb"_sd, builder); + collectUlimit(RLIMIT_CORE, "coreFileSize_blocks"_sd, builder); + collectUlimit(RLIMIT_RSS, "residentSize_kb"_sd, builder); + collectUlimit(RLIMIT_NOFILE, "fileDescriptors"_sd, builder); + collectUlimit(RLIMIT_AS, "addressSpace_kb"_sd, builder); + collectUlimit(RLIMIT_NPROC, "processes"_sd, builder); + collectUlimit(RLIMIT_MEMLOCK, "memLock_kb"_sd, builder); + collectUlimit(RLIMIT_LOCKS, "fileLocks"_sd, builder); + collectUlimit(RLIMIT_SIGPENDING, "pendingSignals"_sd, builder); +} + } // namespace + void installSystemMetricsCollector(FTDCController* controller) { controller->addPeriodicCollector(std::make_unique<LinuxSystemMetricsCollector>()); + + // Collect ULimits settings on rotation. + controller->addOnRotateCollector( + std::make_unique<SimpleFunctionCollector>("ulimits", collectUlimits)); } } // namespace mongo diff --git a/src/mongo/db/geo/geoparser.cpp b/src/mongo/db/geo/geoparser.cpp index 005f6defef2..d7a353fe6e4 100644 --- a/src/mongo/db/geo/geoparser.cpp +++ b/src/mongo/db/geo/geoparser.cpp @@ -49,21 +49,37 @@ namespace mongo { namespace dps = ::mongo::dotted_path_support; -static Status parseFlatPoint(const BSONElement& elem, Point* out, bool allowAddlFields = false) { - if (!elem.isABSONObj()) - return BAD_VALUE("Point must be an array or object"); +Status GeoParser::parseFlatPointCoordinates(const BSONElement& elem, + BSONElement& x, + BSONElement& y, + bool allowAddlFields /* = false */) { + if (!elem.isABSONObj()) { + return BAD_VALUE("Point must be an array or object, instead got type " + << typeName(elem.type())); + } + BSONObjIterator it(elem.Obj()); - BSONElement x = it.next(); + x = it.next(); if (!x.isNumber()) { return BAD_VALUE("Point must only contain numeric elements"); } - BSONElement y = it.next(); + y = it.next(); if (!y.isNumber()) { return BAD_VALUE("Point must only contain numeric elements"); } if (!allowAddlFields && it.more()) { return BAD_VALUE("Point must only contain two numeric elements"); } + return Status::OK(); +} + +static Status parseFlatPoint(const BSONElement& elem, Point* out, bool allowAddlFields = false) { + BSONElement x, y; + auto status = GeoParser::parseFlatPointCoordinates(elem, x, y, allowAddlFields); + if (!status.isOK()) { + return status; + } + out->x = x.number(); out->y = y.number(); // Point coordinates must be finite numbers, neither NaN or infinite. @@ -768,20 +784,23 @@ GeoParser::GeoJSONType GeoParser::parseGeoJSONType(const BSONObj& obj) { if (String != type.type()) { return GeoParser::GEOJSON_UNKNOWN; } - const string& typeString = type.String(); - if (GEOJSON_TYPE_POINT == typeString) { + return geoJSONTypeStringToEnum(type.checkAndGetStringData()); +} + +GeoParser::GeoJSONType GeoParser::geoJSONTypeStringToEnum(StringData type) { + if (GEOJSON_TYPE_POINT == type) { return GeoParser::GEOJSON_POINT; - } else if (GEOJSON_TYPE_LINESTRING == typeString) { + } else if (GEOJSON_TYPE_LINESTRING == type) { return GeoParser::GEOJSON_LINESTRING; - } else if (GEOJSON_TYPE_POLYGON == typeString) { + } else if (GEOJSON_TYPE_POLYGON == type) { return GeoParser::GEOJSON_POLYGON; - } else if (GEOJSON_TYPE_MULTI_POINT == typeString) { + } else if (GEOJSON_TYPE_MULTI_POINT == type) { return GeoParser::GEOJSON_MULTI_POINT; - } else if (GEOJSON_TYPE_MULTI_LINESTRING == typeString) { + } else if (GEOJSON_TYPE_MULTI_LINESTRING == type) { return GeoParser::GEOJSON_MULTI_LINESTRING; - } else if (GEOJSON_TYPE_MULTI_POLYGON == typeString) { + } else if (GEOJSON_TYPE_MULTI_POLYGON == type) { return GeoParser::GEOJSON_MULTI_POLYGON; - } else if (GEOJSON_TYPE_GEOMETRY_COLLECTION == typeString) { + } else if (GEOJSON_TYPE_GEOMETRY_COLLECTION == type) { return GeoParser::GEOJSON_GEOMETRY_COLLECTION; } return GeoParser::GEOJSON_UNKNOWN; diff --git a/src/mongo/db/geo/geoparser.h b/src/mongo/db/geo/geoparser.h index 4d4d1185235..cba5893cf37 100644 --- a/src/mongo/db/geo/geoparser.h +++ b/src/mongo/db/geo/geoparser.h @@ -92,12 +92,17 @@ public: static GeoSpecifier parseGeoSpecifier(const BSONElement& elem); static GeoJSONType parseGeoJSONType(const BSONObj& obj); + static GeoJSONType geoJSONTypeStringToEnum(StringData type); // Legacy points can contain extra data as extra fields - these are valid to index // e.g. { x: 1, y: 1, z: 1 } static Status parseLegacyPoint(const BSONElement& elem, PointWithCRS* out, bool allowAddlFields = false); + static Status parseFlatPointCoordinates(const BSONElement& elem, + BSONElement& x, + BSONElement& y, + bool allowAddlFields = false); // Parse the BSON object after $box, $center, etc. static Status parseLegacyBox(const BSONObj& obj, BoxWithCRS* out); static Status parseLegacyCenter(const BSONObj& obj, CapWithCRS* out); diff --git a/src/mongo/db/global_index_usage_tracker.cpp b/src/mongo/db/global_index_usage_tracker.cpp index 9308307e82a..18849b73254 100644 --- a/src/mongo/db/global_index_usage_tracker.cpp +++ b/src/mongo/db/global_index_usage_tracker.cpp @@ -44,6 +44,7 @@ static const std::string kHashed = "hashed"; static const std::string kId = "id"; static const std::string kNormal = "normal"; static const std::string kPartial = "partial"; +static const std::string kPrepareUnique = "prepareUnique"; static const std::string kSingle = "single"; static const std::string kSparse = "sparse"; static const std::string kText = "text"; @@ -62,6 +63,7 @@ std::map<std::string, IndexFeatureStats> makeFeatureMap() { map[kId]; map[kNormal]; map[kPartial]; + map[kPrepareUnique]; map[kSingle]; map[kSparse]; map[kText]; @@ -92,6 +94,7 @@ IndexFeatures IndexFeatures::make(const IndexDescriptor* desc, bool internal) { features.id = desc->isIdIndex(); features.internal = internal; features.partial = desc->isPartial(); + features.prepareUnique = desc->prepareUnique(); features.sparse = desc->isSparse(); features.ttl = desc->infoObj().hasField(IndexDescriptor::kExpireAfterSecondsFieldName); features.type = indexType; @@ -174,6 +177,9 @@ void GlobalIndexUsageTracker::_updateStatsForEachFeature(const IndexFeatures& fe if (features.partial) { update(&_indexFeatureToStats.at(kPartial)); } + if (features.prepareUnique) { + update(&_indexFeatureToStats.at(kPrepareUnique)); + } if (features.sparse) { update(&_indexFeatureToStats.at(kSparse)); } diff --git a/src/mongo/db/global_index_usage_tracker.h b/src/mongo/db/global_index_usage_tracker.h index c4711667787..55dfa8405ab 100644 --- a/src/mongo/db/global_index_usage_tracker.h +++ b/src/mongo/db/global_index_usage_tracker.h @@ -56,7 +56,7 @@ struct IndexFeatures { bool id = false; bool internal = false; bool partial = false; - bool regular = false; + bool prepareUnique = false; bool sparse = false; bool ttl = false; bool unique = false; diff --git a/src/mongo/db/index/SConscript b/src/mongo/db/index/SConscript index 7b191838e2f..9207e32aa49 100644 --- a/src/mongo/db/index/SConscript +++ b/src/mongo/db/index/SConscript @@ -63,7 +63,6 @@ serveronlyEnv.Library( '$BUILD_DIR/mongo/db/catalog/index_catalog', '$BUILD_DIR/mongo/db/catalog/index_catalog_entry', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/exec/projection_executor', '$BUILD_DIR/mongo/db/exec/working_set', '$BUILD_DIR/mongo/db/fts/base_fts', @@ -74,6 +73,7 @@ serveronlyEnv.Library( '$BUILD_DIR/mongo/db/query/collation/collator_factory_interface', '$BUILD_DIR/mongo/db/query/collation/collator_interface', '$BUILD_DIR/mongo/db/query/projection_ast', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query/sort_pattern', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', diff --git a/src/mongo/db/index/index_access_method.cpp b/src/mongo/db/index/index_access_method.cpp index 5727bfea1df..4d454f2b5a3 100644 --- a/src/mongo/db/index/index_access_method.cpp +++ b/src/mongo/db/index/index_access_method.cpp @@ -412,6 +412,13 @@ void SortedDataIndexAccessMethod::removeOneKey(OperationContext* opCtx, try { _newInterface->unindex(opCtx, keyString, dupsAllowed); } catch (AssertionException& e) { + if (e.code() == ErrorCodes::DataCorruptionDetected) { + // DataCorruptionDetected errors are expected to have logged an error and added an entry + // to the health log with the stack trace at the location where the error was initially + // thrown. No need to do so again. + throw; + } + NamespaceString ns = _indexCatalogEntry->getNSSFromCatalog(opCtx); LOGV2(20683, "Assertion failure: _unindex failed on: {namespace} for index: {indexName}. " @@ -631,7 +638,8 @@ Status SortedDataIndexAccessMethod::doUpdate(OperationContext* opCtx, // Add all new data keys into the index. for (const auto& keyString : ticket.added) { - bool dupsAllowed = !_descriptor->prepareUnique() && ticket.dupsAllowed; + bool dupsAllowed = (!_descriptor->prepareUnique() || !opCtx->isEnforcingConstraints()) && + ticket.dupsAllowed; auto status = _newInterface->insert(opCtx, keyString, dupsAllowed); if (!status.isOK()) return status; diff --git a/src/mongo/db/index/index_descriptor.cpp b/src/mongo/db/index/index_descriptor.cpp index 0fd676a8353..9c3c8a91c67 100644 --- a/src/mongo/db/index/index_descriptor.cpp +++ b/src/mongo/db/index/index_descriptor.cpp @@ -144,10 +144,16 @@ IndexDescriptor::IndexDescriptor(const std::string& accessMethodName, BSONObj in } if (BSONElement prepareUniqueElement = _infoObj[kPrepareUniqueFieldName]) { - uassert( - ErrorCodes::InvalidOptions, - "Index does not support the 'prepareUnique' field", - feature_flags::gCollModIndexUnique.isEnabled(serverGlobalParams.featureCompatibility)); + // If FCV is initialized, check if prepareUnique is supported. + // Otherwise, we are in startup recovery and reading the option from disk, and can skip + // the check because this option could only be gnerated by 6.0+, and we already prevent + // FCV downgrades from 6.0 if any indexes contain prepareUnique. + if (serverGlobalParams.featureCompatibility.isVersionInitialized()) { + uassert(ErrorCodes::InvalidOptions, + "Index does not support the 'prepareUnique' field", + feature_flags::gCollModIndexUnique.isEnabled( + serverGlobalParams.featureCompatibility)); + } _prepareUnique = prepareUniqueElement.trueValue(); } diff --git a/src/mongo/db/index_builds_coordinator.cpp b/src/mongo/db/index_builds_coordinator.cpp index d0dab080431..e6f6d2110ab 100644 --- a/src/mongo/db/index_builds_coordinator.cpp +++ b/src/mongo/db/index_builds_coordinator.cpp @@ -93,6 +93,7 @@ MONGO_FAIL_POINT_DEFINE(hangBeforeBuildingIndex); MONGO_FAIL_POINT_DEFINE(hangBeforeBuildingIndexSecond); MONGO_FAIL_POINT_DEFINE(hangIndexBuildBeforeWaitingUntilMajorityOpTime); MONGO_FAIL_POINT_DEFINE(failSetUpResumeIndexBuild); +MONGO_FAIL_POINT_DEFINE(hangAbortIndexBuildByBuildUUIDAfterLocks); IndexBuildsCoordinator::ActiveIndexBuildsSSS::ActiveIndexBuildsSSS() : ServerStatusSection("activeIndexBuilds"), @@ -556,81 +557,52 @@ Status IndexBuildsCoordinator::_startIndexBuildForRecovery(OperationContext* opC CollectionWriter collection(opCtx, nss); { - // These steps are combined into a single WUOW to ensure there are no commits without - // the indexes. - // 1) Drop all unfinished indexes. - // 2) Start, but do not complete the index build process. + // These steps are combined into a single WUOW to ensure there are no commits without the + // indexes for repair. WriteUnitOfWork wuow(opCtx); - auto indexCatalog = collection.getWritableCollection()->getIndexCatalog(); + // We need to initialize the collection to rebuild the indexes. The collection may already + // be initialized when rebuilding multiple unfinished indexes on the same collection. + if (!collection->isInitialized()) { + collection.getWritableCollection()->init(opCtx); + } - for (size_t i = 0; i < indexNames.size(); i++) { - auto descriptor = indexCatalog->findIndexByName( - opCtx, indexNames[i], IndexCatalog::InclusionPolicy::kReady); - if (descriptor) { - Status s = - indexCatalog->dropIndex(opCtx, collection.getWritableCollection(), descriptor); - if (!s.isOK()) { - return s; - } - continue; + if (storageGlobalParams.repair) { + Status status = _dropIndexesForRepair(opCtx, collection, indexNames); + if (!status.isOK()) { + return status; } + } else { + // Unfinished index builds that are not resumable will drop and recreate the index table + // using the same ident to avoid doing untimestamped writes to the catalog. + for (const auto& indexName : indexNames) { + auto indexCatalog = collection.getWritableCollection()->getIndexCatalog(); + auto desc = + indexCatalog->findIndexByName(opCtx, + indexName, + IndexCatalog::InclusionPolicy::kUnfinished | + IndexCatalog::InclusionPolicy::kFrozen); + Status status = indexCatalog->resetUnfinishedIndexForRecovery( + opCtx, collection.getWritableCollection(), desc); + if (!status.isOK()) { + return status; + } - // If the index is not present in the catalog, then we are trying to drop an already - // aborted index. This may happen when rollback-via-refetch restarts an index build - // after an abort has been rolled back. - if (!collection->isIndexPresent(indexNames[i])) { - LOGV2(20652, - "An index was not found in the catalog while trying to drop the index during " - "recovery", - "buildUUID"_attr = buildUUID, - "index"_attr = indexNames[i]); - continue; - } + const auto durableBuildUUID = collection->getIndexBuildUUID(indexName); - const auto durableBuildUUID = collection->getIndexBuildUUID(indexNames[i]); - - // A build UUID is present if and only if we are rebuilding a two-phase build. - invariant((protocol == IndexBuildProtocol::kTwoPhase) == - durableBuildUUID.is_initialized()); - // When a buildUUID is present, it must match the build UUID parameter to this - // function. - invariant(!durableBuildUUID || *durableBuildUUID == buildUUID, - str::stream() << "durable build UUID: " << durableBuildUUID - << "buildUUID: " << buildUUID); - - // If the unfinished index is in the IndexCatalog, drop it through there, otherwise drop - // it from the DurableCatalog. Rollback-via-refetch does not clear any in-memory state, - // so we should do it manually here. - descriptor = indexCatalog->findIndexByName( - opCtx, - indexNames[i], - IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); - if (descriptor) { - Status s = indexCatalog->dropUnfinishedIndex( - opCtx, collection.getWritableCollection(), descriptor); - if (!s.isOK()) { - return s; - } - } else { - // There are no concurrent users of the index during startup recovery, so it is OK - // to pass in a nullptr for the index 'ident', promising that the index is not in - // use. - catalog::removeIndex( - opCtx, indexNames[i], collection.getWritableCollection(), nullptr /* ident */); + // A build UUID is present if and only if we are rebuilding a two-phase build. + invariant((protocol == IndexBuildProtocol::kTwoPhase) == + durableBuildUUID.has_value()); + // When a buildUUID is present, it must match the build UUID parameter to this + // function. + invariant(!durableBuildUUID || *durableBuildUUID == buildUUID, + str::stream() << "durable build UUID: " << durableBuildUUID + << "buildUUID: " << buildUUID); } } - // We need to initialize the collection to rebuild the indexes. The collection may already - // be initialized when rebuilding indexes with rollback-via-refetch. - if (!collection->isInitialized()) { - collection.getWritableCollection()->init(opCtx); - } - - auto dbName = nss.db().toString(); auto replIndexBuildState = std::make_shared<ReplIndexBuildState>( - buildUUID, collection->uuid(), dbName, specs, protocol); + buildUUID, collection->uuid(), nss.db().toString(), specs, protocol); Status status = activeIndexBuilds.registerIndexBuild(replIndexBuildState); if (!status.isOK()) { @@ -639,6 +611,8 @@ Status IndexBuildsCoordinator::_startIndexBuildForRecovery(OperationContext* opC IndexBuildsManager::SetupOptions options; options.protocol = protocol; + // All indexes are dropped during repair and should be rebuilt normally. + options.forRecovery = !storageGlobalParams.repair; status = _indexBuildsManager.setUpIndexBuild( opCtx, collection, specs, buildUUID, MultiIndexBlock::kNoopOnInitFn, options); if (!status.isOK()) { @@ -653,6 +627,39 @@ Status IndexBuildsCoordinator::_startIndexBuildForRecovery(OperationContext* opC return Status::OK(); } +Status IndexBuildsCoordinator::_dropIndexesForRepair(OperationContext* opCtx, + CollectionWriter& collection, + const std::vector<std::string>& indexNames) { + invariant(collection->isInitialized()); + for (const auto& indexName : indexNames) { + auto indexCatalog = collection.getWritableCollection()->getIndexCatalog(); + auto descriptor = + indexCatalog->findIndexByName(opCtx, indexName, IndexCatalog::InclusionPolicy::kReady); + if (descriptor) { + Status s = + indexCatalog->dropIndex(opCtx, collection.getWritableCollection(), descriptor); + if (!s.isOK()) { + return s; + } + continue; + } + + // The index must be unfinished or frozen if it isn't ready. + descriptor = indexCatalog->findIndexByName(opCtx, + indexName, + IndexCatalog::InclusionPolicy::kUnfinished | + IndexCatalog::InclusionPolicy::kFrozen); + invariant(descriptor); + Status s = indexCatalog->dropUnfinishedIndex( + opCtx, collection.getWritableCollection(), descriptor); + if (!s.isOK()) { + return s; + } + } + + return Status::OK(); +} + Status IndexBuildsCoordinator::_setUpResumeIndexBuild(OperationContext* opCtx, std::string dbName, const UUID& collectionUUID, @@ -949,6 +956,16 @@ void IndexBuildsCoordinator::applyStartIndexBuild(OperationContext* opCtx, uassertStatusOK( indexCatalog->dropIndex(opCtx, coll.getWritableCollection(opCtx), desc)); } + + const IndexDescriptor* desc = indexCatalog->findIndexByKeyPatternAndOptions( + opCtx, + spec.getObjectField(IndexDescriptor::kKeyPatternFieldName), + spec, + IndexCatalog::InclusionPolicy::kReady); + if (desc) { + uassertStatusOK( + indexCatalog->dropIndex(opCtx, coll.getWritableCollection(opCtx), desc)); + } } wuow.commit(); @@ -1251,6 +1268,8 @@ bool IndexBuildsCoordinator::abortIndexBuildByBuildUUID(OperationContext* opCtx, AutoGetCollection indexBuildEntryColl( opCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); + hangAbortIndexBuildByBuildUUIDAfterLocks.pauseWhileSet(); + // If we are using two-phase index builds and are no longer primary after receiving an // abort, we cannot replicate an abortIndexBuild oplog entry. Continue holding the RSTL to // check the replication state and to prevent any state transitions from happening while @@ -2248,11 +2267,26 @@ void IndexBuildsCoordinator::_cleanUpSinglePhaseAfterFailure( runOnAlternateContext( opCtx, "self-abort", [this, replState, status](OperationContext* abortCtx) { ShouldNotConflictWithSecondaryBatchApplicationBlock noConflict(abortCtx->lockState()); - auto autoGetColl = std::move( - _autoGetCollectionExclusiveWithTimeout(abortCtx, replState.get()).getValue()); - AutoGetCollection indexBuildEntryColl( - abortCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); - _completeSelfAbort(abortCtx, replState, *indexBuildEntryColl, status); + // To avoid potential deadlocks with concurrent external aborts, which hold the + // collection MODE_X lock while waiting for this thread to signal its exit, the + // collection lock is acquired with a timeout, and retried only if the build is not + // already aborted (externally). + while (!replState->isAborted()) { + auto swLocks = + _autoGetCollectionExclusiveWithTimeout(abortCtx, replState.get(), false); + if (!swLocks.isOK()) { + LOGV2(7677700, + "Unable to acquire collection lock within the timeout, a concurrent " + "abort might be waiting for the builder thread to exit. Rechecking if " + "self abort is still required.", + "buildUUID"_attr = replState->buildUUID); + continue; + } + + AutoGetCollection indexBuildEntryColl( + abortCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); + _completeSelfAbort(abortCtx, replState, *indexBuildEntryColl, status); + } }); } @@ -2274,23 +2308,40 @@ void IndexBuildsCoordinator::_cleanUpTwoPhaseAfterFailure( opCtx, "self-abort", [this, replState, status](OperationContext* abortCtx) { ShouldNotConflictWithSecondaryBatchApplicationBlock noConflict(abortCtx->lockState()); - auto autoGetColl = std::move( - _autoGetCollectionExclusiveWithTimeout(abortCtx, replState.get()).getValue()); - - // Index builds may not fail on secondaries. If a primary replicated an abortIndexBuild - // oplog entry, then this index build would have received an IndexBuildAborted error - // code. - const NamespaceStringOrUUID dbAndUUID(replState->dbName, replState->collectionUUID); - auto replCoord = repl::ReplicationCoordinator::get(abortCtx); - if (!replCoord->canAcceptWritesFor(abortCtx, dbAndUUID)) { - fassert(51101, - status.withContext(str::stream() << "Index build: " << replState->buildUUID - << "; Database: " << replState->dbName)); - } + // To avoid potential deadlocks with concurrent external aborts, which hold the + // collection MODE_X lock while waiting for this thread to signal its exit, the + // collection lock is acquired with a timeout, and retried only if the build is not + // already aborted (externally). + while (!replState->isAborted()) { + // Take RSTL to observe and prevent replication state from changing. This is done + // with the release/reacquire strategy to avoid deadlock with prepared txns. + auto swLocks = + _autoGetCollectionExclusiveWithTimeout(abortCtx, replState.get(), false); + if (!swLocks.isOK()) { + LOGV2_DEBUG(7677701, + 1, + "Index build: lock acquisition for self-abort failed, will retry.", + "buildUUD"_attr = replState->buildUUID, + "error"_attr = swLocks.getStatus()); + continue; + } + + const NamespaceStringOrUUID dbAndUUID(replState->dbName, replState->collectionUUID); + auto replCoord = repl::ReplicationCoordinator::get(abortCtx); + if (!replCoord->canAcceptWritesFor(abortCtx, dbAndUUID)) { + // Index builds may not fail on secondaries. If a primary replicated an + // abortIndexBuild oplog entry, then this index build would have received an + // IndexBuildAborted error code. + fassert(51101, + status.withContext(str::stream() + << "Index build: " << replState->buildUUID + << "; Database: " << replState->dbName)); + } - AutoGetCollection indexBuildEntryColl( - abortCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); - _completeSelfAbort(abortCtx, replState, *indexBuildEntryColl, status); + AutoGetCollection indexBuildEntryColl( + abortCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); + _completeSelfAbort(abortCtx, replState, *indexBuildEntryColl, status); + } }); } @@ -2348,6 +2399,13 @@ void IndexBuildsCoordinator::_runIndexBuildInner( uassertStatusOK(status); } + // It is also possible for the concurrent abort to happen after the check. This is an issue as + // external aborters hold the collection MODE_X lock while waiting for this thread to signal the + // promise, but if this thread proceeds beyond this check first it will try to acquire the + // collection lock before signaling the promise, potentially creating a deadlock. This is worked + // around by adding a timeout to the collection lock in the self-abort path, and rechecking if + // the build was aborted externally on timeout. + // We do not hold a collection lock here, but we are protected against the collection being // dropped while the index build is still registered for the collection -- until abortIndexBuild // is called. The collection can be renamed, but it is OK for the name to be stale just for diff --git a/src/mongo/db/index_builds_coordinator.h b/src/mongo/db/index_builds_coordinator.h index 4ff14fc360f..8d6cc91c54d 100644 --- a/src/mongo/db/index_builds_coordinator.h +++ b/src/mongo/db/index_builds_coordinator.h @@ -532,8 +532,8 @@ private: /** * Sets up the in-memory and durable state of the index build. * - * This function should only be called when in recovery mode, because we drop and replace - * existing indexes in a single WriteUnitOfWork. + * This function should only be called when in recovery mode, because the index tables are + * recreated. */ Status _startIndexBuildForRecovery(OperationContext* opCtx, const NamespaceString& nss, @@ -541,6 +541,16 @@ private: const UUID& buildUUID, IndexBuildProtocol protocol); + /** + * Removes the in-memory and durable state of the passed in indexes in preparation of rebuilding + * them for repair. + * + * This function should only be called when in recovery mode. + */ + Status _dropIndexesForRepair(OperationContext* opCtx, + CollectionWriter& collection, + const std::vector<std::string>& indexNames); + protected: /** * Acquire the collection MODE_X lock (and other locks up the hierarchy) as usual, with a diff --git a/src/mongo/db/initialize_operation_session_info.cpp b/src/mongo/db/initialize_operation_session_info.cpp index a3b75ec443d..5eccaf3faa7 100644 --- a/src/mongo/db/initialize_operation_session_info.cpp +++ b/src/mongo/db/initialize_operation_session_info.cpp @@ -36,9 +36,20 @@ #include "mongo/db/logical_session_cache.h" #include "mongo/db/logical_session_id_helpers.h" #include "mongo/db/operation_context.h" +#include "mongo/db/stats/counters.h" namespace mongo { +/** + * A client is internal if the connection is a self connection, a connection from a mongos or + * different mongod or a direct client connection. + */ +bool isInternalClient(OperationContext* opCtx) { + return !opCtx->getClient()->session() || + (opCtx->getClient()->session()->getTags() & transport::Session::kInternalClient) || + opCtx->getClient()->isInDirectClient(); +} + OperationSessionInfoFromClient initializeOperationSessionInfo(OperationContext* opCtx, const BSONObj& requestBody, bool requiresAuth, @@ -166,6 +177,20 @@ OperationSessionInfoFromClient initializeOperationSessionInfo(OperationContext* osi.getStartTransaction().value()); } + if (osi.getTxnNumber()) { + if (!osi.getAutocommit()) { + if (isInternalClient(opCtx)) { + internalRetryableWriteCount.increment(1); + } else { + externalRetryableWriteCount.increment(1); + } + } else { + if (osi.getSessionId()->getTxnNumber() && osi.getSessionId()->getTxnUUID()) { + retryableInternalTransactionCount.increment(1); + } + } + } + return osi; } diff --git a/src/mongo/db/introspect.cpp b/src/mongo/db/introspect.cpp index 18608a8dad0..cbbb710e6d7 100644 --- a/src/mongo/db/introspect.cpp +++ b/src/mongo/db/introspect.cpp @@ -88,7 +88,7 @@ void profile(OperationContext* opCtx, NetworkOp op) { AuthorizationSession* authSession = AuthorizationSession::get(opCtx->getClient()); OpDebug::appendUserInfo(*CurOp::get(opCtx), b, authSession); - const BSONObj p = b.done(); + const BSONObj p = b.done().redact(BSONObj::RedactLevel::sensitiveOnly); const string dbName(nsToDatabase(CurOp::get(opCtx)->getNS())); diff --git a/src/mongo/db/keypattern.h b/src/mongo/db/keypattern.h index 986d259eebd..034bfb5833d 100644 --- a/src/mongo/db/keypattern.h +++ b/src/mongo/db/keypattern.h @@ -33,6 +33,7 @@ #include "mongo/base/string_data.h" #include "mongo/bson/util/builder.h" #include "mongo/db/jsobj.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/util/str.h" namespace mongo { @@ -84,6 +85,14 @@ public: return _pattern; } + BSONObj serializeForIDL(const SerializationOptions& options = {}) const { + BSONObjBuilder bob; + for (const auto& e : _pattern) { + bob.appendAs(e, options.serializeIdentifier(e.fieldNameStringData())); + } + return bob.obj(); + } + /** * Returns a string representation of this KeyPattern. */ diff --git a/src/mongo/db/keypattern.idl b/src/mongo/db/keypattern.idl index 158c742faab..cd67f8e5428 100644 --- a/src/mongo/db/keypattern.idl +++ b/src/mongo/db/keypattern.idl @@ -38,5 +38,5 @@ types: bson_serialization_type: object description: An expression describing a transformation of a document into a document key. cpp_type: KeyPattern - serializer: KeyPattern::toBSON + serializer: KeyPattern::serializeForIDL deserializer: KeyPattern::fromBSON diff --git a/src/mongo/db/keys_collection_client_direct.cpp b/src/mongo/db/keys_collection_client_direct.cpp index 30048be948b..0a2836a8f07 100644 --- a/src/mongo/db/keys_collection_client_direct.cpp +++ b/src/mongo/db/keys_collection_client_direct.cpp @@ -164,13 +164,23 @@ Status KeysCollectionClientDirect::_insert(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& doc, const WriteConcernOptions& writeConcern) { - BatchedCommandRequest request([&] { + BatchedCommandRequest batchRequest([&] { write_ops::InsertCommandRequest insertOp(nss); insertOp.setDocuments({doc}); return insertOp; }()); - request.setWriteConcern(writeConcern.toBSON()); - const BSONObj cmdObj = request.toBSON(); + + // A request dispatched through a local client is served within the same thread that submits it + // (so that the opCtx needs to be used as the vehicle to pass the WC to the ServiceEntryPoint). + const auto originalWC = opCtx->getWriteConcern(); + ScopeGuard resetWCGuard([&] { opCtx->setWriteConcern(originalWC); }); + opCtx->setWriteConcern(writeConcern); + + const BSONObj cmdObj = [&] { + BSONObjBuilder cmdObjBuilder; + batchRequest.serialize(&cmdObjBuilder); + return cmdObjBuilder.obj(); + }(); for (int retry = 1; retry <= kOnErrorNumRetries; ++retry) { // Note: write commands can only be issued against a primary. diff --git a/src/mongo/db/matcher/SConscript b/src/mongo/db/matcher/SConscript index 4d484b0fadb..27ae56fac0e 100644 --- a/src/mongo/db/matcher/SConscript +++ b/src/mongo/db/matcher/SConscript @@ -27,6 +27,7 @@ env.Library( 'expression_array.cpp', 'expression_expr.cpp', 'expression_geo.cpp', + 'expression_geo_serializer.cpp', 'expression_internal_bucket_geo_within.cpp', 'expression_leaf.cpp', 'expression_parameterization.cpp', @@ -73,6 +74,7 @@ env.Library( '$BUILD_DIR/mongo/db/pipeline/expression_context', '$BUILD_DIR/mongo/db/query/collation/collator_interface', '$BUILD_DIR/mongo/db/query/query_knobs', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape_common', '$BUILD_DIR/mongo/db/stats/counters', '$BUILD_DIR/mongo/idl/idl_parser', '$BUILD_DIR/mongo/util/regex_util', diff --git a/src/mongo/db/matcher/expression.cpp b/src/mongo/db/matcher/expression.cpp index bb13e28a9e0..a228fd86916 100644 --- a/src/mongo/db/matcher/expression.cpp +++ b/src/mongo/db/matcher/expression.cpp @@ -175,6 +175,74 @@ void MatchExpression::addDependencies(DepsTracker* deps) const { _doAddDependencies(deps); } +bool MatchExpression::isInternalNodeWithPath(MatchType m) { + switch (m) { + case ELEM_MATCH_OBJECT: + case ELEM_MATCH_VALUE: + case INTERNAL_SCHEMA_OBJECT_MATCH: + case INTERNAL_SCHEMA_MATCH_ARRAY_INDEX: + // This node generates a child expression with a field that isn't prefixed by the path + // of the node. + case INTERNAL_SCHEMA_ALL_ELEM_MATCH_FROM_INDEX: + // This node generates a child expression with a field that isn't prefixed by the path + // of the node. + return true; + + case AND: + case OR: + case SIZE: + case EQ: + case LTE: + case LT: + case GT: + case GTE: + case REGEX: + case MOD: + case EXISTS: + case MATCH_IN: + case BITS_ALL_SET: + case BITS_ALL_CLEAR: + case BITS_ANY_SET: + case BITS_ANY_CLEAR: + case NOT: + case NOR: + case TYPE_OPERATOR: + case GEO: + case WHERE: + case EXPRESSION: + case ALWAYS_FALSE: + case ALWAYS_TRUE: + case GEO_NEAR: + case TEXT: + case INTERNAL_2D_POINT_IN_ANNULUS: + case INTERNAL_BUCKET_GEO_WITHIN: + case INTERNAL_EXPR_EQ: + case INTERNAL_EXPR_GT: + case INTERNAL_EXPR_GTE: + case INTERNAL_EXPR_LT: + case INTERNAL_EXPR_LTE: + case INTERNAL_SCHEMA_ALLOWED_PROPERTIES: + case INTERNAL_SCHEMA_BIN_DATA_ENCRYPTED_TYPE: + case INTERNAL_SCHEMA_BIN_DATA_FLE2_ENCRYPTED_TYPE: + case INTERNAL_SCHEMA_BIN_DATA_SUBTYPE: + case INTERNAL_SCHEMA_COND: + case INTERNAL_SCHEMA_EQ: + case INTERNAL_SCHEMA_FMOD: + case INTERNAL_SCHEMA_MAX_ITEMS: + case INTERNAL_SCHEMA_MAX_LENGTH: + case INTERNAL_SCHEMA_MAX_PROPERTIES: + case INTERNAL_SCHEMA_MIN_ITEMS: + case INTERNAL_SCHEMA_MIN_LENGTH: + case INTERNAL_SCHEMA_MIN_PROPERTIES: + case INTERNAL_SCHEMA_ROOT_DOC_EQ: + case INTERNAL_SCHEMA_TYPE: + case INTERNAL_SCHEMA_UNIQUE_ITEMS: + case INTERNAL_SCHEMA_XOR: + return false; + } + MONGO_UNREACHABLE; +} + MatchExpression::ErrorAnnotation::SchemaAnnotations::SchemaAnnotations( const BSONObj& jsonSchemaElement) { auto title = jsonSchemaElement[JSONSchemaParser::kSchemaTitleKeyword]; diff --git a/src/mongo/db/matcher/expression.h b/src/mongo/db/matcher/expression.h index 33ea3ce8790..f2ae57e8d28 100644 --- a/src/mongo/db/matcher/expression.h +++ b/src/mongo/db/matcher/expression.h @@ -41,6 +41,7 @@ #include "mongo/db/matcher/match_details.h" #include "mongo/db/matcher/matchable.h" #include "mongo/db/pipeline/dependencies.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/util/fail_point.h" namespace mongo { @@ -477,19 +478,42 @@ public: void addDependencies(DepsTracker* deps) const; /** - * Serialize the MatchExpression to BSON, appending to 'out'. Output of this method is expected - * to be a valid query object, that, when parsed, produces a logically equivalent - * MatchExpression. If 'includePath' is false then the serialization should assume it's in a - * context where the path has been serialized elsewhere, such as within an $elemMatch value. + * Serialize the MatchExpression to BSON, appending to 'out'. + * + * See 'SerializationOptions' for some options. + * + * Generally, the output of this method is expected to be a valid query object that, when + * parsed, produces a logically equivalent MatchExpression. However, if special options are set, + * this no longer holds. + * + * If 'options.literalPolicy' is set to 'kToDebugTypeString', the result is no longer expected + * to re-parse, since we will put strings in places where strings may not be accpeted + * syntactically (e.g. a number is always expected, as in with the $mod expression). + * + * includePath: + * If set to false, serializes without including the path. For example {a: {$gt: 2}} would + * serialize as just {$gt: 2}. + * + * It is expected that most callers want to set 'includePath' to true to get a correct + * serialization. Internally, we may set this to false if we have a situation where an outer + * expression serializes a path and we don't want to repeat the path in the inner expression. + + * For example in {a: {$elemMatch: {$eq: 2}}} the "a" is serialized by the $elemMatch, and + * should not be serialized by the EQ child. + * The $elemMatch will serialize {a: {$elemMatch: <recurse>}} and the EQ will serialize just + * {$eq: 2} instead of its usual {a: {$eq: 2}}. */ - virtual void serialize(BSONObjBuilder* out, bool includePath = true) const = 0; + virtual void serialize(BSONObjBuilder* out, + const SerializationOptions& options = {}, + bool includePath = true) const = 0; /** - * Convenience method which serializes this MatchExpression to a BSONObj. + * Convenience method which serializes this MatchExpression to a BSONObj. See the override with + * a BSONObjBuilder* argument for details. */ - BSONObj serialize(bool includePath = true) const { + BSONObj serialize(const SerializationOptions& options = {}, bool includePath = true) const { BSONObjBuilder bob; - serialize(&bob, includePath); + serialize(&bob, options, includePath); return bob.obj(); } @@ -547,6 +571,13 @@ public: */ std::string toString() const; + /** + * Returns true if the match type represents a node that + * (1) has a path and + * (2) has children that can operate on that path. + */ + static bool isInternalNodeWithPath(MatchType m); + protected: /** * An ExpressionOptimizerFunc implements tree simplifications for a MatchExpression tree with a diff --git a/src/mongo/db/matcher/expression_algo.cpp b/src/mongo/db/matcher/expression_algo.cpp index 558ef533380..36ae0ecbc92 100644 --- a/src/mongo/db/matcher/expression_algo.cpp +++ b/src/mongo/db/matcher/expression_algo.cpp @@ -913,11 +913,6 @@ bool isPathPrefixOf(StringData first, StringData second) { return second.startsWith(first) && second[first.size()] == '.'; } -bool bidirectionalPathPrefixOf(StringData first, StringData second) { - return first == second || expression::isPathPrefixOf(first, second) || - expression::isPathPrefixOf(second, first); -} - boost::optional<StringMap<std::unique_ptr<MatchExpression>>> splitMatchExpressionForColumns( const MatchExpression* me) { boost::optional<StringMap<std::unique_ptr<MatchExpression>>> out; diff --git a/src/mongo/db/matcher/expression_algo.h b/src/mongo/db/matcher/expression_algo.h index fc42fca7814..8c7c9b71c21 100644 --- a/src/mongo/db/matcher/expression_algo.h +++ b/src/mongo/db/matcher/expression_algo.h @@ -131,12 +131,6 @@ bool isOnlyDependentOn(const MatchExpression& expr, const OrderedPathSet& pathSe bool isPathPrefixOf(StringData first, StringData second); /** - * Returns true if the first path is equal to the second path or if either is a prefix - * of the other. - */ -bool bidirectionalPathPrefixOf(StringData first, StringData second); - -/** * Applies 'func' to each node of 'expr', where the first argument is a pointer to that actual node * (not a copy), and the second argument is the path to that node. Callers should not depend on the * order of the traversal of the nodes. diff --git a/src/mongo/db/matcher/expression_algo_test.cpp b/src/mongo/db/matcher/expression_algo_test.cpp index 3e0922bd9fc..23d3e2f4606 100644 --- a/src/mongo/db/matcher/expression_algo_test.cpp +++ b/src/mongo/db/matcher/expression_algo_test.cpp @@ -36,6 +36,7 @@ #include "mongo/db/matcher/expression.h" #include "mongo/db/matcher/expression_algo.h" #include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/matcher/parsed_match_expression_for_test.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/collation/collator_interface_mock.h" #include "mongo/platform/decimal128.h" @@ -44,30 +45,16 @@ namespace mongo { using std::unique_ptr; -/** - * A MatchExpression does not hold the memory for BSONElements, so use ParsedMatchExpression to - * ensure that the BSONObj outlives the MatchExpression. - */ -class ParsedMatchExpression { -public: - ParsedMatchExpression(const std::string& str, const CollatorInterface* collator = nullptr) - : _obj(fromjson(str)) { - _expCtx = make_intrusive<ExpressionContextForTest>(); - _expCtx->setCollator(CollatorInterface::cloneCollator(collator)); - StatusWithMatchExpression result = MatchExpressionParser::parse(_obj, _expCtx); - ASSERT_OK(result.getStatus()); - _expr = std::move(result.getValue()); - } - const MatchExpression* get() const { - return _expr.get(); +void assertMatchesEqual(const ParsedMatchExpressionForTest& expected, + const std::unique_ptr<MatchExpression>& actual) { + if (expected.get() == nullptr) { + ASSERT(actual == nullptr); + return; } - -private: - const BSONObj _obj; - std::unique_ptr<MatchExpression> _expr; - boost::intrusive_ptr<ExpressionContext> _expCtx; -}; + ASSERT(actual != nullptr); + ASSERT_EQ(expected.get()->toString(), actual.get()->toString()); +} TEST(ExpressionAlgoIsSubsetOf, NullAndOmittedField) { // Verify that the ComparisonMatchExpression constructor prohibits creating a match expression @@ -77,29 +64,29 @@ TEST(ExpressionAlgoIsSubsetOf, NullAndOmittedField) { ASSERT_EQUALS(ErrorCodes::BadValue, MatchExpressionParser::parse(undefined, std::move(expCtx)).getStatus()); - ParsedMatchExpression empty("{}"); - ParsedMatchExpression null("{a: null}"); + ParsedMatchExpressionForTest empty("{}"); + ParsedMatchExpressionForTest null("{a: null}"); ASSERT_TRUE(expression::isSubsetOf(null.get(), empty.get())); ASSERT_FALSE(expression::isSubsetOf(empty.get(), null.get())); - ParsedMatchExpression b1("{b: 1}"); - ParsedMatchExpression aNullB1("{a: null, b: 1}"); + ParsedMatchExpressionForTest b1("{b: 1}"); + ParsedMatchExpressionForTest aNullB1("{a: null, b: 1}"); ASSERT_TRUE(expression::isSubsetOf(aNullB1.get(), b1.get())); ASSERT_FALSE(expression::isSubsetOf(b1.get(), aNullB1.get())); - ParsedMatchExpression a1C3("{a: 1, c: 3}"); - ParsedMatchExpression a1BNullC3("{a: 1, b: null, c: 3}"); + ParsedMatchExpressionForTest a1C3("{a: 1, c: 3}"); + ParsedMatchExpressionForTest a1BNullC3("{a: 1, b: null, c: 3}"); ASSERT_TRUE(expression::isSubsetOf(a1BNullC3.get(), a1C3.get())); ASSERT_FALSE(expression::isSubsetOf(a1C3.get(), a1BNullC3.get())); } TEST(ExpressionAlgoIsSubsetOf, NullAndIn) { - ParsedMatchExpression eqNull("{x: null}"); - ParsedMatchExpression inNull("{x: {$in: [null]}}"); - ParsedMatchExpression inNullOr2("{x: {$in: [null, 2]}}"); + ParsedMatchExpressionForTest eqNull("{x: null}"); + ParsedMatchExpressionForTest inNull("{x: {$in: [null]}}"); + ParsedMatchExpressionForTest inNullOr2("{x: {$in: [null, 2]}}"); ASSERT_TRUE(expression::isSubsetOf(inNull.get(), eqNull.get())); ASSERT_FALSE(expression::isSubsetOf(inNullOr2.get(), eqNull.get())); @@ -109,19 +96,19 @@ TEST(ExpressionAlgoIsSubsetOf, NullAndIn) { } TEST(ExpressionAlgoIsSubsetOf, NullAndExists) { - ParsedMatchExpression null("{x: null}"); - ParsedMatchExpression exists("{x: {$exists: true}}"); + ParsedMatchExpressionForTest null("{x: null}"); + ParsedMatchExpressionForTest exists("{x: {$exists: true}}"); ASSERT_FALSE(expression::isSubsetOf(null.get(), exists.get())); ASSERT_FALSE(expression::isSubsetOf(exists.get(), null.get())); } TEST(ExpressionAlgoIsSubsetOf, Compare_NaN) { - ParsedMatchExpression nan("{x: NaN}"); - ParsedMatchExpression lt("{x: {$lt: 5}}"); - ParsedMatchExpression lte("{x: {$lte: 5}}"); - ParsedMatchExpression gte("{x: {$gte: 5}}"); - ParsedMatchExpression gt("{x: {$gt: 5}}"); - ParsedMatchExpression in("{x: {$in: [5]}}"); + ParsedMatchExpressionForTest nan("{x: NaN}"); + ParsedMatchExpressionForTest lt("{x: {$lt: 5}}"); + ParsedMatchExpressionForTest lte("{x: {$lte: 5}}"); + ParsedMatchExpressionForTest gte("{x: {$gte: 5}}"); + ParsedMatchExpressionForTest gt("{x: {$gt: 5}}"); + ParsedMatchExpressionForTest in("{x: {$in: [5]}}"); ASSERT_TRUE(expression::isSubsetOf(nan.get(), nan.get())); ASSERT_FALSE(expression::isSubsetOf(nan.get(), lt.get())); @@ -135,7 +122,7 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_NaN) { ASSERT_FALSE(expression::isSubsetOf(nan.get(), in.get())); ASSERT_FALSE(expression::isSubsetOf(in.get(), nan.get())); - ParsedMatchExpression decNan("{x : NumberDecimal(\"NaN\") }"); + ParsedMatchExpressionForTest decNan("{x : NumberDecimal(\"NaN\") }"); ASSERT_TRUE(expression::isSubsetOf(decNan.get(), decNan.get())); ASSERT_TRUE(expression::isSubsetOf(nan.get(), decNan.get())); ASSERT_TRUE(expression::isSubsetOf(decNan.get(), nan.get())); @@ -150,9 +137,9 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_NaN) { } TEST(ExpressionAlgoIsSubsetOf, Compare_EQ) { - ParsedMatchExpression a5("{a: 5}"); - ParsedMatchExpression a6("{a: 6}"); - ParsedMatchExpression b5("{b: 5}"); + ParsedMatchExpressionForTest a5("{a: 5}"); + ParsedMatchExpressionForTest a6("{a: 6}"); + ParsedMatchExpressionForTest b5("{b: 5}"); ASSERT_TRUE(expression::isSubsetOf(a5.get(), a5.get())); ASSERT_FALSE(expression::isSubsetOf(a5.get(), a6.get())); @@ -160,10 +147,10 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_EQ) { } TEST(ExpressionAlgoIsSubsetOf, CompareAnd_EQ) { - ParsedMatchExpression a1B2("{a: 1, b: 2}"); - ParsedMatchExpression a1B7("{a: 1, b: 7}"); - ParsedMatchExpression a1("{a: 1}"); - ParsedMatchExpression b2("{b: 2}"); + ParsedMatchExpressionForTest a1B2("{a: 1, b: 2}"); + ParsedMatchExpressionForTest a1B7("{a: 1, b: 7}"); + ParsedMatchExpressionForTest a1("{a: 1}"); + ParsedMatchExpressionForTest b2("{b: 2}"); ASSERT_TRUE(expression::isSubsetOf(a1B2.get(), a1B2.get())); ASSERT_FALSE(expression::isSubsetOf(a1B2.get(), a1B7.get())); @@ -174,74 +161,74 @@ TEST(ExpressionAlgoIsSubsetOf, CompareAnd_EQ) { } TEST(ExpressionAlgoIsSubsetOf, CompareAnd_GT) { - ParsedMatchExpression filter("{a: {$gt: 5}, b: {$gt: 6}}"); - ParsedMatchExpression query("{a: {$gt: 5}, b: {$gt: 6}, c: {$gt: 7}}"); + ParsedMatchExpressionForTest filter("{a: {$gt: 5}, b: {$gt: 6}}"); + ParsedMatchExpressionForTest query("{a: {$gt: 5}, b: {$gt: 6}, c: {$gt: 7}}"); ASSERT_TRUE(expression::isSubsetOf(query.get(), filter.get())); ASSERT_FALSE(expression::isSubsetOf(filter.get(), query.get())); } TEST(ExpressionAlgoIsSubsetOf, CompareAnd_SingleField) { - ParsedMatchExpression filter("{a: {$gt: 5, $lt: 7}}"); - ParsedMatchExpression query("{a: {$gt: 5, $lt: 6}}"); + ParsedMatchExpressionForTest filter("{a: {$gt: 5, $lt: 7}}"); + ParsedMatchExpressionForTest query("{a: {$gt: 5, $lt: 6}}"); ASSERT_TRUE(expression::isSubsetOf(query.get(), filter.get())); ASSERT_FALSE(expression::isSubsetOf(filter.get(), query.get())); } TEST(ExpressionAlgoIsSubsetOf, CompareOr_LT) { - ParsedMatchExpression lt5("{a: {$lt: 5}}"); - ParsedMatchExpression eq2OrEq3("{$or: [{a: 2}, {a: 3}]}"); - ParsedMatchExpression eq4OrEq5("{$or: [{a: 4}, {a: 5}]}"); - ParsedMatchExpression eq4OrEq6("{$or: [{a: 4}, {a: 6}]}"); + ParsedMatchExpressionForTest lt5("{a: {$lt: 5}}"); + ParsedMatchExpressionForTest eq2OrEq3("{$or: [{a: 2}, {a: 3}]}"); + ParsedMatchExpressionForTest eq4OrEq5("{$or: [{a: 4}, {a: 5}]}"); + ParsedMatchExpressionForTest eq4OrEq6("{$or: [{a: 4}, {a: 6}]}"); ASSERT_TRUE(expression::isSubsetOf(eq2OrEq3.get(), lt5.get())); ASSERT_FALSE(expression::isSubsetOf(eq4OrEq5.get(), lt5.get())); ASSERT_FALSE(expression::isSubsetOf(eq4OrEq6.get(), lt5.get())); - ParsedMatchExpression lt4OrLt5("{$or: [{a: {$lt: 4}}, {a: {$lt: 5}}]}"); + ParsedMatchExpressionForTest lt4OrLt5("{$or: [{a: {$lt: 4}}, {a: {$lt: 5}}]}"); ASSERT_TRUE(expression::isSubsetOf(lt4OrLt5.get(), lt5.get())); ASSERT_TRUE(expression::isSubsetOf(lt5.get(), lt4OrLt5.get())); - ParsedMatchExpression lt7OrLt8("{$or: [{a: {$lt: 7}}, {a: {$lt: 8}}]}"); + ParsedMatchExpressionForTest lt7OrLt8("{$or: [{a: {$lt: 7}}, {a: {$lt: 8}}]}"); ASSERT_FALSE(expression::isSubsetOf(lt7OrLt8.get(), lt5.get())); ASSERT_TRUE(expression::isSubsetOf(lt5.get(), lt7OrLt8.get())); } TEST(ExpressionAlgoIsSubsetOf, CompareOr_GTE) { - ParsedMatchExpression gte5("{a: {$gte: 5}}"); - ParsedMatchExpression eq4OrEq6("{$or: [{a: 4}, {a: 6}]}"); - ParsedMatchExpression eq5OrEq6("{$or: [{a: 5}, {a: 6}]}"); - ParsedMatchExpression eq7OrEq8("{$or: [{a: 7}, {a: 8}]}"); + ParsedMatchExpressionForTest gte5("{a: {$gte: 5}}"); + ParsedMatchExpressionForTest eq4OrEq6("{$or: [{a: 4}, {a: 6}]}"); + ParsedMatchExpressionForTest eq5OrEq6("{$or: [{a: 5}, {a: 6}]}"); + ParsedMatchExpressionForTest eq7OrEq8("{$or: [{a: 7}, {a: 8}]}"); ASSERT_FALSE(expression::isSubsetOf(eq4OrEq6.get(), gte5.get())); ASSERT_TRUE(expression::isSubsetOf(eq5OrEq6.get(), gte5.get())); ASSERT_TRUE(expression::isSubsetOf(eq7OrEq8.get(), gte5.get())); - ParsedMatchExpression gte5OrGte6("{$or: [{a: {$gte: 5}}, {a: {$gte: 6}}]}"); + ParsedMatchExpressionForTest gte5OrGte6("{$or: [{a: {$gte: 5}}, {a: {$gte: 6}}]}"); ASSERT_TRUE(expression::isSubsetOf(gte5OrGte6.get(), gte5.get())); ASSERT_TRUE(expression::isSubsetOf(gte5.get(), gte5OrGte6.get())); - ParsedMatchExpression gte3OrGte4("{$or: [{a: {$gte: 3}}, {a: {$gte: 4}}]}"); + ParsedMatchExpressionForTest gte3OrGte4("{$or: [{a: {$gte: 3}}, {a: {$gte: 4}}]}"); ASSERT_FALSE(expression::isSubsetOf(gte3OrGte4.get(), gte5.get())); ASSERT_TRUE(expression::isSubsetOf(gte5.get(), gte3OrGte4.get())); } TEST(ExpressionAlgoIsSubsetOf, DifferentCanonicalTypes) { - ParsedMatchExpression number("{x: {$gt: 1}}"); - ParsedMatchExpression string("{x: {$gt: 'a'}}"); + ParsedMatchExpressionForTest number("{x: {$gt: 1}}"); + ParsedMatchExpressionForTest string("{x: {$gt: 'a'}}"); ASSERT_FALSE(expression::isSubsetOf(number.get(), string.get())); ASSERT_FALSE(expression::isSubsetOf(string.get(), number.get())); } TEST(ExpressionAlgoIsSubsetOf, DifferentNumberTypes) { - ParsedMatchExpression numberDouble("{x: 5.0}"); - ParsedMatchExpression numberInt("{x: NumberInt(5)}"); - ParsedMatchExpression numberLong("{x: NumberLong(5)}"); + ParsedMatchExpressionForTest numberDouble("{x: 5.0}"); + ParsedMatchExpressionForTest numberInt("{x: NumberInt(5)}"); + ParsedMatchExpressionForTest numberLong("{x: NumberLong(5)}"); ASSERT_TRUE(expression::isSubsetOf(numberDouble.get(), numberInt.get())); ASSERT_TRUE(expression::isSubsetOf(numberDouble.get(), numberLong.get())); @@ -252,15 +239,15 @@ TEST(ExpressionAlgoIsSubsetOf, DifferentNumberTypes) { } TEST(ExpressionAlgoIsSubsetOf, PointInUnboundedRange) { - ParsedMatchExpression a4("{a: 4}"); - ParsedMatchExpression a5("{a: 5}"); - ParsedMatchExpression a6("{a: 6}"); - ParsedMatchExpression b5("{b: 5}"); + ParsedMatchExpressionForTest a4("{a: 4}"); + ParsedMatchExpressionForTest a5("{a: 5}"); + ParsedMatchExpressionForTest a6("{a: 6}"); + ParsedMatchExpressionForTest b5("{b: 5}"); - ParsedMatchExpression lt5("{a: {$lt: 5}}"); - ParsedMatchExpression lte5("{a: {$lte: 5}}"); - ParsedMatchExpression gte5("{a: {$gte: 5}}"); - ParsedMatchExpression gt5("{a: {$gt: 5}}"); + ParsedMatchExpressionForTest lt5("{a: {$lt: 5}}"); + ParsedMatchExpressionForTest lte5("{a: {$lte: 5}}"); + ParsedMatchExpressionForTest gte5("{a: {$gte: 5}}"); + ParsedMatchExpressionForTest gt5("{a: {$gt: 5}}"); ASSERT_TRUE(expression::isSubsetOf(a4.get(), lte5.get())); ASSERT_TRUE(expression::isSubsetOf(a5.get(), lte5.get())); @@ -292,26 +279,26 @@ TEST(ExpressionAlgoIsSubsetOf, PointInUnboundedRange) { } TEST(ExpressionAlgoIsSubsetOf, PointInBoundedRange) { - ParsedMatchExpression filter("{a: {$gt: 5, $lt: 10}}"); - ParsedMatchExpression query("{a: 6}"); + ParsedMatchExpressionForTest filter("{a: {$gt: 5, $lt: 10}}"); + ParsedMatchExpressionForTest query("{a: 6}"); ASSERT_TRUE(expression::isSubsetOf(query.get(), filter.get())); ASSERT_FALSE(expression::isSubsetOf(filter.get(), query.get())); } TEST(ExpressionAlgoIsSubsetOf, PointInBoundedRange_FakeAnd) { - ParsedMatchExpression filter("{a: {$gt: 5, $lt: 10}}"); - ParsedMatchExpression query("{$and: [{a: 6}, {a: 6}]}"); + ParsedMatchExpressionForTest filter("{a: {$gt: 5, $lt: 10}}"); + ParsedMatchExpressionForTest query("{$and: [{a: 6}, {a: 6}]}"); ASSERT_TRUE(expression::isSubsetOf(query.get(), filter.get())); ASSERT_FALSE(expression::isSubsetOf(filter.get(), query.get())); } TEST(ExpressionAlgoIsSubsetOf, MultiplePointsInBoundedRange) { - ParsedMatchExpression filter("{a: {$gt: 5, $lt: 10}}"); - ParsedMatchExpression queryAllInside("{a: {$in: [6, 7, 8]}}"); - ParsedMatchExpression queryStraddleLower("{a: {$in: [4.9, 5.1]}}"); - ParsedMatchExpression queryStraddleUpper("{a: {$in: [9.9, 10.1]}}"); + ParsedMatchExpressionForTest filter("{a: {$gt: 5, $lt: 10}}"); + ParsedMatchExpressionForTest queryAllInside("{a: {$in: [6, 7, 8]}}"); + ParsedMatchExpressionForTest queryStraddleLower("{a: {$in: [4.9, 5.1]}}"); + ParsedMatchExpressionForTest queryStraddleUpper("{a: {$in: [9.9, 10.1]}}"); ASSERT_TRUE(expression::isSubsetOf(queryAllInside.get(), filter.get())); ASSERT_FALSE(expression::isSubsetOf(queryStraddleLower.get(), filter.get())); @@ -319,18 +306,18 @@ TEST(ExpressionAlgoIsSubsetOf, MultiplePointsInBoundedRange) { } TEST(ExpressionAlgoIsSubsetOf, PointInCompoundRange) { - ParsedMatchExpression filter("{a: {$gt: 5}, b: {$gt: 6}, c: {$gt: 7}}"); - ParsedMatchExpression query("{a: 10, b: 10, c: 10}"); + ParsedMatchExpressionForTest filter("{a: {$gt: 5}, b: {$gt: 6}, c: {$gt: 7}}"); + ParsedMatchExpressionForTest query("{a: 10, b: 10, c: 10}"); ASSERT_TRUE(expression::isSubsetOf(query.get(), filter.get())); ASSERT_FALSE(expression::isSubsetOf(filter.get(), query.get())); } TEST(ExpressionAlgoIsSubsetOf, Compare_LT_LTE) { - ParsedMatchExpression lte4("{x: {$lte: 4}}"); - ParsedMatchExpression lt5("{x: {$lt: 5}}"); - ParsedMatchExpression lte5("{x: {$lte: 5}}"); - ParsedMatchExpression lt6("{x: {$lt: 6}}"); + ParsedMatchExpressionForTest lte4("{x: {$lte: 4}}"); + ParsedMatchExpressionForTest lt5("{x: {$lt: 5}}"); + ParsedMatchExpressionForTest lte5("{x: {$lte: 5}}"); + ParsedMatchExpressionForTest lt6("{x: {$lt: 6}}"); ASSERT_TRUE(expression::isSubsetOf(lte4.get(), lte5.get())); ASSERT_TRUE(expression::isSubsetOf(lt5.get(), lte5.get())); @@ -344,10 +331,10 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_LT_LTE) { } TEST(ExpressionAlgoIsSubsetOf, Compare_GT_GTE) { - ParsedMatchExpression gte6("{x: {$gte: 6}}"); - ParsedMatchExpression gt5("{x: {$gt: 5}}"); - ParsedMatchExpression gte5("{x: {$gte: 5}}"); - ParsedMatchExpression gt4("{x: {$gt: 4}}"); + ParsedMatchExpressionForTest gte6("{x: {$gte: 6}}"); + ParsedMatchExpressionForTest gt5("{x: {$gt: 5}}"); + ParsedMatchExpressionForTest gte5("{x: {$gte: 5}}"); + ParsedMatchExpressionForTest gt4("{x: {$gt: 4}}"); ASSERT_TRUE(expression::isSubsetOf(gte6.get(), gte5.get())); ASSERT_TRUE(expression::isSubsetOf(gt5.get(), gte5.get())); @@ -361,18 +348,19 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_GT_GTE) { } TEST(ExpressionAlgoIsSubsetOf, BoundedRangeInUnboundedRange) { - ParsedMatchExpression filter("{a: {$gt: 1}}"); - ParsedMatchExpression query("{a: {$gt: 5, $lt: 10}}"); + ParsedMatchExpressionForTest filter("{a: {$gt: 1}}"); + ParsedMatchExpressionForTest query("{a: {$gt: 5, $lt: 10}}"); ASSERT_TRUE(expression::isSubsetOf(query.get(), filter.get())); ASSERT_FALSE(expression::isSubsetOf(filter.get(), query.get())); } TEST(ExpressionAlgoIsSubsetOf, MultipleRangesInUnboundedRange) { - ParsedMatchExpression filter("{a: {$gt: 1}}"); - ParsedMatchExpression negative("{$or: [{a: {$gt: 5, $lt: 10}}, {a: {$lt: 0}}]}"); - ParsedMatchExpression unbounded("{$or: [{a: {$gt: 5, $lt: 10}}, {a: {$gt: 15}}]}"); - ParsedMatchExpression bounded("{$or: [{a: {$gt: 5, $lt: 10}}, {a: {$gt: 20, $lt: 30}}]}"); + ParsedMatchExpressionForTest filter("{a: {$gt: 1}}"); + ParsedMatchExpressionForTest negative("{$or: [{a: {$gt: 5, $lt: 10}}, {a: {$lt: 0}}]}"); + ParsedMatchExpressionForTest unbounded("{$or: [{a: {$gt: 5, $lt: 10}}, {a: {$gt: 15}}]}"); + ParsedMatchExpressionForTest bounded( + "{$or: [{a: {$gt: 5, $lt: 10}}, {a: {$gt: 20, $lt: 30}}]}"); ASSERT_FALSE(expression::isSubsetOf(negative.get(), filter.get())); ASSERT_TRUE(expression::isSubsetOf(unbounded.get(), filter.get())); @@ -380,10 +368,10 @@ TEST(ExpressionAlgoIsSubsetOf, MultipleRangesInUnboundedRange) { } TEST(ExpressionAlgoIsSubsetOf, MultipleFields) { - ParsedMatchExpression filter("{a: {$gt: 5}, b: {$lt: 10}}"); - ParsedMatchExpression onlyA("{$or: [{a: 6, b: {$lt: 4}}, {a: {$gt: 11}}]}"); - ParsedMatchExpression onlyB("{$or: [{b: {$lt: 4}}, {a: {$gt: 11}, b: 9}]}"); - ParsedMatchExpression both("{$or: [{a: 6, b: {$lt: 4}}, {a: {$gt: 11}, b: 9}]}"); + ParsedMatchExpressionForTest filter("{a: {$gt: 5}, b: {$lt: 10}}"); + ParsedMatchExpressionForTest onlyA("{$or: [{a: 6, b: {$lt: 4}}, {a: {$gt: 11}}]}"); + ParsedMatchExpressionForTest onlyB("{$or: [{b: {$lt: 4}}, {a: {$gt: 11}, b: 9}]}"); + ParsedMatchExpressionForTest both("{$or: [{a: 6, b: {$lt: 4}}, {a: {$gt: 11}, b: 9}]}"); ASSERT_FALSE(expression::isSubsetOf(onlyA.get(), filter.get())); ASSERT_FALSE(expression::isSubsetOf(onlyB.get(), filter.get())); @@ -391,18 +379,18 @@ TEST(ExpressionAlgoIsSubsetOf, MultipleFields) { } TEST(ExpressionAlgoIsSubsetOf, Compare_LT_In) { - ParsedMatchExpression lt("{a: {$lt: 5}}"); + ParsedMatchExpressionForTest lt("{a: {$lt: 5}}"); - ParsedMatchExpression inLt("{a: {$in: [4.9]}}"); - ParsedMatchExpression inEq("{a: {$in: [5]}}"); - ParsedMatchExpression inGt("{a: {$in: [5.1]}}"); - ParsedMatchExpression inNull("{a: {$in: [null]}}"); + ParsedMatchExpressionForTest inLt("{a: {$in: [4.9]}}"); + ParsedMatchExpressionForTest inEq("{a: {$in: [5]}}"); + ParsedMatchExpressionForTest inGt("{a: {$in: [5.1]}}"); + ParsedMatchExpressionForTest inNull("{a: {$in: [null]}}"); - ParsedMatchExpression inAllEq("{a: {$in: [5, 5.0]}}"); - ParsedMatchExpression inAllLte("{a: {$in: [4.9, 5]}}"); - ParsedMatchExpression inAllLt("{a: {$in: [2, 3, 4]}}"); - ParsedMatchExpression inStraddle("{a: {$in: [4, 6]}}"); - ParsedMatchExpression inLtAndNull("{a: {$in: [1, null]}}"); + ParsedMatchExpressionForTest inAllEq("{a: {$in: [5, 5.0]}}"); + ParsedMatchExpressionForTest inAllLte("{a: {$in: [4.9, 5]}}"); + ParsedMatchExpressionForTest inAllLt("{a: {$in: [2, 3, 4]}}"); + ParsedMatchExpressionForTest inStraddle("{a: {$in: [4, 6]}}"); + ParsedMatchExpressionForTest inLtAndNull("{a: {$in: [1, null]}}"); ASSERT_TRUE(expression::isSubsetOf(inLt.get(), lt.get())); ASSERT_FALSE(expression::isSubsetOf(inEq.get(), lt.get())); @@ -419,18 +407,18 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_LT_In) { } TEST(ExpressionAlgoIsSubsetOf, Compare_LTE_In) { - ParsedMatchExpression lte("{a: {$lte: 5}}"); + ParsedMatchExpressionForTest lte("{a: {$lte: 5}}"); - ParsedMatchExpression inLt("{a: {$in: [4.9]}}"); - ParsedMatchExpression inEq("{a: {$in: [5]}}"); - ParsedMatchExpression inGt("{a: {$in: [5.1]}}"); - ParsedMatchExpression inNull("{a: {$in: [null]}}"); + ParsedMatchExpressionForTest inLt("{a: {$in: [4.9]}}"); + ParsedMatchExpressionForTest inEq("{a: {$in: [5]}}"); + ParsedMatchExpressionForTest inGt("{a: {$in: [5.1]}}"); + ParsedMatchExpressionForTest inNull("{a: {$in: [null]}}"); - ParsedMatchExpression inAllEq("{a: {$in: [5, 5.0]}}"); - ParsedMatchExpression inAllLte("{a: {$in: [4.9, 5]}}"); - ParsedMatchExpression inAllLt("{a: {$in: [2, 3, 4]}}"); - ParsedMatchExpression inStraddle("{a: {$in: [4, 6]}}"); - ParsedMatchExpression inLtAndNull("{a: {$in: [1, null]}}"); + ParsedMatchExpressionForTest inAllEq("{a: {$in: [5, 5.0]}}"); + ParsedMatchExpressionForTest inAllLte("{a: {$in: [4.9, 5]}}"); + ParsedMatchExpressionForTest inAllLt("{a: {$in: [2, 3, 4]}}"); + ParsedMatchExpressionForTest inStraddle("{a: {$in: [4, 6]}}"); + ParsedMatchExpressionForTest inLtAndNull("{a: {$in: [1, null]}}"); ASSERT_TRUE(expression::isSubsetOf(inLt.get(), lte.get())); ASSERT_TRUE(expression::isSubsetOf(inEq.get(), lte.get())); @@ -447,16 +435,16 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_LTE_In) { } TEST(ExpressionAlgoIsSubsetOf, Compare_EQ_In) { - ParsedMatchExpression eq("{a: 5}"); + ParsedMatchExpressionForTest eq("{a: 5}"); - ParsedMatchExpression inLt("{a: {$in: [4.9]}}"); - ParsedMatchExpression inEq("{a: {$in: [5]}}"); - ParsedMatchExpression inGt("{a: {$in: [5.1]}}"); - ParsedMatchExpression inNull("{a: {$in: [null]}}"); + ParsedMatchExpressionForTest inLt("{a: {$in: [4.9]}}"); + ParsedMatchExpressionForTest inEq("{a: {$in: [5]}}"); + ParsedMatchExpressionForTest inGt("{a: {$in: [5.1]}}"); + ParsedMatchExpressionForTest inNull("{a: {$in: [null]}}"); - ParsedMatchExpression inAllEq("{a: {$in: [5, 5.0]}}"); - ParsedMatchExpression inStraddle("{a: {$in: [4, 6]}}"); - ParsedMatchExpression inEqAndNull("{a: {$in: [5, null]}}"); + ParsedMatchExpressionForTest inAllEq("{a: {$in: [5, 5.0]}}"); + ParsedMatchExpressionForTest inStraddle("{a: {$in: [4, 6]}}"); + ParsedMatchExpressionForTest inEqAndNull("{a: {$in: [5, null]}}"); ASSERT_FALSE(expression::isSubsetOf(inLt.get(), eq.get())); ASSERT_TRUE(expression::isSubsetOf(inEq.get(), eq.get())); @@ -471,18 +459,18 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_EQ_In) { } TEST(ExpressionAlgoIsSubsetOf, Compare_GT_In) { - ParsedMatchExpression gt("{a: {$gt: 5}}"); + ParsedMatchExpressionForTest gt("{a: {$gt: 5}}"); - ParsedMatchExpression inLt("{a: {$in: [4.9]}}"); - ParsedMatchExpression inEq("{a: {$in: [5]}}"); - ParsedMatchExpression inGt("{a: {$in: [5.1]}}"); - ParsedMatchExpression inNull("{a: {$in: [null]}}"); + ParsedMatchExpressionForTest inLt("{a: {$in: [4.9]}}"); + ParsedMatchExpressionForTest inEq("{a: {$in: [5]}}"); + ParsedMatchExpressionForTest inGt("{a: {$in: [5.1]}}"); + ParsedMatchExpressionForTest inNull("{a: {$in: [null]}}"); - ParsedMatchExpression inAllEq("{a: {$in: [5, 5.0]}}"); - ParsedMatchExpression inAllGte("{a: {$in: [5, 5.1]}}"); - ParsedMatchExpression inAllGt("{a: {$in: [6, 7, 8]}}"); - ParsedMatchExpression inStraddle("{a: {$in: [4, 6]}}"); - ParsedMatchExpression inGtAndNull("{a: {$in: [9, null]}}"); + ParsedMatchExpressionForTest inAllEq("{a: {$in: [5, 5.0]}}"); + ParsedMatchExpressionForTest inAllGte("{a: {$in: [5, 5.1]}}"); + ParsedMatchExpressionForTest inAllGt("{a: {$in: [6, 7, 8]}}"); + ParsedMatchExpressionForTest inStraddle("{a: {$in: [4, 6]}}"); + ParsedMatchExpressionForTest inGtAndNull("{a: {$in: [9, null]}}"); ASSERT_FALSE(expression::isSubsetOf(inLt.get(), gt.get())); ASSERT_FALSE(expression::isSubsetOf(inEq.get(), gt.get())); @@ -499,18 +487,18 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_GT_In) { } TEST(ExpressionAlgoIsSubsetOf, Compare_GTE_In) { - ParsedMatchExpression gte("{a: {$gte: 5}}"); + ParsedMatchExpressionForTest gte("{a: {$gte: 5}}"); - ParsedMatchExpression inLt("{a: {$in: [4.9]}}"); - ParsedMatchExpression inEq("{a: {$in: [5]}}"); - ParsedMatchExpression inGt("{a: {$in: [5.1]}}"); - ParsedMatchExpression inNull("{a: {$in: [null]}}"); + ParsedMatchExpressionForTest inLt("{a: {$in: [4.9]}}"); + ParsedMatchExpressionForTest inEq("{a: {$in: [5]}}"); + ParsedMatchExpressionForTest inGt("{a: {$in: [5.1]}}"); + ParsedMatchExpressionForTest inNull("{a: {$in: [null]}}"); - ParsedMatchExpression inAllEq("{a: {$in: [5, 5.0]}}"); - ParsedMatchExpression inAllGte("{a: {$in: [5, 5.1]}}"); - ParsedMatchExpression inAllGt("{a: {$in: [6, 7, 8]}}"); - ParsedMatchExpression inStraddle("{a: {$in: [4, 6]}}"); - ParsedMatchExpression inGtAndNull("{a: {$in: [9, null]}}"); + ParsedMatchExpressionForTest inAllEq("{a: {$in: [5, 5.0]}}"); + ParsedMatchExpressionForTest inAllGte("{a: {$in: [5, 5.1]}}"); + ParsedMatchExpressionForTest inAllGt("{a: {$in: [6, 7, 8]}}"); + ParsedMatchExpressionForTest inStraddle("{a: {$in: [4, 6]}}"); + ParsedMatchExpressionForTest inGtAndNull("{a: {$in: [9, null]}}"); ASSERT_FALSE(expression::isSubsetOf(inLt.get(), gte.get())); ASSERT_TRUE(expression::isSubsetOf(inEq.get(), gte.get())); @@ -527,12 +515,12 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_GTE_In) { } TEST(ExpressionAlgoIsSubsetOf, RegexAndIn) { - ParsedMatchExpression eq1("{x: 1}"); - ParsedMatchExpression eqA("{x: 'a'}"); - ParsedMatchExpression inRegexA("{x: {$in: [/a/]}}"); - ParsedMatchExpression inRegexAbc("{x: {$in: [/abc/]}}"); - ParsedMatchExpression inRegexAOrEq1("{x: {$in: [/a/, 1]}}"); - ParsedMatchExpression inRegexAOrNull("{x: {$in: [/a/, null]}}"); + ParsedMatchExpressionForTest eq1("{x: 1}"); + ParsedMatchExpressionForTest eqA("{x: 'a'}"); + ParsedMatchExpressionForTest inRegexA("{x: {$in: [/a/]}}"); + ParsedMatchExpressionForTest inRegexAbc("{x: {$in: [/abc/]}}"); + ParsedMatchExpressionForTest inRegexAOrEq1("{x: {$in: [/a/, 1]}}"); + ParsedMatchExpressionForTest inRegexAOrNull("{x: {$in: [/a/, null]}}"); ASSERT_FALSE(expression::isSubsetOf(inRegexAOrEq1.get(), eq1.get())); ASSERT_FALSE(expression::isSubsetOf(inRegexA.get(), eqA.get())); @@ -544,10 +532,10 @@ TEST(ExpressionAlgoIsSubsetOf, RegexAndIn) { } TEST(ExpressionAlgoIsSubsetOf, Exists) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression bExists("{b: {$exists: true}}"); - ParsedMatchExpression aExistsBExists("{a: {$exists: true}, b: {$exists: true}}"); - ParsedMatchExpression aExistsBExistsC5("{a: {$exists: true}, b: {$exists: true}, c: 5}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest bExists("{b: {$exists: true}}"); + ParsedMatchExpressionForTest aExistsBExists("{a: {$exists: true}, b: {$exists: true}}"); + ParsedMatchExpressionForTest aExistsBExistsC5("{a: {$exists: true}, b: {$exists: true}, c: 5}"); ASSERT_TRUE(expression::isSubsetOf(aExists.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(aExists.get(), bExists.get())); @@ -562,10 +550,10 @@ TEST(ExpressionAlgoIsSubsetOf, Exists) { } TEST(ExpressionAlgoIsSubsetOf, Compare_Exists) { - ParsedMatchExpression exists("{a: {$exists: true}}"); - ParsedMatchExpression eq("{a: 1}"); - ParsedMatchExpression gt("{a: {$gt: 4}}"); - ParsedMatchExpression lte("{a: {$lte: 7}}"); + ParsedMatchExpressionForTest exists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest eq("{a: 1}"); + ParsedMatchExpressionForTest gt("{a: {$gt: 4}}"); + ParsedMatchExpressionForTest lte("{a: {$lte: 7}}"); ASSERT_TRUE(expression::isSubsetOf(eq.get(), exists.get())); ASSERT_TRUE(expression::isSubsetOf(gt.get(), exists.get())); @@ -577,9 +565,9 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_Exists) { } TEST(ExpressionAlgoIsSubsetOf, Type) { - ParsedMatchExpression aType1("{a: {$type: 1}}"); - ParsedMatchExpression aType2("{a: {$type: 2}}"); - ParsedMatchExpression bType2("{b: {$type: 2}}"); + ParsedMatchExpressionForTest aType1("{a: {$type: 1}}"); + ParsedMatchExpressionForTest aType2("{a: {$type: 2}}"); + ParsedMatchExpressionForTest bType2("{b: {$type: 2}}"); ASSERT_FALSE(expression::isSubsetOf(aType1.get(), aType2.get())); ASSERT_FALSE(expression::isSubsetOf(aType2.get(), aType1.get())); @@ -589,9 +577,9 @@ TEST(ExpressionAlgoIsSubsetOf, Type) { } TEST(ExpressionAlgoIsSubsetOf, TypeAndExists) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aType2("{a: {$type: 2}}"); - ParsedMatchExpression bType2("{b: {$type: 2}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aType2("{a: {$type: 2}}"); + ParsedMatchExpressionForTest bType2("{b: {$type: 2}}"); ASSERT_TRUE(expression::isSubsetOf(aType2.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(aExists.get(), aType2.get())); @@ -599,10 +587,10 @@ TEST(ExpressionAlgoIsSubsetOf, TypeAndExists) { } TEST(ExpressionAlgoIsSubsetOf, AllAndExists) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aAll("{a: {$all: ['x', 'y', 'z']}}"); - ParsedMatchExpression bAll("{b: {$all: ['x', 'y', 'z']}}"); - ParsedMatchExpression aAllWithNull("{a: {$all: ['x', null, 'z']}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aAll("{a: {$all: ['x', 'y', 'z']}}"); + ParsedMatchExpressionForTest bAll("{b: {$all: ['x', 'y', 'z']}}"); + ParsedMatchExpressionForTest aAllWithNull("{a: {$all: ['x', null, 'z']}}"); ASSERT_TRUE(expression::isSubsetOf(aAll.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(bAll.get(), aExists.get())); @@ -610,10 +598,10 @@ TEST(ExpressionAlgoIsSubsetOf, AllAndExists) { } TEST(ExpressionAlgoIsSubsetOf, ElemMatchAndExists_Value) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aElemMatch("{a: {$elemMatch: {$gt: 5, $lte: 10}}}"); - ParsedMatchExpression bElemMatch("{b: {$elemMatch: {$gt: 5, $lte: 10}}}"); - ParsedMatchExpression aElemMatchNull("{a: {$elemMatch: {$eq: null}}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aElemMatch("{a: {$elemMatch: {$gt: 5, $lte: 10}}}"); + ParsedMatchExpressionForTest bElemMatch("{b: {$elemMatch: {$gt: 5, $lte: 10}}}"); + ParsedMatchExpressionForTest aElemMatchNull("{a: {$elemMatch: {$eq: null}}}"); ASSERT_TRUE(expression::isSubsetOf(aElemMatch.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(aExists.get(), aElemMatch.get())); @@ -622,10 +610,10 @@ TEST(ExpressionAlgoIsSubsetOf, ElemMatchAndExists_Value) { } TEST(ExpressionAlgoIsSubsetOf, ElemMatchAndExists_Object) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aElemMatch("{a: {$elemMatch: {x: {$gt: 5}, y: {$lte: 10}}}}"); - ParsedMatchExpression bElemMatch("{b: {$elemMatch: {x: {$gt: 5}, y: {$lte: 10}}}}"); - ParsedMatchExpression aElemMatchNull("{a: {$elemMatch: {x: null, y: null}}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aElemMatch("{a: {$elemMatch: {x: {$gt: 5}, y: {$lte: 10}}}}"); + ParsedMatchExpressionForTest bElemMatch("{b: {$elemMatch: {x: {$gt: 5}, y: {$lte: 10}}}}"); + ParsedMatchExpressionForTest aElemMatchNull("{a: {$elemMatch: {x: null, y: null}}}"); ASSERT_TRUE(expression::isSubsetOf(aElemMatch.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(aExists.get(), aElemMatch.get())); @@ -634,11 +622,11 @@ TEST(ExpressionAlgoIsSubsetOf, ElemMatchAndExists_Object) { } TEST(ExpressionAlgoIsSubsetOf, SizeAndExists) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aSize0("{a: {$size: 0}}"); - ParsedMatchExpression aSize1("{a: {$size: 1}}"); - ParsedMatchExpression aSize3("{a: {$size: 3}}"); - ParsedMatchExpression bSize3("{b: {$size: 3}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aSize0("{a: {$size: 0}}"); + ParsedMatchExpressionForTest aSize1("{a: {$size: 1}}"); + ParsedMatchExpressionForTest aSize3("{a: {$size: 3}}"); + ParsedMatchExpressionForTest bSize3("{b: {$size: 3}}"); ASSERT_TRUE(expression::isSubsetOf(aSize0.get(), aExists.get())); ASSERT_TRUE(expression::isSubsetOf(aSize1.get(), aExists.get())); @@ -648,28 +636,28 @@ TEST(ExpressionAlgoIsSubsetOf, SizeAndExists) { } TEST(ExpressionAlgoIsSubsetOf, ModAndExists) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aMod5("{a: {$mod: [5, 0]}}"); - ParsedMatchExpression bMod5("{b: {$mod: [5, 0]}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aMod5("{a: {$mod: [5, 0]}}"); + ParsedMatchExpressionForTest bMod5("{b: {$mod: [5, 0]}}"); ASSERT_TRUE(expression::isSubsetOf(aMod5.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(bMod5.get(), aExists.get())); } TEST(ExpressionAlgoIsSubsetOf, RegexAndExists) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aRegex("{a: {$regex: 'pattern'}}"); - ParsedMatchExpression bRegex("{b: {$regex: 'pattern'}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aRegex("{a: {$regex: 'pattern'}}"); + ParsedMatchExpressionForTest bRegex("{b: {$regex: 'pattern'}}"); ASSERT_TRUE(expression::isSubsetOf(aRegex.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(bRegex.get(), aExists.get())); } TEST(ExpressionAlgoIsSubsetOf, InAndExists) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aIn("{a: {$in: [1, 2, 3]}}"); - ParsedMatchExpression bIn("{b: {$in: [1, 2, 3]}}"); - ParsedMatchExpression aInWithNull("{a: {$in: [1, null, 3]}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aIn("{a: {$in: [1, 2, 3]}}"); + ParsedMatchExpressionForTest bIn("{b: {$in: [1, 2, 3]}}"); + ParsedMatchExpressionForTest aInWithNull("{a: {$in: [1, null, 3]}}"); ASSERT_TRUE(expression::isSubsetOf(aIn.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(bIn.get(), aExists.get())); @@ -681,10 +669,10 @@ TEST(ExpressionAlgoIsSubsetOf, InAndExists) { } TEST(ExpressionAlgoIsSubsetOf, NinAndExists) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aNin("{a: {$nin: [1, 2, 3]}}"); - ParsedMatchExpression bNin("{b: {$nin: [1, 2, 3]}}"); - ParsedMatchExpression aNinWithNull("{a: {$nin: [1, null, 3]}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aNin("{a: {$nin: [1, 2, 3]}}"); + ParsedMatchExpressionForTest bNin("{b: {$nin: [1, 2, 3]}}"); + ParsedMatchExpressionForTest aNinWithNull("{a: {$nin: [1, null, 3]}}"); ASSERT_FALSE(expression::isSubsetOf(aNin.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(bNin.get(), aExists.get())); @@ -692,10 +680,10 @@ TEST(ExpressionAlgoIsSubsetOf, NinAndExists) { } TEST(ExpressionAlgoIsSubsetOf, Compare_Exists_NE) { - ParsedMatchExpression aExists("{a: {$exists: true}}"); - ParsedMatchExpression aNotEqual1("{a: {$ne: 1}}"); - ParsedMatchExpression bNotEqual1("{b: {$ne: 1}}"); - ParsedMatchExpression aNotEqualNull("{a: {$ne: null}}"); + ParsedMatchExpressionForTest aExists("{a: {$exists: true}}"); + ParsedMatchExpressionForTest aNotEqual1("{a: {$ne: 1}}"); + ParsedMatchExpressionForTest bNotEqual1("{b: {$ne: 1}}"); + ParsedMatchExpressionForTest aNotEqualNull("{a: {$ne: null}}"); ASSERT_FALSE(expression::isSubsetOf(aNotEqual1.get(), aExists.get())); ASSERT_FALSE(expression::isSubsetOf(bNotEqual1.get(), aExists.get())); @@ -704,13 +692,13 @@ TEST(ExpressionAlgoIsSubsetOf, Compare_Exists_NE) { TEST(ExpressionAlgoIsSubsetOf, CollationAwareStringComparison) { CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString); - ParsedMatchExpression lhs("{a: {$gt: 'abc'}}", &collator); - ParsedMatchExpression rhs("{a: {$gt: 'cba'}}", &collator); + ParsedMatchExpressionForTest lhs("{a: {$gt: 'abc'}}", &collator); + ParsedMatchExpressionForTest rhs("{a: {$gt: 'cba'}}", &collator); ASSERT_TRUE(expression::isSubsetOf(lhs.get(), rhs.get())); - ParsedMatchExpression lhsLT("{a: {$lt: 'abc'}}", &collator); - ParsedMatchExpression rhsLT("{a: {$lt: 'cba'}}", &collator); + ParsedMatchExpressionForTest lhsLT("{a: {$lt: 'abc'}}", &collator); + ParsedMatchExpressionForTest rhsLT("{a: {$lt: 'cba'}}", &collator); ASSERT_FALSE(expression::isSubsetOf(lhsLT.get(), rhsLT.get())); } @@ -718,27 +706,27 @@ TEST(ExpressionAlgoIsSubsetOf, CollationAwareStringComparison) { TEST(ExpressionAlgoIsSubsetOf, NonMatchingCollationsStringComparison) { CollatorInterfaceMock collatorAlwaysEqual(CollatorInterfaceMock::MockType::kAlwaysEqual); CollatorInterfaceMock collatorReverseString(CollatorInterfaceMock::MockType::kReverseString); - ParsedMatchExpression lhs("{a: {$gt: 'abc'}}", &collatorAlwaysEqual); - ParsedMatchExpression rhs("{a: {$gt: 'cba'}}", &collatorReverseString); + ParsedMatchExpressionForTest lhs("{a: {$gt: 'abc'}}", &collatorAlwaysEqual); + ParsedMatchExpressionForTest rhs("{a: {$gt: 'cba'}}", &collatorReverseString); ASSERT_FALSE(expression::isSubsetOf(lhs.get(), rhs.get())); - ParsedMatchExpression lhsLT("{a: {$lt: 'abc'}}", &collatorAlwaysEqual); - ParsedMatchExpression rhsLT("{a: {$lt: 'cba'}}", &collatorReverseString); + ParsedMatchExpressionForTest lhsLT("{a: {$lt: 'abc'}}", &collatorAlwaysEqual); + ParsedMatchExpressionForTest rhsLT("{a: {$lt: 'cba'}}", &collatorReverseString); ASSERT_FALSE(expression::isSubsetOf(lhsLT.get(), rhsLT.get())); } TEST(ExpressionAlgoIsSubsetOf, CollationAwareStringComparisonIn) { CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString); - ParsedMatchExpression lhsAllGTcba("{a: {$in: ['abc', 'cbc']}}", &collator); - ParsedMatchExpression lhsSomeGTcba("{a: {$in: ['abc', 'aba']}}", &collator); - ParsedMatchExpression rhs("{a: {$gt: 'cba'}}", &collator); + ParsedMatchExpressionForTest lhsAllGTcba("{a: {$in: ['abc', 'cbc']}}", &collator); + ParsedMatchExpressionForTest lhsSomeGTcba("{a: {$in: ['abc', 'aba']}}", &collator); + ParsedMatchExpressionForTest rhs("{a: {$gt: 'cba'}}", &collator); ASSERT_TRUE(expression::isSubsetOf(lhsAllGTcba.get(), rhs.get())); ASSERT_FALSE(expression::isSubsetOf(lhsSomeGTcba.get(), rhs.get())); - ParsedMatchExpression rhsLT("{a: {$lt: 'cba'}}", &collator); + ParsedMatchExpressionForTest rhsLT("{a: {$lt: 'cba'}}", &collator); ASSERT_FALSE(expression::isSubsetOf(lhsAllGTcba.get(), rhsLT.get())); ASSERT_FALSE(expression::isSubsetOf(lhsSomeGTcba.get(), rhsLT.get())); @@ -748,8 +736,8 @@ TEST(ExpressionAlgoIsSubsetOf, CollationAwareStringComparisonIn) { TEST(ExpressionAlgoIsSubsetOf, NonMatchingCollationsNoStringComparisonLHS) { CollatorInterfaceMock collatorAlwaysEqual(CollatorInterfaceMock::MockType::kAlwaysEqual); CollatorInterfaceMock collatorReverseString(CollatorInterfaceMock::MockType::kReverseString); - ParsedMatchExpression lhs("{a: {b: 1}}", &collatorAlwaysEqual); - ParsedMatchExpression rhs("{a: {$lt: {b: 'abc'}}}", &collatorReverseString); + ParsedMatchExpressionForTest lhs("{a: {b: 1}}", &collatorAlwaysEqual); + ParsedMatchExpressionForTest rhs("{a: {$lt: {b: 'abc'}}}", &collatorReverseString); ASSERT_FALSE(expression::isSubsetOf(lhs.get(), rhs.get())); } @@ -757,72 +745,75 @@ TEST(ExpressionAlgoIsSubsetOf, NonMatchingCollationsNoStringComparisonLHS) { TEST(ExpressionAlgoIsSubsetOf, NonMatchingCollationsNoStringComparison) { CollatorInterfaceMock collatorAlwaysEqual(CollatorInterfaceMock::MockType::kAlwaysEqual); CollatorInterfaceMock collatorReverseString(CollatorInterfaceMock::MockType::kReverseString); - ParsedMatchExpression lhs("{a: 1}", &collatorAlwaysEqual); - ParsedMatchExpression rhs("{a: {$gt: 0}}", &collatorReverseString); + ParsedMatchExpressionForTest lhs("{a: 1}", &collatorAlwaysEqual); + ParsedMatchExpressionForTest rhs("{a: {$gt: 0}}", &collatorReverseString); ASSERT_TRUE(expression::isSubsetOf(lhs.get(), rhs.get())); } TEST(ExpressionAlgoIsSubsetOf, InternalExprEqIsSubsetOfNothing) { - ParsedMatchExpression exprEq("{a: {$_internalExprEq: 0}}"); - ParsedMatchExpression regularEq("{a: {$eq: 0}}"); + ParsedMatchExpressionForTest exprEq("{a: {$_internalExprEq: 0}}"); + ParsedMatchExpressionForTest regularEq("{a: {$eq: 0}}"); { - ParsedMatchExpression rhs("{a: {$gte: 0}}"); + ParsedMatchExpressionForTest rhs("{a: {$gte: 0}}"); ASSERT_FALSE(expression::isSubsetOf(exprEq.get(), rhs.get())); ASSERT_TRUE(expression::isSubsetOf(regularEq.get(), rhs.get())); } { - ParsedMatchExpression rhs("{a: {$lte: 0}}"); + ParsedMatchExpressionForTest rhs("{a: {$lte: 0}}"); ASSERT_FALSE(expression::isSubsetOf(exprEq.get(), rhs.get())); ASSERT_TRUE(expression::isSubsetOf(regularEq.get(), rhs.get())); } } TEST(ExpressionAlgoIsSubsetOf, IsSubsetOfRHSAndWithinOr) { - ParsedMatchExpression rhs("{$or: [{a: 3}, {$and: [{a: 5}, {b: 5}]}]}"); + ParsedMatchExpressionForTest rhs("{$or: [{a: 3}, {$and: [{a: 5}, {b: 5}]}]}"); { - ParsedMatchExpression lhs("{a:5, b:5}"); + ParsedMatchExpressionForTest lhs("{a:5, b:5}"); ASSERT_TRUE(expression::isSubsetOf(lhs.get(), rhs.get())); } } TEST(ExpressionAlgoIsSubsetOf, IsSubsetOfComplexRHSExpression) { - ParsedMatchExpression complex("{$or: [{z: 1}, {$and: [{x: 1}, {$or: [{y: 1}, {y: 2}]}]}]}"); + ParsedMatchExpressionForTest complex( + "{$or: [{z: 1}, {$and: [{x: 1}, {$or: [{y: 1}, {y: 2}]}]}]}"); { - ParsedMatchExpression lhs("{z: 1}"); + ParsedMatchExpressionForTest lhs("{z: 1}"); ASSERT_TRUE(expression::isSubsetOf(lhs.get(), complex.get())); } { - ParsedMatchExpression lhs("{z: 1, x: 1, y:2}"); + ParsedMatchExpressionForTest lhs("{z: 1, x: 1, y:2}"); ASSERT_TRUE(expression::isSubsetOf(lhs.get(), complex.get())); } { - ParsedMatchExpression lhs("{$or: [{z: 1}, {$and: [{x: 1}, {$or: [{y: 1}, {y: 2}]}]}]}"); + ParsedMatchExpressionForTest lhs( + "{$or: [{z: 1}, {$and: [{x: 1}, {$or: [{y: 1}, {y: 2}]}]}]}"); ASSERT_TRUE(expression::isSubsetOf(lhs.get(), complex.get())); } { - ParsedMatchExpression lhs("{$or: [{z: 2}, {$and: [{x: 2}, {$or: [{y: 3}, {y: 4}]}]}]}"); + ParsedMatchExpressionForTest lhs( + "{$or: [{z: 2}, {$and: [{x: 2}, {$or: [{y: 3}, {y: 4}]}]}]}"); ASSERT_FALSE(expression::isSubsetOf(lhs.get(), complex.get())); } { - ParsedMatchExpression lhs("{z: 1, y:2}"); + ParsedMatchExpressionForTest lhs("{z: 1, y:2}"); ASSERT_TRUE(expression::isSubsetOf(lhs.get(), complex.get())); } { - ParsedMatchExpression lhs("{z: 2, y: 1}"); + ParsedMatchExpressionForTest lhs("{z: 2, y: 1}"); ASSERT_FALSE(expression::isSubsetOf(lhs.get(), complex.get())); } { - ParsedMatchExpression lhs("{x: 1, y: 3}"); + ParsedMatchExpressionForTest lhs("{x: 1, y: 3}"); ASSERT_FALSE(expression::isSubsetOf(lhs.get(), complex.get())); } } @@ -959,15 +950,11 @@ TEST(SplitMatchExpression, AndWithSplittableChildrenIsSplittable) { expression::splitMatchExpressionBy(std::move(status.getValue()), {"b"}, {}); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{a: {$eq: 1}}")); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{b: {$eq: 1}}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), fromjson("{a: {$eq: 1}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{b: {$eq: 1}}")); } TEST(SplitMatchExpression, NorWithIndependentChildrenIsSplittable) { @@ -981,15 +968,11 @@ TEST(SplitMatchExpression, NorWithIndependentChildrenIsSplittable) { expression::splitMatchExpressionBy(std::move(status.getValue()), {"b"}, {}); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{$nor: [{a: {$eq: 1}}]}")); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{$nor: [{b: {$eq: 1}}]}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), fromjson("{$nor: [{a: {$eq: 1}}]}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{$nor: [{b: {$eq: 1}}]}")); } TEST(SplitMatchExpression, NotWithIndependentChildIsSplittable) { @@ -1003,10 +986,8 @@ TEST(SplitMatchExpression, NotWithIndependentChildIsSplittable) { expression::splitMatchExpressionBy(std::move(status.getValue()), {"y"}, {}); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{x: {$not: {$gt: 4}}}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), fromjson("{x: {$not: {$gt: 4}}}")); ASSERT_FALSE(splitExpr.second); } @@ -1021,11 +1002,10 @@ TEST(SplitMatchExpression, OrWithOnlyIndependentChildrenIsNotSplittable) { expression::splitMatchExpressionBy(std::move(status.getValue()), {"b"}, {}); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder bob; - splitExpr.second->serialize(&bob, true); ASSERT_FALSE(splitExpr.first); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{$or: [{a: {$eq: 1}}, {b: {$eq: 1}}]}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), + fromjson("{$or: [{a: {$eq: 1}}, {b: {$eq: 1}}]}")); } TEST(SplitMatchExpression, ComplexMatchExpressionSplitsCorrectly) { @@ -1042,15 +1022,12 @@ TEST(SplitMatchExpression, ComplexMatchExpressionSplitsCorrectly) { expression::splitMatchExpressionBy(std::move(status.getValue()), {"x"}, {}); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{$or: [{'a.b': {$eq: 3}}, {'a.b.c': {$eq: 4}}]}")); - ASSERT_BSONOBJ_EQ(secondBob.obj(), + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), + fromjson("{$or: [{'a.b': {$eq: 3}}, {'a.b.c': {$eq: 4}}]}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{$and: [{x: {$not: {$size: 2}}}, {$nor: [{x: {$gt: 4}}, {$and: " "[{x: {$not: {$eq: 1}}}, {y: {$eq: 3}}]}]}]}")); } @@ -1067,15 +1044,12 @@ TEST(SplitMatchExpression, ShouldNotExtractPrefixOfDottedPathAsIndependent) { expression::splitMatchExpressionBy(std::move(status.getValue()), {"a.b"}, {}); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{'a.c': {$eq: 1}}")); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{$and: [{a: {$eq: 1}}, {'a.b': {$eq: 1}}]}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), fromjson("{'a.c': {$eq: 1}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), + fromjson("{$and: [{a: {$eq: 1}}, {'a.b': {$eq: 1}}]}")); } TEST(SplitMatchExpression, ShouldMoveIndependentLeafPredicateAcrossRename) { @@ -1089,9 +1063,7 @@ TEST(SplitMatchExpression, ShouldMoveIndependentLeafPredicateAcrossRename) { expression::splitMatchExpressionBy(std::move(matcher.getValue()), {}, renames); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{b: {$eq: 1}}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), fromjson("{b: {$eq: 1}}")); ASSERT_FALSE(splitExpr.second.get()); } @@ -1107,9 +1079,8 @@ TEST(SplitMatchExpression, ShouldMoveIndependentAndPredicateAcrossRename) { expression::splitMatchExpressionBy(std::move(matcher.getValue()), {}, renames); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{$and: [{c: {$eq: 1}}, {b: {$eq: 2}}]}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), + fromjson("{$and: [{c: {$eq: 1}}, {b: {$eq: 2}}]}")); ASSERT_FALSE(splitExpr.second.get()); } @@ -1125,14 +1096,10 @@ TEST(SplitMatchExpression, ShouldSplitPartiallyDependentAndPredicateAcrossRename expression::splitMatchExpressionBy(std::move(matcher.getValue()), {"b"}, renames); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{c: {$eq: 1}}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), fromjson("{c: {$eq: 1}}")); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{b: {$eq: 2}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{b: {$eq: 2}}")); } TEST(SplitMatchExpression, ShouldSplitPartiallyDependentComplexPredicateMultipleRenames) { @@ -1146,14 +1113,11 @@ TEST(SplitMatchExpression, ShouldSplitPartiallyDependentComplexPredicateMultiple expression::splitMatchExpressionBy(std::move(matcher.getValue()), {"a"}, renames); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{$or: [{d: {$eq: 2}}, {e: {$eq: 3}}]}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), + fromjson("{$or: [{d: {$eq: 2}}, {e: {$eq: 3}}]}")); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{a: {$eq: 1}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{a: {$eq: 1}}")); } TEST(SplitMatchExpression, @@ -1168,14 +1132,11 @@ TEST(SplitMatchExpression, expression::splitMatchExpressionBy(std::move(matcher.getValue()), {"a"}, renames); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{$or: [{x: {$eq: 2}}, {y: {$eq: 3}}]}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), + fromjson("{$or: [{x: {$eq: 2}}, {y: {$eq: 3}}]}")); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{a: {$eq: 1}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{a: {$eq: 1}}")); } TEST(SplitMatchExpression, ShouldNotMoveElemMatchObjectAcrossRename) { @@ -1191,9 +1152,7 @@ TEST(SplitMatchExpression, ShouldNotMoveElemMatchObjectAcrossRename) { ASSERT_FALSE(splitExpr.first.get()); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{a: {$elemMatch: {b: {$eq: 3}}}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{a: {$elemMatch: {b: {$eq: 3}}}}")); } TEST(SplitMatchExpression, ShouldNotMoveElemMatchValueAcrossRename) { @@ -1209,9 +1168,7 @@ TEST(SplitMatchExpression, ShouldNotMoveElemMatchValueAcrossRename) { ASSERT_FALSE(splitExpr.first.get()); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{a: {$elemMatch: {$eq: 3}}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{a: {$elemMatch: {$eq: 3}}}")); } TEST(SplitMatchExpression, ShouldMoveTypeAcrossRename) { @@ -1224,10 +1181,7 @@ TEST(SplitMatchExpression, ShouldMoveTypeAcrossRename) { std::pair<unique_ptr<MatchExpression>, unique_ptr<MatchExpression>> splitExpr = expression::splitMatchExpressionBy(std::move(matcher.getValue()), {}, renames); - ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{c: {$type: [16]}}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), fromjson("{c: {$type: [16]}}")); ASSERT_FALSE(splitExpr.second.get()); } @@ -1245,9 +1199,7 @@ TEST(SplitMatchExpression, ShouldNotMoveSizeAcrossRename) { ASSERT_FALSE(splitExpr.first.get()); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{a: {$size: 3}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{a: {$size: 3}}")); } TEST(SplitMatchExpression, ShouldNotMoveMinItemsAcrossRename) { @@ -1263,9 +1215,8 @@ TEST(SplitMatchExpression, ShouldNotMoveMinItemsAcrossRename) { ASSERT_FALSE(splitExpr.first.get()); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{a: {$_internalSchemaMinItems: 3}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), + fromjson("{a: {$_internalSchemaMinItems: 3}}")); } TEST(SplitMatchExpression, ShouldNotMoveMaxItemsAcrossRename) { @@ -1281,9 +1232,8 @@ TEST(SplitMatchExpression, ShouldNotMoveMaxItemsAcrossRename) { ASSERT_FALSE(splitExpr.first.get()); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), fromjson("{a: {$_internalSchemaMaxItems: 3}}")); + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), + fromjson("{a: {$_internalSchemaMaxItems: 3}}")); } TEST(SplitMatchExpression, ShouldNotMoveMaxItemsInLogicalExpressionAcrossRename) { @@ -1301,9 +1251,7 @@ TEST(SplitMatchExpression, ShouldNotMoveMaxItemsInLogicalExpressionAcrossRename) ASSERT_FALSE(splitExpr.first.get()); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{$or: [{a: {$_internalSchemaMaxItems: 3}}," " {a: {$_internalSchemaMaxItems: 4}}]}")); } @@ -1323,9 +1271,7 @@ TEST(SplitMatchExpression, ShouldNotMoveInternalSchemaObjectMatchInLogicalExpres ASSERT_FALSE(splitExpr.first.get()); ASSERT_TRUE(splitExpr.second.get()); - BSONObjBuilder secondBob; - splitExpr.second->serialize(&secondBob, true); - ASSERT_BSONOBJ_EQ(secondBob.obj(), + ASSERT_BSONOBJ_EQ(splitExpr.second->serialize(), fromjson("{$or: [{a: {$_internalSchemaObjectMatch: {b: {$eq: 1}}}}," " {a: {$_internalSchemaObjectMatch: {b: {$eq: 1}}}}]}")); } @@ -1341,9 +1287,8 @@ TEST(SplitMatchExpression, ShouldMoveMinLengthAcrossRename) { expression::splitMatchExpressionBy(std::move(matcher.getValue()), {}, renames); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{c: {$_internalSchemaMinLength: 3}}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), + fromjson("{c: {$_internalSchemaMinLength: 3}}")); ASSERT_FALSE(splitExpr.second.get()); } @@ -1359,9 +1304,8 @@ TEST(SplitMatchExpression, ShouldMoveMaxLengthAcrossRename) { expression::splitMatchExpressionBy(std::move(matcher.getValue()), {}, renames); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{c: {$_internalSchemaMaxLength: 3}}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), + fromjson("{c: {$_internalSchemaMaxLength: 3}}")); ASSERT_FALSE(splitExpr.second.get()); } @@ -1378,9 +1322,7 @@ TEST(SplitMatchExpression, ShouldMoveIndependentPredicateWhenThereAreMultipleRen expression::splitMatchExpressionBy(std::move(matcher.getValue()), {}, renames); ASSERT_TRUE(splitExpr.first.get()); - BSONObjBuilder firstBob; - splitExpr.first->serialize(&firstBob, true); - ASSERT_BSONOBJ_EQ(firstBob.obj(), fromjson("{x: {$eq: 3}}")); + ASSERT_BSONOBJ_EQ(splitExpr.first->serialize(), fromjson("{x: {$eq: 3}}")); ASSERT_FALSE(splitExpr.second.get()); } @@ -1398,9 +1340,7 @@ TEST(SplitMatchExpression, ShouldNotSplitWhenRand) { ASSERT_FALSE(split.get()); ASSERT_TRUE(residual.get()); - BSONObjBuilder oldBob; - residual->serialize(&oldBob, true); - ASSERT_BSONOBJ_EQ(oldBob.obj(), fromjson(randExpr)); + ASSERT_BSONOBJ_EQ(residual->serialize(), fromjson(randExpr)); }; // We should not push down a $match with a $rand expression. @@ -1605,7 +1545,7 @@ TEST(HasExistencePredicateOnPath, ReturnsFalseWhenExistsOnSubpath) { } TEST(SplitMatchExpressionForColumns, PreservesEmptyPredicates) { - ParsedMatchExpression empty("{}"); + ParsedMatchExpressionForTest empty("{}"); auto splitUp = expression::splitMatchExpressionForColumns(empty.get()); ASSERT(splitUp); ASSERT(splitUp->size() == 0); @@ -1614,21 +1554,21 @@ TEST(SplitMatchExpressionForColumns, PreservesEmptyPredicates) { TEST(SplitMatchExpressionForColumns, RejectsUnsupportedPredicates) { { // Future work. - ParsedMatchExpression orClause("{$or: [{a: 1}, {b: 2}]}"); + ParsedMatchExpressionForTest orClause("{$or: [{a: 1}, {b: 2}]}"); auto splitUp = expression::splitMatchExpressionForColumns(orClause.get()); ASSERT(!splitUp); } { // Would match missing values, not safe for a columnar index. - ParsedMatchExpression alwaysTrue("{$alwaysTrue: 1}"); + ParsedMatchExpressionForTest alwaysTrue("{$alwaysTrue: 1}"); auto splitUp = expression::splitMatchExpressionForColumns(alwaysTrue.get()); ASSERT(!splitUp); } { // Future work. - ParsedMatchExpression exprClause("{$expr: {$eq: ['$x', 0]}}"); + ParsedMatchExpressionForTest exprClause("{$expr: {$eq: ['$x', 0]}}"); auto splitUp = expression::splitMatchExpressionForColumns(exprClause.get()); ASSERT(!splitUp); } @@ -1638,7 +1578,7 @@ TEST(SplitMatchExpressionForColumns, RejectsUnsupportedPredicates) { TEST(SplitMatchExpressionForColumns, SplitsSafeEqualities) { { - ParsedMatchExpression singleEqualsNumber("{albatross: 1}"); + ParsedMatchExpressionForTest singleEqualsNumber("{albatross: 1}"); auto splitUp = expression::splitMatchExpressionForColumns(singleEqualsNumber.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1648,7 +1588,7 @@ TEST(SplitMatchExpressionForColumns, SplitsSafeEqualities) { } { - ParsedMatchExpression singleEqualsString("{albatross: 'flying'}"); + ParsedMatchExpressionForTest singleEqualsString("{albatross: 'flying'}"); auto splitUp = expression::splitMatchExpressionForColumns(singleEqualsString.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1658,7 +1598,7 @@ TEST(SplitMatchExpressionForColumns, SplitsSafeEqualities) { } { - ParsedMatchExpression doubleEqualsNumber("{albatross: 1, blackbird: 2}"); + ParsedMatchExpressionForTest doubleEqualsNumber("{albatross: 1, blackbird: 2}"); auto splitUp = expression::splitMatchExpressionForColumns(doubleEqualsNumber.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 2) << splitUp->size(); @@ -1671,7 +1611,7 @@ TEST(SplitMatchExpressionForColumns, SplitsSafeEqualities) { } { - ParsedMatchExpression mixedEquals( + ParsedMatchExpressionForTest mixedEquals( "{albatross: 1," " blackbird: 'flying'," " cowbird: {$eq: /oreo/}," @@ -1699,7 +1639,7 @@ TEST(SplitMatchExpressionForColumns, SplitsSafeEqualities) { TEST(SplitMatchExpressionForColumns, SupportsEqualityToEmptyObjects) { { - ParsedMatchExpression equalsEmptyObj("{albatross: {}}"); + ParsedMatchExpressionForTest equalsEmptyObj("{albatross: {}}"); auto splitUp = expression::splitMatchExpressionForColumns(equalsEmptyObj.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1711,7 +1651,7 @@ TEST(SplitMatchExpressionForColumns, SupportsEqualityToEmptyObjects) { TEST(SplitMatchExpressionForColumns, SupportsEqualityToEmptyArray) { { - ParsedMatchExpression equalsEmptyArray("{albatross: []}"); + ParsedMatchExpressionForTest equalsEmptyArray("{albatross: []}"); auto splitUp = expression::splitMatchExpressionForColumns(equalsEmptyArray.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1723,7 +1663,7 @@ TEST(SplitMatchExpressionForColumns, SupportsEqualityToEmptyArray) { TEST(SplitMatchExpressionForColumns, DoesNotSupportEqualsNull) { { - ParsedMatchExpression equalsNull("{a: null}"); + ParsedMatchExpressionForTest equalsNull("{a: null}"); auto splitUp = expression::splitMatchExpressionForColumns(equalsNull.get()); ASSERT(!splitUp); } @@ -1731,7 +1671,7 @@ TEST(SplitMatchExpressionForColumns, DoesNotSupportEqualsNull) { TEST(SplitMatchExpressionForColumns, DoesSupportNotEqualsNull) { { - ParsedMatchExpression neNull("{a: {$ne: null}}"); + ParsedMatchExpressionForTest neNull("{a: {$ne: null}}"); auto splitUp = expression::splitMatchExpressionForColumns(neNull.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1740,7 +1680,7 @@ TEST(SplitMatchExpressionForColumns, DoesSupportNotEqualsNull) { << splitUp->at("a")->toString(); } { - ParsedMatchExpression notEqualsNull("{a: {$not: {$eq: null}}}"); + ParsedMatchExpressionForTest notEqualsNull("{a: {$not: {$eq: null}}}"); auto splitUp = expression::splitMatchExpressionForColumns(notEqualsNull.get()); ASSERT(splitUp) << notEqualsNull.get()->toString(); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1752,28 +1692,28 @@ TEST(SplitMatchExpressionForColumns, DoesSupportNotEqualsNull) { TEST(SplitMatchExpressionForColumns, DoesNotSupportCompoundEquals) { { - ParsedMatchExpression implicitEqualsArray("{a: [1, 2]}"); + ParsedMatchExpressionForTest implicitEqualsArray("{a: [1, 2]}"); auto splitUp = expression::splitMatchExpressionForColumns(implicitEqualsArray.get()); ASSERT(!splitUp); } { - ParsedMatchExpression explicitEqualsArray("{a: {$eq: [1, 2]}}"); + ParsedMatchExpressionForTest explicitEqualsArray("{a: {$eq: [1, 2]}}"); auto splitUp = expression::splitMatchExpressionForColumns(explicitEqualsArray.get()); ASSERT(!splitUp); } { - ParsedMatchExpression implicitEqualsObject("{a: {boats: 1, planes: 2}}"); + ParsedMatchExpressionForTest implicitEqualsObject("{a: {boats: 1, planes: 2}}"); auto splitUp = expression::splitMatchExpressionForColumns(implicitEqualsObject.get()); ASSERT(!splitUp); } { - ParsedMatchExpression explicitEqualsObject("{a: {$eq: {boats: 1, planes: 2}}}"); + ParsedMatchExpressionForTest explicitEqualsObject("{a: {$eq: {boats: 1, planes: 2}}}"); auto splitUp = expression::splitMatchExpressionForColumns(explicitEqualsObject.get()); ASSERT(!splitUp); } // We should be able to do dotted path version though, as a potential workaround. { - ParsedMatchExpression equalsDotted("{'a.boats': 1, 'a.planes': 2}"); + ParsedMatchExpressionForTest equalsDotted("{'a.boats': 1, 'a.planes': 2}"); auto splitUp = expression::splitMatchExpressionForColumns(equalsDotted.get()); ASSERT(splitUp); ASSERT(splitUp->size() == 2); @@ -1790,7 +1730,7 @@ TEST(SplitMatchExpressionForColumns, DoesNotSupportCompoundEquals) { TEST(SplitMatchExpressionForColumns, SupportsComparisonsLikeEqualities) { { - ParsedMatchExpression singleLtNumber("{albatross: {$lt: 1}}"); + ParsedMatchExpressionForTest singleLtNumber("{albatross: {$lt: 1}}"); auto splitUp = expression::splitMatchExpressionForColumns(singleLtNumber.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1799,7 +1739,7 @@ TEST(SplitMatchExpressionForColumns, SupportsComparisonsLikeEqualities) { << splitUp->at("albatross")->toString(); } { - ParsedMatchExpression singleLteNumber("{albatross: {$lte: 1}}"); + ParsedMatchExpressionForTest singleLteNumber("{albatross: {$lte: 1}}"); auto splitUp = expression::splitMatchExpressionForColumns(singleLteNumber.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1808,7 +1748,7 @@ TEST(SplitMatchExpressionForColumns, SupportsComparisonsLikeEqualities) { << splitUp->at("albatross")->toString(); } { - ParsedMatchExpression singleGtNumber("{albatross: {$gt: 1}}"); + ParsedMatchExpressionForTest singleGtNumber("{albatross: {$gt: 1}}"); auto splitUp = expression::splitMatchExpressionForColumns(singleGtNumber.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1817,7 +1757,7 @@ TEST(SplitMatchExpressionForColumns, SupportsComparisonsLikeEqualities) { << splitUp->at("albatross")->toString(); } { - ParsedMatchExpression singleGteNumber("{albatross: {$gte: 1}}"); + ParsedMatchExpressionForTest singleGteNumber("{albatross: {$gte: 1}}"); auto splitUp = expression::splitMatchExpressionForColumns(singleGteNumber.get()); ASSERT(splitUp); ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); @@ -1826,7 +1766,7 @@ TEST(SplitMatchExpressionForColumns, SupportsComparisonsLikeEqualities) { << splitUp->at("albatross")->toString(); } { - ParsedMatchExpression combinationPredicate( + ParsedMatchExpressionForTest combinationPredicate( "{" " albatross: {$lt: 100}," " blackbird: {$gt: 0}," @@ -1850,42 +1790,42 @@ TEST(SplitMatchExpressionForColumns, SupportsComparisonsLikeEqualities) { // While equality to [] or {} is OK, inequality is not so obvious. Left as future work. TEST(SplitMatchExpressionForColumns, DoesNotSupportInequalitiesToObjectsOrArrays) { { - ParsedMatchExpression ltArray("{albatross: {$lt: []}}"); + ParsedMatchExpressionForTest ltArray("{albatross: {$lt: []}}"); auto splitUp = expression::splitMatchExpressionForColumns(ltArray.get()); ASSERT(!splitUp); } { - ParsedMatchExpression ltObject("{albatross: {$lt: {}}}"); + ParsedMatchExpressionForTest ltObject("{albatross: {$lt: {}}}"); auto splitUp = expression::splitMatchExpressionForColumns(ltObject.get()); ASSERT(!splitUp); } { - ParsedMatchExpression lteArray("{albatross: {$lte: []}}"); + ParsedMatchExpressionForTest lteArray("{albatross: {$lte: []}}"); auto splitUp = expression::splitMatchExpressionForColumns(lteArray.get()); ASSERT(!splitUp); } { - ParsedMatchExpression lteObject("{albatross: {$lte: {}}}"); + ParsedMatchExpressionForTest lteObject("{albatross: {$lte: {}}}"); auto splitUp = expression::splitMatchExpressionForColumns(lteObject.get()); ASSERT(!splitUp); } { - ParsedMatchExpression gtArray("{albatross: {$gt: []}}"); + ParsedMatchExpressionForTest gtArray("{albatross: {$gt: []}}"); auto splitUp = expression::splitMatchExpressionForColumns(gtArray.get()); ASSERT(!splitUp); } { - ParsedMatchExpression gtObject("{albatross: {$gt: {}}}"); + ParsedMatchExpressionForTest gtObject("{albatross: {$gt: {}}}"); auto splitUp = expression::splitMatchExpressionForColumns(gtObject.get()); ASSERT(!splitUp); } { - ParsedMatchExpression gteArray("{albatross: {$gte: []}}"); + ParsedMatchExpressionForTest gteArray("{albatross: {$gte: []}}"); auto splitUp = expression::splitMatchExpressionForColumns(gteArray.get()); ASSERT(!splitUp); } { - ParsedMatchExpression gteObject("{albatross: {$gte: {}}}"); + ParsedMatchExpressionForTest gteObject("{albatross: {$gte: {}}}"); auto splitUp = expression::splitMatchExpressionForColumns(gteObject.get()); ASSERT(!splitUp); } @@ -1893,7 +1833,7 @@ TEST(SplitMatchExpressionForColumns, DoesNotSupportInequalitiesToObjectsOrArrays // Tests that comparisons which only match values of a certain type are allowed. TEST(SplitMatchExpressionForColumns, SupportsTypeSpecificPredicates) { - ParsedMatchExpression combinationPredicate( + ParsedMatchExpressionForTest combinationPredicate( "{" " albatross: /oreo/," " blackbird: {$mod: [2, 0]}," @@ -1928,7 +1868,7 @@ TEST(SplitMatchExpressionForColumns, SupportsTypeSpecificPredicates) { TEST(SplitMatchExpressionForColumns, SupportsInWithRegexes) { { // First confirm a $in clause is supported without regexes. - ParsedMatchExpression stringInClause("{albatross: {$in: ['big', 'ol', 'bird']}}"); + ParsedMatchExpressionForTest stringInClause("{albatross: {$in: ['big', 'ol', 'bird']}}"); auto splitUp = expression::splitMatchExpressionForColumns(stringInClause.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -1938,7 +1878,7 @@ TEST(SplitMatchExpressionForColumns, SupportsInWithRegexes) { } { // Test that $in with regexes is supported also work. - ParsedMatchExpression regexInClause("{albatross: {$in: [/big/, /bird/]}}"); + ParsedMatchExpressionForTest regexInClause("{albatross: {$in: [/big/, /bird/]}}"); auto splitUp = expression::splitMatchExpressionForColumns(regexInClause.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -1948,7 +1888,7 @@ TEST(SplitMatchExpressionForColumns, SupportsInWithRegexes) { } { // Test that a mix of both is supported - ParsedMatchExpression regexInClause("{albatross: {$in: [/big/, 'bird']}}"); + ParsedMatchExpressionForTest regexInClause("{albatross: {$in: [/big/, 'bird']}}"); auto splitUp = expression::splitMatchExpressionForColumns(regexInClause.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -1958,14 +1898,14 @@ TEST(SplitMatchExpressionForColumns, SupportsInWithRegexes) { } { // Test that it is still disallowed if there's a disqualifying equality such as a null. - ParsedMatchExpression regexInClause("{albatross: {$in: [/big/, null, 'bird']}}"); + ParsedMatchExpressionForTest regexInClause("{albatross: {$in: [/big/, null, 'bird']}}"); auto splitUp = expression::splitMatchExpressionForColumns(regexInClause.get()); ASSERT(!splitUp); } } TEST(SplitMatchExpressionForColumns, SupportsExistsTrue) { - ParsedMatchExpression existsPredicate("{albatross: {$exists: true}}"); + ParsedMatchExpressionForTest existsPredicate("{albatross: {$exists: true}}"); auto splitUp = expression::splitMatchExpressionForColumns(existsPredicate.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -1975,7 +1915,7 @@ TEST(SplitMatchExpressionForColumns, SupportsExistsTrue) { } TEST(SplitMatchExpressionForColumns, DoesNotSupportExistsFalse) { - ParsedMatchExpression existsPredicate("{albatross: {$exists: false}}"); + ParsedMatchExpressionForTest existsPredicate("{albatross: {$exists: false}}"); auto splitUp = expression::splitMatchExpressionForColumns(existsPredicate.get()); ASSERT(!splitUp); } @@ -1984,7 +1924,7 @@ TEST(SplitMatchExpressionForColumns, DoesNotSupportExistsFalse) { // next test. TEST(SplitMatchExpressionForColumns, SupportsInPredicates) { { - ParsedMatchExpression emptyIn("{albatross: {$in: []}}"); + ParsedMatchExpressionForTest emptyIn("{albatross: {$in: []}}"); auto splitUp = expression::splitMatchExpressionForColumns(emptyIn.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -1993,7 +1933,7 @@ TEST(SplitMatchExpressionForColumns, SupportsInPredicates) { ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); } { - ParsedMatchExpression singleElementIn("{albatross: {$in: [4]}}"); + ParsedMatchExpressionForTest singleElementIn("{albatross: {$in: [4]}}"); auto splitUp = expression::splitMatchExpressionForColumns(singleElementIn.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -2002,7 +1942,7 @@ TEST(SplitMatchExpressionForColumns, SupportsInPredicates) { ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); } { - ParsedMatchExpression inWithEmptyArray("{albatross: {$in: [[]]}}"); + ParsedMatchExpressionForTest inWithEmptyArray("{albatross: {$in: [[]]}}"); auto splitUp = expression::splitMatchExpressionForColumns(inWithEmptyArray.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -2011,7 +1951,7 @@ TEST(SplitMatchExpressionForColumns, SupportsInPredicates) { ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); } { - ParsedMatchExpression inWithEmptyObject("{albatross: {$in: [{}]}}"); + ParsedMatchExpressionForTest inWithEmptyObject("{albatross: {$in: [{}]}}"); auto splitUp = expression::splitMatchExpressionForColumns(inWithEmptyObject.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -2020,7 +1960,8 @@ TEST(SplitMatchExpressionForColumns, SupportsInPredicates) { ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); } { - ParsedMatchExpression mixedTypeIn("{albatross: {$in: [4, {}, [], 'string', /regex/]}}"); + ParsedMatchExpressionForTest mixedTypeIn( + "{albatross: {$in: [4, {}, [], 'string', /regex/]}}"); auto splitUp = expression::splitMatchExpressionForColumns(mixedTypeIn.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -2033,22 +1974,22 @@ TEST(SplitMatchExpressionForColumns, SupportsInPredicates) { // We can't support compound types, just like for equality. TEST(SplitMatchExpressionForColumns, DoesNotSupportCertainInEdgeCases) { { - ParsedMatchExpression inWithArray("{albatross: {$in: [[2,3]]}}"); + ParsedMatchExpressionForTest inWithArray("{albatross: {$in: [[2,3]]}}"); auto splitUp = expression::splitMatchExpressionForColumns(inWithArray.get()); ASSERT(!splitUp); } { - ParsedMatchExpression inWithObject("{albatross: {$in: [{wings: 2}]}}"); + ParsedMatchExpressionForTest inWithObject("{albatross: {$in: [{wings: 2}]}}"); auto splitUp = expression::splitMatchExpressionForColumns(inWithObject.get()); ASSERT(!splitUp); } { - ParsedMatchExpression inWithNull("{albatross: {$in: [null]}}"); + ParsedMatchExpressionForTest inWithNull("{albatross: {$in: [null]}}"); auto splitUp = expression::splitMatchExpressionForColumns(inWithNull.get()); ASSERT(!splitUp); } { - ParsedMatchExpression unsupporedMixedInWithSupported( + ParsedMatchExpressionForTest unsupporedMixedInWithSupported( "{albatross: {$in: ['strings', 1, null, {x: 4}, [0, 0], 4]}}"); auto splitUp = expression::splitMatchExpressionForColumns(unsupporedMixedInWithSupported.get()); @@ -2058,7 +1999,7 @@ TEST(SplitMatchExpressionForColumns, DoesNotSupportCertainInEdgeCases) { TEST(SplitMatchExpressionForColumns, SupportsTypePredicates) { { - ParsedMatchExpression intFilter("{albatross: {$type: 'int'}}"); + ParsedMatchExpressionForTest intFilter("{albatross: {$type: 'int'}}"); auto splitUp = expression::splitMatchExpressionForColumns(intFilter.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -2067,7 +2008,7 @@ TEST(SplitMatchExpressionForColumns, SupportsTypePredicates) { ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); } { - ParsedMatchExpression numberFilter("{albatross: {$type: 'number'}}"); + ParsedMatchExpressionForTest numberFilter("{albatross: {$type: 'number'}}"); auto splitUp = expression::splitMatchExpressionForColumns(numberFilter.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -2076,7 +2017,7 @@ TEST(SplitMatchExpressionForColumns, SupportsTypePredicates) { ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); } { - ParsedMatchExpression stringFilter("{albatross: {$type: 'string'}}"); + ParsedMatchExpressionForTest stringFilter("{albatross: {$type: 'string'}}"); auto splitUp = expression::splitMatchExpressionForColumns(stringFilter.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -2085,7 +2026,7 @@ TEST(SplitMatchExpressionForColumns, SupportsTypePredicates) { ASSERT_EQ(splitUp->size(), 1) << splitUp->size(); } { - ParsedMatchExpression nullFilter("{albatross: {$type: 'null'}}"); + ParsedMatchExpressionForTest nullFilter("{albatross: {$type: 'null'}}"); auto splitUp = expression::splitMatchExpressionForColumns(nullFilter.get()); ASSERT(splitUp); ASSERT(splitUp->contains("albatross")); @@ -2096,20 +2037,20 @@ TEST(SplitMatchExpressionForColumns, SupportsTypePredicates) { } TEST(SplitMatchExpressionForColumns, DoesNotSupportQueriesForTypeObject) { - ParsedMatchExpression objectFilter("{albatross: {$type: 'object'}}"); + ParsedMatchExpressionForTest objectFilter("{albatross: {$type: 'object'}}"); auto splitUp = expression::splitMatchExpressionForColumns(objectFilter.get()); ASSERT(!splitUp); } // This may be workable. But until we can prove it we'll disallow {$type: "array"}. TEST(SplitMatchExpressionForColumns, DoesNotSupportQueriesForTypeArray) { - ParsedMatchExpression objectFilter("{albatross: {$type: 'array'}}"); + ParsedMatchExpressionForTest objectFilter("{albatross: {$type: 'array'}}"); auto splitUp = expression::splitMatchExpressionForColumns(objectFilter.get()); ASSERT(!splitUp); } TEST(SplitMatchExpressionForColumns, CanCombinePredicates) { - ParsedMatchExpression compoundFilter( + ParsedMatchExpressionForTest compoundFilter( "{" " albatross: {$gte: 100}," " albatross: {$mod: [2, 0]}" @@ -2134,7 +2075,7 @@ TEST(SplitMatchExpressionForColumns, CanCombinePredicates) { } TEST(SplitMatchExpressionForColumns, SupportsDottedPaths) { - ParsedMatchExpression compoundFilter( + ParsedMatchExpressionForTest compoundFilter( "{" " albatross: /oreo/," " \"blackbird.feet\": {$mod: [2, 0]}," @@ -2177,7 +2118,7 @@ TEST(SplitMatchExpressionForColumns, SupportsDottedPaths) { } TEST(SplitMatchExpressionForColumns, LeavesOriginalMatchExpressionFunctional) { - ParsedMatchExpression combinationPredicate( + ParsedMatchExpressionForTest combinationPredicate( "{" " albatross: {$lt: 100}," " blackbird: {$gt: 0}," diff --git a/src/mongo/db/matcher/expression_always_boolean.h b/src/mongo/db/matcher/expression_always_boolean.h index 6e0e2cdfbc8..2ad7e25d85c 100644 --- a/src/mongo/db/matcher/expression_always_boolean.h +++ b/src/mongo/db/matcher/expression_always_boolean.h @@ -62,8 +62,10 @@ public: debug << name() << ": 1\n"; } - void serialize(BSONObjBuilder* out, bool includePath) const final { - out->append(name(), 1); + void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const final { + opts.appendLiteral(out, name(), 1); } bool equivalent(const MatchExpression* other) const final { diff --git a/src/mongo/db/matcher/expression_arity.h b/src/mongo/db/matcher/expression_arity.h index 90286200e21..840d4be61f6 100644 --- a/src/mongo/db/matcher/expression_arity.h +++ b/src/mongo/db/matcher/expression_arity.h @@ -58,7 +58,7 @@ public: _debugAddSpace(debug, indentationLevel); BSONObjBuilder builder; - serialize(&builder, true); + serialize(&builder, {}); debug << builder.obj().toString(); } @@ -107,11 +107,13 @@ public: /** * Serializes each subexpression sequentially in a BSONArray. */ - void serialize(BSONObjBuilder* builder, bool includePath) const final { + void serialize(BSONObjBuilder* builder, + const SerializationOptions& opts = {}, + bool includePath = true) const final { BSONArrayBuilder exprArray(builder->subarrayStart(name())); for (const auto& expr : _expressions) { BSONObjBuilder exprBuilder(exprArray.subobjStart()); - expr->serialize(&exprBuilder, includePath); + expr->serialize(&exprBuilder, opts, includePath); exprBuilder.doneFast(); } exprArray.doneFast(); diff --git a/src/mongo/db/matcher/expression_array.cpp b/src/mongo/db/matcher/expression_array.cpp index cbff0adf1ca..42d30aaf0f1 100644 --- a/src/mongo/db/matcher/expression_array.cpp +++ b/src/mongo/db/matcher/expression_array.cpp @@ -101,10 +101,13 @@ void ElemMatchObjectMatchExpression::debugString(StringBuilder& debug, int inden _sub->debugString(debug, indentationLevel + 1); } -BSONObj ElemMatchObjectMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder subBob; - _sub->serialize(&subBob, true); - return BSON("$elemMatch" << subBob.obj()); +void ElemMatchObjectMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + BSONObjBuilder elemMatchBob = bob->subobjStart("$elemMatch"); + SerializationOptions options = opts; + _sub->serialize(&elemMatchBob, options, true); + elemMatchBob.doneFast(); } MatchExpression::ExpressionOptimizerFunc ElemMatchObjectMatchExpression::getOptimizer() const { @@ -170,14 +173,15 @@ void ElemMatchValueMatchExpression::debugString(StringBuilder& debug, int indent } } -BSONObj ElemMatchValueMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder emBob; - +void ElemMatchValueMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + BSONObjBuilder emBob = bob->subobjStart("$elemMatch"); + SerializationOptions options = opts; for (auto&& child : _subs) { - child->serialize(&emBob, false); + child->serialize(&emBob, options, false); } - - return BSON("$elemMatch" << emBob.obj()); + emBob.doneFast(); } MatchExpression::ExpressionOptimizerFunc ElemMatchValueMatchExpression::getOptimizer() const { @@ -215,8 +219,10 @@ void SizeMatchExpression::debugString(StringBuilder& debug, int indentationLevel } } -BSONObj SizeMatchExpression::getSerializedRightHandSide() const { - return BSON("$size" << _size); +void SizeMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + opts.appendLiteral(bob, "$size", _size); } bool SizeMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/expression_array.h b/src/mongo/db/matcher/expression_array.h index 1b6cb5dfcc9..10f30629c92 100644 --- a/src/mongo/db/matcher/expression_array.h +++ b/src/mongo/db/matcher/expression_array.h @@ -90,7 +90,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; std::vector<std::unique_ptr<MatchExpression>>* getChildVector() final { return nullptr; @@ -157,7 +159,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; std::vector<std::unique_ptr<MatchExpression>>* getChildVector() final { return &_subs; @@ -230,7 +234,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; virtual bool equivalent(const MatchExpression* other) const; diff --git a/src/mongo/db/matcher/expression_expr.cpp b/src/mongo/db/matcher/expression_expr.cpp index fa9faa3c2c9..1ce4f0f6954 100644 --- a/src/mongo/db/matcher/expression_expr.cpp +++ b/src/mongo/db/matcher/expression_expr.cpp @@ -77,8 +77,10 @@ Value ExprMatchExpression::evaluateExpression(const MatchableDocument* doc) cons return _expression->evaluate(document, &variables); } -void ExprMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const { - *out << "$expr" << _expression->serialize(false); +void ExprMatchExpression::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, + bool includePath) const { + *out << "$expr" << _expression->serialize(opts); } bool ExprMatchExpression::equivalent(const MatchExpression* other) const { @@ -94,8 +96,8 @@ bool ExprMatchExpression::equivalent(const MatchExpression* other) const { } // TODO SERVER-30982: Add mechanism to allow for checking Expression equivalency. - return ValueComparator().evaluate(_expression->serialize(false) == - realOther->_expression->serialize(false)); + return ValueComparator().evaluate(_expression->serialize() == + realOther->_expression->serialize()); } void ExprMatchExpression::_doSetCollator(const CollatorInterface* collator) { @@ -114,7 +116,7 @@ void ExprMatchExpression::_doSetCollator(const CollatorInterface* collator) { std::unique_ptr<MatchExpression> ExprMatchExpression::shallowClone() const { // TODO SERVER-31003: Replace Expression clone via serialization with Expression::clone(). BSONObjBuilder bob; - bob << "" << _expression->serialize(false); + bob << "" << _expression->serialize(); boost::intrusive_ptr<Expression> clonedExpr = Expression::parseOperand( _expCtx.get(), bob.obj().firstElement(), _expCtx->variablesParseState); diff --git a/src/mongo/db/matcher/expression_expr.h b/src/mongo/db/matcher/expression_expr.h index 281fdd3afd6..b3f670e4597 100644 --- a/src/mongo/db/matcher/expression_expr.h +++ b/src/mongo/db/matcher/expression_expr.h @@ -72,10 +72,12 @@ public: void debugString(StringBuilder& debug, int indentationLevel = 0) const final { _debugAddSpace(debug, indentationLevel); - debug << "$expr " << _expression->serialize(false).toString(); + debug << "$expr " << _expression->serialize().toString(); } - void serialize(BSONObjBuilder* out, bool includePath) const final; + void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/expression_expr_test.cpp b/src/mongo/db/matcher/expression_expr_test.cpp index 5ff0cbf88e9..6f91f12bc5d 100644 --- a/src/mongo/db/matcher/expression_expr_test.cpp +++ b/src/mongo/db/matcher/expression_expr_test.cpp @@ -37,6 +37,8 @@ #include "mongo/db/matcher/matcher.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/collation/collator_interface_mock.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/unittest/death_test.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -81,6 +83,10 @@ public: return checked_cast<ExprMatchExpression*>(_matchExpression.get()); } + BSONObj serialize(const SerializationOptions& opts) { + return _matchExpression->serialize(opts); + } + private: const boost::intrusive_ptr<ExpressionContextForTest> _expCtx; std::unique_ptr<MatchExpression> _matchExpression; @@ -714,17 +720,10 @@ TEST(ExprMatchTest, OptimizingExprAbsorbsAndOfAnd) { // The optimized match expression should not have and AND children of AND nodes. This should be // collapsed during optimization. - BSONObj serialized; - { - BSONObjBuilder builder; - optimized->serialize(&builder, true); - serialized = builder.obj(); - } - BSONObj expectedSerialization = fromjson( "{$and: [{$expr: {$and: [{$eq: ['$a', {$const: 1}]}, {$eq: ['$b', {$const: 2}]}]}}," "{a: {$_internalExprEq: 1}}, {b: {$_internalExprEq: 2}}]}"); - ASSERT_BSONOBJ_EQ(serialized, expectedSerialization); + ASSERT_BSONOBJ_EQ(optimized->serialize(), expectedSerialization); } TEST_F(ExprMatchTest, ExpressionEvaluationReturnsResultsCorrectly) { @@ -735,5 +734,198 @@ TEST_F(ExprMatchTest, ExpressionEvaluationReturnsResultsCorrectly) { ASSERT_EQUALS(-2, expressionResult.coerceToInt()); } +TEST_F(ExprMatchTest, ExprRedactsCorrectly) { + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + createMatcher(fromjson("{$expr: {$sum: [\"$a\", \"$b\"]}}")); + + SerializationOptions opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$expr":{"$sum":["$HASH<a>","$HASH<b>"]}})", + serialize(opts)); + + createMatcher(fromjson("{$expr: {$sum: [\"$a\", \"b\"]}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$expr":{"$sum":["$HASH<a>","?string"]}})", + serialize(opts)); + + createMatcher(fromjson("{$expr: {$sum: [\"$a.b\", \"$b\"]}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$expr":{"$sum":["$HASH<a>.HASH<b>","$HASH<b>"]}})", + serialize(opts)); + + createMatcher(fromjson("{$expr: {$eq: [\"$a\", \"$$NOW\"]}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "HASH<a>": { + "$_internalExprEq": "?date" + } + }, + { + "$expr": { + "$eq": [ + "$HASH<a>", + "?date" + ] + } + } + ] + })", + serialize(opts)); + + createMatcher(fromjson("{$expr: {$eq: [\"$a\", \"$$NOW\"]}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "HASH<a>": { + "$_internalExprEq": "?date" + } + }, + { + "$expr": { + "$eq": [ + "$HASH<a>", + "?date" + ] + } + } + ] + })", + serialize(opts)); + + createMatcher(fromjson("{$expr: {$getField: {field: \"b\", input: {a: 1, b: 2}}}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$expr":{"$getField":{"field":"HASH<b>","input":"?object"}}})", + serialize(opts)); + + createMatcher(fromjson("{$expr: {$getField: {field: \"b\", input: \"$a\"}}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$expr":{"$getField":{"field":"HASH<b>","input":"$HASH<a>"}}})", + serialize(opts)); + + createMatcher(fromjson("{$expr: {$getField: {field: \"b\", input: {a: 1, b: \"$c\"}}}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$expr": { + "$getField": { + "field": "HASH<b>", + "input": { + "HASH<a>": "?number", + "HASH<b>": "$HASH<c>" + } + } + } + })", + serialize(opts)); + + createMatcher(fromjson("{$expr: {$getField: {field: \"b.c\", input: {a: 1, b: \"$c\"}}}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$expr": { + "$getField": { + "field": "HASH<b>.HASH<c>", + "input": { + "HASH<a>": "?number", + "HASH<b>": "$HASH<c>" + } + } + } + })", + serialize(opts)); + + createMatcher( + fromjson("{$expr: {$setField: {field: \"b\", input: {a: 1, b: \"$c\"}, value: 5}}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$expr": { + "$setField": { + "field": "HASH<b>", + "input": { + "HASH<a>": "?number", + "HASH<b>": "$HASH<c>" + }, + "value": "?number" + } + } + })", + serialize(opts)); + + createMatcher(fromjson( + "{$expr: {$setField: {field: \"b.c\", input: {a: 1, b: \"$c\"}, value: \"$d\"}}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$expr": { + "$setField": { + "field": "HASH<b>.HASH<c>", + "input": { + "HASH<a>": "?number", + "HASH<b>": "$HASH<c>" + }, + "value": "$HASH<d>" + } + } + })", + serialize(opts)); + + createMatcher(fromjson( + "{$expr: {$setField: {field: \"b.c\", input: {a: 1, b: \"$c\"}, value: \"$d.e\"}}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$expr": { + "$setField": { + "field": "HASH<b>.HASH<c>", + "input": { + "HASH<a>": "?number", + "HASH<b>": "$HASH<c>" + }, + "value": "$HASH<d>.HASH<e>" + } + } + })", + serialize(opts)); + + createMatcher( + fromjson("{$expr: {$setField: {field: \"b\", input: {a: 1, b: \"$c\"}, value: {a: 1, b: 2, " + "c: 3}}}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$expr": { + "$setField": { + "field": "HASH<b>", + "input": { + "HASH<a>": "?number", + "HASH<b>": "$HASH<c>" + }, + "value": "?object" + } + } + })", + serialize(opts)); + + createMatcher( + fromjson("{$expr: {$setField: {field: \"b\", input: {a: 1, b: \"$c\"}, value: {a: 1, b: 2, " + "c: \"$d\"}}}}")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$expr": { + "$setField": { + "field": "HASH<b>", + "input": { + "HASH<a>": "?number", + "HASH<b>": "$HASH<c>" + }, + "value": { + "HASH<a>": "?number", + "HASH<b>": "?number", + "HASH<c>": "$HASH<d>" + } + } + } + })", + serialize(opts)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/matcher/expression_geo.cpp b/src/mongo/db/matcher/expression_geo.cpp index fe592df742e..9050de83741 100644 --- a/src/mongo/db/matcher/expression_geo.cpp +++ b/src/mongo/db/matcher/expression_geo.cpp @@ -33,6 +33,7 @@ #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/db/geo/geoparser.h" +#include "mongo/db/matcher/expression_geo_serializer.h" #include "mongo/db/matcher/expression_parser.h" #include "mongo/logv2/log.h" #include "mongo/platform/basic.h" @@ -244,6 +245,11 @@ Status GeoNearExpression::parseNewQuery(const BSONObj& obj) { if (fieldName == "$geometry") { if (e.isABSONObj()) { BSONObj embeddedObj = e.embeddedObject(); + // TODO SERVER-84598: $geoNear queries don't do a type check on the type of the + // geometry provided into the query. This is different than the behaviour we see for + // other GeoExpressions that use a GeometryContainer. GeoNear is a special case that + // only ever has type 'Point', however it may be a good idea to consider removing + // this silent acceptance of invalid "type" in the long term. Status status = GeoParser::parseQueryPoint(e, centroid.get()); if (!status.isOK()) { return Status(ErrorCodes::BadValue, @@ -426,7 +432,7 @@ void GeoMatchExpression::debugString(StringBuilder& debug, int indentationLevel) _debugAddSpace(debug, indentationLevel); BSONObjBuilder builder; - serialize(&builder, true); + serialize(&builder, {}); debug << "GEO raw = " << builder.obj().toString(); MatchExpression::TagData* td = getTag(); @@ -437,10 +443,14 @@ void GeoMatchExpression::debugString(StringBuilder& debug, int indentationLevel) debug << "\n"; } -BSONObj GeoMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder subobj; - subobj.appendElements(_rawObj); - return subobj.obj(); +void GeoMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + geoExpressionCustomSerialization(*bob, _rawObj, opts, includePath); + return; + } + bob->appendElements(_rawObj); } bool GeoMatchExpression::equivalent(const MatchExpression* other) const { @@ -495,10 +505,14 @@ void GeoNearMatchExpression::debugString(StringBuilder& debug, int indentationLe debug << "\n"; } -BSONObj GeoNearMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder objBuilder; - objBuilder.appendElements(_rawObj); - return objBuilder.obj(); +void GeoNearMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + geoNearExpressionCustomSerialization(*bob, _rawObj, opts, includePath); + return; + } + bob->appendElements(_rawObj); } bool GeoNearMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/expression_geo.h b/src/mongo/db/matcher/expression_geo.h index bd3752399c8..b58904a7eff 100644 --- a/src/mongo/db/matcher/expression_geo.h +++ b/src/mongo/db/matcher/expression_geo.h @@ -73,7 +73,8 @@ private: // e.g. // { "$intersect" : { "$geometry" : { "type" : "Point", "coordinates": [ 40, 5 ] } } } Status parseQuery(const BSONObj& obj); - + BSONObj redactGeoExpression(const BSONObj& obj, + boost::optional<StringData> literalArgsReplacement); // Name of the field in the query. std::string field; std::shared_ptr<GeometryContainer> geoContainer; @@ -106,7 +107,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel = 0) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; virtual bool equivalent(const MatchExpression* other) const; @@ -206,7 +209,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel = 0) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; virtual bool equivalent(const MatchExpression* other) const; @@ -246,8 +251,10 @@ public: TwoDPtInAnnulusExpression(const R2Annulus& annulus, StringData twoDPath) : LeafMatchExpression(INTERNAL_2D_POINT_IN_ANNULUS, twoDPath), _annulus(annulus) {} - void serialize(BSONObjBuilder* out, bool includePath) const final { - out->append("TwoDPtInAnnulusExpression", true); + void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const final { + out->append("$TwoDPtInAnnulusExpression", true); } bool matchesSingleElement(const BSONElement& e, MatchDetails* details = nullptr) const final { @@ -265,7 +272,9 @@ public: // These won't be called. // - BSONObj getSerializedRightHandSide() const final { + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final { MONGO_UNREACHABLE; } diff --git a/src/mongo/db/matcher/expression_geo_serializer.cpp b/src/mongo/db/matcher/expression_geo_serializer.cpp new file mode 100644 index 00000000000..04643133335 --- /dev/null +++ b/src/mongo/db/matcher/expression_geo_serializer.cpp @@ -0,0 +1,391 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/matcher/expression_geo_serializer.h" + +#include "mongo/db/geo/geoparser.h" + +namespace mongo { +namespace { +void appendGeoNearLegacyArray(BSONObjBuilder& bob, + const BSONElement& e, + const SerializationOptions& opts) { + if (opts.literalPolicy != LiteralSerializationPolicy::kToRepresentativeParseableValue) { + opts.appendLiteral(&bob, e); + } else { + // Legacy $geoNear, $nearSphere, and $near require at minimum 2 coordinates to be + // re-parseable, so the representative value is [1, 1]. + StringData fieldName = e.fieldNameStringData(); + bob.appendArray(fieldName, BSON_ARRAY(1 << 1)); + } +} + +void appendShapeOperator(BSONObjBuilder& bob, + const BSONElement& e, + const SerializationOptions& opts) { + if (opts.literalPolicy != LiteralSerializationPolicy::kToRepresentativeParseableValue) { + opts.appendLiteral(&bob, e); + return; + } + + StringData fieldName = e.fieldNameStringData(); + if (fieldName == "$center"_sd || fieldName == "$centerSphere"_sd) { + // $center and $centerSphere requires a pair of coordinates and a radius to be + // re-parseable, so the representative value is [[1, 1],1]. + bob.appendArray(fieldName, BSON_ARRAY(BSON_ARRAY(1 << 1) << 1)); + } else if (fieldName == "$box"_sd) { + // $box requires two pairs of coordinates to be re-parseable, so the + // representative value is [[1, 1],[1,1]]. + bob.appendArray(fieldName, BSON_ARRAY(BSON_ARRAY(1 << 1) << BSON_ARRAY(1 << 1))); + } else if (fieldName == "$polygon"_sd) { + // $polygon requires three pairs of coordinates to be re-parseable, so the representative + // value is [[0,0],[0,1],[1,1]]. + bob.appendArray(fieldName, + BSON_ARRAY(BSON_ARRAY(0 << 0) << BSON_ARRAY(0 << 1) << BSON_ARRAY(1 << 1))); + } else { + opts.appendLiteral(&bob, e); + } +} + +void appendGeoJSONCoordinatesLiteral(BSONObjBuilder& bob, + const BSONElement& coordinatesElem, + const BSONElement& typeElem, + const SerializationOptions& opts) { + if (opts.literalPolicy != LiteralSerializationPolicy::kToRepresentativeParseableValue) { + opts.appendLiteral(&bob, coordinatesElem); + return; + } + + StringData fieldName = coordinatesElem.fieldNameStringData(); + + // When a $geoNear expression is parsed (see GeoNearExpression::parseNewQuery()), a $geometry + // object defaults to being parsed as a point, without checking the type of the geometry object. + // This means we can query for a $geoNear expression that specifies an invalid type, or no type + // at all. In order to accomodate this case, we default to type: 'Point' to ensure our + // representative shape is re-parseable. + auto geoType = GeoParser::geoJSONTypeStringToEnum(typeElem.valueStringData()); + if (geoType == GeoParser::GEOJSON_UNKNOWN) { + geoType = GeoParser::GEOJSON_POINT; + } + switch (geoType) { + case GeoParser::GEOJSON_POLYGON: { + // Polygon requires four pairs of coordinates in a closed loop wrapped in an array to be + // re-parseable, so the representative value is [[[0,0],[0,1],[1,1],[0,0]]]. + bob.appendArray( + fieldName, + BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(0 << 0) << BSON_ARRAY(0 << 1) << BSON_ARRAY(1 << 1) + << BSON_ARRAY(0 << 0)))); + return; + } + case GeoParser::GEOJSON_MULTI_POLYGON: { + // MultiPolygon requires four pairs of coordinates in a closed loop wrapped in 2 arrays + // to be re-parseable, so the representative value is [[[[0,0],[0,1],[1,1],[0,0]]]]. + bob.appendArray(fieldName, + BSON_ARRAY(BSON_ARRAY(BSON_ARRAY( + BSON_ARRAY(0 << 0) << BSON_ARRAY(0 << 1) << BSON_ARRAY(1 << 1) + << BSON_ARRAY(0 << 0))))); + return; + } + case GeoParser::GEOJSON_POINT: { + // Point requires a pair of coordinates to be re-parseable, so the representative + // value is [1,1]. + bob.appendArray(fieldName, BSON_ARRAY(1 << 1)); + return; + } + case GeoParser::GEOJSON_MULTI_POINT: { + // MultiPoint requires a pair of coordinates wrapped in an array to be re-parseable, so + // the representative value is [[1,1]]. + bob.appendArray(fieldName, BSON_ARRAY(BSON_ARRAY(1 << 1))); + return; + } + case GeoParser::GEOJSON_LINESTRING: { + // LineString requires two pairs of coordinates to be re-parseable, so the + // representative value is [[0,0],[1,1]]. + bob.appendArray(fieldName, BSON_ARRAY(BSON_ARRAY(0 << 0) << BSON_ARRAY(1 << 1))); + return; + } + case GeoParser::GEOJSON_MULTI_LINESTRING: { + // MultiLineString requires two LineStrings wrapped in an array to be re-parseable, so + // the representative value is [[[0,0],[1,1]],[[0,0],[1,1]]]. + bob.appendArray(fieldName, + BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(0 << 0) << BSON_ARRAY(1 << 1)) + << BSON_ARRAY(BSON_ARRAY(0 << 0) << BSON_ARRAY(1 << 1)))); + return; + } + case GeoParser::GEOJSON_GEOMETRY_COLLECTION: + opts.appendLiteral(&bob, coordinatesElem); + return; + case GeoParser::GEOJSON_UNKNOWN: + break; + } + + tasserted(8456600, + str::stream() << "unexpected geo type found in coordinates serialization: " + << geoType); +} + +void appendCRSObject(BSONObjBuilder& bob, + const BSONElement& crsObj, + const SerializationOptions& opts) { + // 'crs' is always an object. + tassert(7559700, "Expected 'crs' to be an object", crsObj.type() == BSONType::Object); + // 'crs' is required to have a 'type' field with the value 'name'. + // Additionally, it is required to have an object properties field + // with a single 'name' field. + tassert(7559701, + str::stream() << "Expected 'crs' to contain a string 'type' field, got " << crsObj, + crsObj["type"] && crsObj["type"].type() == BSONType::String); + tassert(7559702, + str::stream() << "Expected 'crs' to contain a 'properties' object, got , " << crsObj, + crsObj["properties"] && crsObj["properties"].type() == BSONType::Object); + tassert(7559703, + str::stream() << "Expected 'crs.properties' to contain a 'name' " + "string field, got " + << crsObj["properties"], + crsObj["properties"].Obj()["name"] && + crsObj["properties"].Obj()["name"].type() == BSONType::String); + + // The CRS "type" and "properties.name" fields must be preserved for + // kToRepresentativeParseableValue serialization policy so the query + // shape can be re-parsed (and will be preserved for kUnchanged policy + // as well). + BSONObjBuilder crsObjBuilder(bob.subobjStart("crs")); + if (opts.literalPolicy == LiteralSerializationPolicy::kToDebugTypeString) { + opts.appendLiteral(&crsObjBuilder, crsObj["type"]); + } else { + crsObjBuilder.append(crsObj["type"]); + } + BSONObjBuilder crsPropBuilder(crsObjBuilder.subobjStart("properties")); + if (opts.literalPolicy == LiteralSerializationPolicy::kToDebugTypeString) { + opts.appendLiteral(&crsPropBuilder, crsObj["properties"].Obj()["name"]); + } else { + crsPropBuilder.append(crsObj["properties"].Obj()["name"]); + } + crsPropBuilder.doneFast(); + crsObjBuilder.doneFast(); +} + +// This function maps closely to GeoParser::parseFromGeoJSON. We serialize a GeoJSON object that +// should have a type, coordinates, etc. This format may have been used as the RHS for a $geometry +// obj, or implicitly as the RHS of a $geoNear. +void appendGeoJSONObj(BSONObjBuilder& bob, + const BSONObj& geometryObj, + const SerializationOptions& opts) { + auto typeElem = geometryObj["type"]; + if (typeElem) { + bob.append(typeElem); + } + if (auto coordinatesElem = geometryObj["coordinates"]) { + appendGeoJSONCoordinatesLiteral(bob, coordinatesElem, typeElem, opts); + } else if (auto geometriesElem = geometryObj[GEOJSON_GEOMETRIES]) { + // We have a collection of geometries rather than a single one. Recursively + // serialize them and add to the output object. + BSONArrayBuilder geometriesArrBuilder; + for (const auto& geometry : geometriesElem.Array()) { + BSONObjBuilder geometryBuilder; + appendGeoJSONObj(geometryBuilder, geometry.Obj(), opts); + geometriesArrBuilder.append(geometryBuilder.obj()); + } + bob.append(GEOJSON_GEOMETRIES, geometriesArrBuilder.arr()); + } + + // 'crs' can be present if users want to use STRICT_SPHERE coordinate + // system. + if (auto crsElt = geometryObj["crs"]) { + appendCRSObject(bob, crsElt, opts); + } +} + +/* appendGeometryOperator() implements the serialization of a $geometry, which could be either an + * Array or Object. This code is closely tied to GeometryContainer parsing. During GeometryContainer + * parsing, if the $geometry element is an array type, or if the first element of the object is a + * number, we parse the element as a Point. Parsing a legacy point does not enforce specific field + * names, the only constraint is that the coordinates be numeric. So you can essentially have a + * $geometry: {"foo":1, "banana": 2} and it will interpret is as a point with x:1, y: 2. + */ +void appendGeometryOperator(BSONObjBuilder& bob, + const BSONElement& geometryElem, + const SerializationOptions& opts) { + if (geometryElem.type() == BSONType::Array) { + // This would be like {$geometry: [0, 0]} which must be a point. + auto asArray = geometryElem.Array(); + tassert(7539807, + "Expected the point to have exactly 2 elements: an x and y.", + asArray.size() == 2UL); + bob.appendArray( + "$geometry"_sd, + BSON_ARRAY(opts.serializeLiteral(asArray[0]) << opts.serializeLiteral(asArray[1]))); + } else { + const auto& geometryObj = geometryElem.Obj(); + BSONObjBuilder nestedSubObj = bob.subobjStart("$geometry"_sd); + if (geometryObj.firstElement().isNumber()) { + BSONElement x, y; + auto status = GeoParser::parseFlatPointCoordinates(geometryElem, x, y); + tassert(8548500, + "Expected the point to have exactly 2 elements: an x and y.", + status.isOK()); + opts.appendLiteral(&nestedSubObj, x); + opts.appendLiteral(&nestedSubObj, y); + } else { + appendGeoJSONObj(nestedSubObj, geometryObj, opts); + } + nestedSubObj.doneFast(); + } +} + +/** + * Typically, geoNear expressions have a single embedded object under the top-level + * geoNear operator, but there is an exception for syntax that allows geoJSON + * coordinates without specifying $geometry (e.g., {$nearSphere: {type: 'Point', + * coordinates: [1,2]}}). We're iterating outer_it and appending extra literals to + * handle $minDistance and $maxDistance fields that could be included outside the + * primary geo object in those edge cases (e.g., {$nearSphere: {type: 'Point', + * coordinates: [1,2]}, $minDistance: 10}). + */ +void appendGeoNearOperator(BSONObjBuilder& bob, + StringData fieldName, + const BSONElement& geoNearElem, + const SerializationOptions& opts) { + if (geoNearElem.type() == mongo::Array) { + appendGeoNearLegacyArray(bob, geoNearElem, opts); + } else { + BSONObjBuilder subObj = BSONObjBuilder(bob.subobjStart(fieldName)); + auto geoNearObj = geoNearElem.Obj(); + BSONObjIterator embedded_it(geoNearObj); + tassert(8548501, "Expected non-empty geometry object.", embedded_it.more()); + + // If the first element of the embedded object is numeric, we could be dealing + // with a legacy style embedded coordinate pair. + if (geoNearObj.firstElement().isNumber()) { + BSONElement x, y; + auto status = GeoParser::parseFlatPointCoordinates(geoNearObj.firstElement(), x, y); + // If we successfully parsed a legacy flat point as an embedded object, we + // can return as we are done parsing the geometry. + if (status.isOK()) { + opts.appendLiteral(&subObj, x); + opts.appendLiteral(&subObj, y); + subObj.doneFast(); + return; + } + } + // We couldn't parse a legacy coordinate pair expressed as an embedded object, so we + // enumerate the embedded geometry obj and parse the internals. Typically we expect to find + // $geometry, a GeoJSONPoint or $minDistance/$maxDistance/$uniqueDocs. + while (embedded_it.more()) { + auto embeddedElem = embedded_it.next(); + auto embeddedFieldName = embeddedElem.fieldNameStringData(); + if (embeddedFieldName == "$geometry"_sd) { + appendGeometryOperator(subObj, embeddedElem, opts); + } else if (embeddedFieldName == "crs" || embeddedFieldName == "type"_sd || + embeddedFieldName == "coordinates"_sd) { + appendGeoJSONObj(subObj, geoNearObj, opts); + } else { + // $minDistance/$maxDistance/$uniqueDocs. + opts.appendLiteral(&subObj, embeddedElem); + } + } + subObj.doneFast(); + } +} +} // namespace + +/** + * geoNearExpressionCustomSerialization implements the serialization of geoNear queries (i.e $near, + * $geoNear, $nearSphere). GeoNear queries can be broken down into two styles of queries: + * 1. Legacy Style (see GeoNearExpression::parseLegacyQuery()): + * t.find({ loc : { $nearSphere: [0,0], $minDistance: 1, $maxDistance: 3 }}) + * t.find({ loc : { $nearSphere: [0,0] }}) + * t.find({ loc : { $near : [0, 0, 1] } }); + * t.find({ loc : { $near: { someGeoJSONPoint}}) + * t.find({ loc : { $geoNear: { someGeoJSONPoint}}) + * 2. New style queries: + * t.find({loc: {$geoNear : { $geometry: {someGeoJSONPoint}, $maxDistance:3 }}}) + */ +void geoNearExpressionCustomSerialization(BSONObjBuilder& bob, + const BSONObj& obj, + const SerializationOptions& opts, + bool includePath) { + BSONObjIterator outer_it(obj); + while (outer_it.more()) { + auto elem = outer_it.next(); + if (elem.isABSONObj()) { + StringData fieldName = elem.fieldNameStringData(); + if (fieldName == "$near"_sd || fieldName == "$geoNear"_sd || + fieldName == "$nearSphere"_sd) { + appendGeoNearOperator(bob, fieldName, elem, opts); + } else if (fieldName == "$geometry"_sd) { + appendGeometryOperator(bob, elem, opts); + } else { + opts.appendLiteral(&bob, elem); + } + } else { + // Here we only expect to see $minDistance/$maxDistance/$uniqueDocs + opts.appendLiteral(&bob, elem); + } + } +} + +/** + * geoExpressionCustomSerialization() implements the serialization of geoExpressions ($within, + * $geoWithin, $geoIntersects). Examples of such expressions are: + * { $geoWithin : { $geometry : <GeoJSON> } } + * { $geoIntersects : { $geometry : <GeoJSON> } } + * { $geoWithin : { $box : [[x1, y1], [x2, y2]] } } + * { $geoWithin : { $polygon : [[x1, y1], [x1, y2], [x2, y2], [x2, y1]] } } + * { $geoWithin : { $center : [[x1, y1], r], } } + * { $geoWithin : { $centerSphere : [[x, y], radius] } } + * { $geoIntersects : { $geometry : [1, 2] } } + */ +void geoExpressionCustomSerialization(BSONObjBuilder& bob, + const BSONObj& obj, + const SerializationOptions& opts, + bool includePath) { + BSONObjIterator outerIt(obj); + BSONElement geoExprElem = outerIt.next(); + tassert(8548502, "Invalid extra fields in geo expression.", !outerIt.more()); + tassert(8548503, "Geo expression must be an object.", geoExprElem.type() == mongo::Object); + auto fieldName = geoExprElem.fieldNameStringData(); + + BSONObjBuilder subObj = BSONObjBuilder(bob.subobjStart(fieldName)); + auto geoObj = geoExprElem.Obj(); + BSONObjIterator embedded_it(geoObj); + while (embedded_it.more()) { + auto elem = embedded_it.next(); + fieldName = elem.fieldNameStringData(); + // $geoWithin/$geoIntersects can only have a $geometry or shape operators as operands + // (i.e $polygon, $box, etc.) + if (fieldName == "$geometry"_sd) { + appendGeometryOperator(subObj, elem, opts); + break; + } else { + appendShapeOperator(subObj, elem, opts); + } + } +} +} // namespace mongo diff --git a/src/mongo/db/matcher/expression_geo_serializer.h b/src/mongo/db/matcher/expression_geo_serializer.h new file mode 100644 index 00000000000..aac11318466 --- /dev/null +++ b/src/mongo/db/matcher/expression_geo_serializer.h @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/bson/bsonelement.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/db/query/query_shape/serialization_options.h" + +namespace mongo { + +/** + * We rely on these custom serializers for geo expressions to handle serialization with + * kToRepresentativeParseableValue and kToDebugTypeString policies since the original raw query + * needs to be re-parsed in order to properly serialize. + * + * Ideally each sub operator ($minDistance, $maxDistance, $geometry, $box) would serialize itself, + * rather than GeoExpression reparse the query during serialization. However, GeoExpression and + * GeoNearExpression don't capture the nesting of the various sub-operators. Re-parsing is therefore + * required to serialize GeoMatchExpression and GeoNearMatchExpression into BSON representative of + * the correct original query. + * + * To further complicate the serialization, serializing with policy + * kToRepresentativeParseableValue requires output that can again be + * re-parsed, and the geoparser performs validation checking to make sure input coordinates apply to + * the correct geo type. For example, a GeoJSON Polygon must have minimum four pairs of coordinates + * in a closed loop. The default representative parseable array value used in const + * SerializationOptions (an empty array) is not useful here since it won't pass geo validation + * checks. As a workaround, this custom serializer determines a parseable value for each shape or + * point type. + */ +void geoNearExpressionCustomSerialization(BSONObjBuilder& bob, + const BSONObj& obj, + const SerializationOptions& opts = {}, + bool includePath = true); + +void geoExpressionCustomSerialization(BSONObjBuilder& bob, + const BSONObj& obj, + const SerializationOptions& opts = {}, + bool includePath = true); +} // namespace mongo diff --git a/src/mongo/db/matcher/expression_geo_test.cpp b/src/mongo/db/matcher/expression_geo_test.cpp index 4115285de18..4c2d81cbfa8 100644 --- a/src/mongo/db/matcher/expression_geo_test.cpp +++ b/src/mongo/db/matcher/expression_geo_test.cpp @@ -33,11 +33,13 @@ #include <memory> +#include "mongo/db/exec/document_value/document_value_test_util.h" #include "mongo/db/jsobj.h" #include "mongo/db/json.h" #include "mongo/db/matcher/expression.h" #include "mongo/db/matcher/expression_geo.h" + namespace mongo { TEST(ExpressionGeoTest, Geo1) { @@ -153,6 +155,143 @@ TEST(ExpressionGeoTest, GeoNearEquivalent) { } } + +TEST(ExpressionGeoTest, SerializeGeoExpressions) { + SerializationOptions opts = {}; + opts.transformIdentifiers = true; + opts.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + { + BSONObj query = fromjson("{$within: {$box: [{x: 4, y: 4}, [6, 6]]}}"); + std::unique_ptr<GeoMatchExpression> ge(makeGeoMatchExpression(query)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$within":{"$box":"?array<>"}})", + ge->getSerializedRightHandSide(opts)); + } + { + BSONObj query = fromjson( + "{$geoWithin: {$geometry: {type: \"MultiPolygon\", coordinates: [[[[20.0, 70.0],[30.0, " + "70.0],[30.0, 50.0],[20.0, 50.0],[20.0, 70.0]]]]}}}"); + std::unique_ptr<GeoMatchExpression> ge(makeGeoMatchExpression(query)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$geoWithin": { + "$geometry": { + "type": "MultiPolygon", + "coordinates": "?array<?array>" + } + } + })", + ge->getSerializedRightHandSide(opts)); + } + { + BSONObj query = fromjson( + R"({ + "$geoIntersects": { + "$geometry": { + "type": "MultiPolygon", + "coordinates": [[[ + [-20.0, -70.0], + [-30.0, -70.0], + [-30.0, -50.0], + [-20.0, -50.0], + [-20.0, -70.0] + ]]] + } + } + })"); + std::unique_ptr<GeoMatchExpression> ge(makeGeoMatchExpression(query)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$geoIntersects": { + "$geometry": { + "type": "MultiPolygon", + "coordinates": "?array<?array>" + } + } + })", + ge->getSerializedRightHandSide(opts)); + } + { + BSONObj query1 = fromjson( + R"({$within: { + $geometry: { + type: 'Polygon', + coordinates: [[[0, 0], [3, 6], [6, 1], [0, 0]]] + } + }})"); + std::unique_ptr<GeoMatchExpression> ge(makeGeoMatchExpression(query1)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$within":{"$geometry":{"type":"Polygon","coordinates":"?array<?array>"}}})", + ge->getSerializedRightHandSide(opts)); + } + { + BSONObj query = fromjson( + "{$near: {$maxDistance: 100, " + "$geometry: {type: 'Point', coordinates: [0, 0]}}}"); + std::unique_ptr<GeoNearMatchExpression> gne(makeGeoNearMatchExpression(query)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$near": { + "$maxDistance": "?number", + "$geometry": { + "type": "Point", + "coordinates": "?array<?number>" + } + } + })", + gne->getSerializedRightHandSide(opts)); + } + { + BSONObj query = fromjson("{ $nearSphere: [0,0], $minDistance: 1, $maxDistance: 3 }"); + std::unique_ptr<GeoNearMatchExpression> gne(makeGeoNearMatchExpression(query)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$nearSphere": "?array<?number>", + "$minDistance": "?number", + "$maxDistance": "?number" + })", + gne->getSerializedRightHandSide(opts)); + } + + { + BSONObj query = fromjson("{$near : [0, 0, 1] }"); + std::unique_ptr<GeoNearMatchExpression> gne(makeGeoNearMatchExpression(query)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$near":"?array<?number>"})", + gne->getSerializedRightHandSide(opts)); + } + { + BSONObj query = fromjson("{$geoNear: [0, 0, 100]}"); + std::unique_ptr<GeoNearMatchExpression> gne(makeGeoNearMatchExpression(query)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$geoNear":"?array<?number>"})", + gne->getSerializedRightHandSide(opts)); + } + { + BSONObj query = fromjson("{$geoNear: [0, 10], $maxDistance: 80 }"); + std::unique_ptr<GeoNearMatchExpression> gne(makeGeoNearMatchExpression(query)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$geoNear":"?array<?number>","$maxDistance":"?number"})", + gne->getSerializedRightHandSide(opts)); + } + { + BSONObj query = fromjson("{$geoIntersects: {$geometry: [0, 0]}}"); + std::unique_ptr<GeoMatchExpression> ge(makeGeoMatchExpression(query)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$geoIntersects":{"$geometry":["?number","?number"]}})", + ge->getSerializedRightHandSide(opts)); + } + { + // Make sure we reject arrays with <2 or >2 elements. + BSONObj query = fromjson("{$geoIntersects: {$geometry: [0, 0, 1]}}"); + std::unique_ptr<GeoExpression> gq(new GeoExpression); + ASSERT_NOT_OK(gq->parseFrom(query)); + query = fromjson("{$geoIntersects: {$geometry: [0]}}"); + ASSERT_NOT_OK(gq->parseFrom(query)); + } +} + /** * A geo expression being not equivalent to another expression. */ @@ -182,4 +321,280 @@ TEST(ExpressionGeoTest, GeoNearNotEquivalent) { gne2(makeGeoNearMatchExpression(query2)); ASSERT(!gne1->equivalent(gne2.get())); } + +TEST(ExpressionGeoTest, SerializeWithCRSIFSpecifiedWithChangedOptions) { + BSONObj query1 = fromjson( + "{$within: {$geometry: {type: 'Polygon'," + "coordinates: [[[0, 0], [3, 6], [6, 1], [0, 0]]]," + "crs: {" + "type: 'name'," + "properties: { name: 'urn:x-mongodb:crs:strictwinding:EPSG:4326' }" + "}}}}"); + std::unique_ptr<GeoMatchExpression> ge1(makeGeoMatchExpression(query1)); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + auto serialized = ge1->getSerializedRightHandSide(opts); + ASSERT_BSONOBJ_EQ_AUTO( + R"({ + "$within": { + "$geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 0, + 0 + ], + [ + 0, + 1 + ], + [ + 1, + 1 + ], + [ + 0, + 0 + ] + ] + ], + "crs": { + "type": "name", + "properties": { + "name": "urn:x-mongodb:crs:strictwinding:EPSG:4326" + } + } + } + } + })", + serialized); + serialized = ge1->getSerializedRightHandSide(opts); + ASSERT_BSONOBJ_EQ_AUTO( + R"({ + "$within": { + "$geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 0, + 0 + ], + [ + 0, + 1 + ], + [ + 1, + 1 + ], + [ + 0, + 0 + ] + ] + ], + "crs": { + "type": "name", + "properties": { + "name": "urn:x-mongodb:crs:strictwinding:EPSG:4326" + } + } + } + } + })", + serialized); +} + +template <typename CreateFn> +void assertRepresentativeShapeIsStable(BSONObj inputExpr, + BSONObj expectedRepresentativeExpr, + CreateFn createFn) { + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + auto ge(createFn(inputExpr)); + + auto serializedExpr = ge->getSerializedRightHandSide(opts); + ASSERT_BSONOBJ_EQ(serializedExpr, expectedRepresentativeExpr); + + auto roundTripped = createFn(serializedExpr); + ASSERT_BSONOBJ_EQ(roundTripped->getSerializedRightHandSide(opts), serializedExpr); +} + +void assertRepresentativeGeoShapeIsStable(BSONObj inputExpr, BSONObj expectedRepresentativeExpr) { + assertRepresentativeShapeIsStable( + inputExpr, expectedRepresentativeExpr, [](const BSONObj& input) { + return makeGeoMatchExpression(input); + }); +} + +void assertRepresentativeGeoNearShapeIsStable(BSONObj inputExpr, + BSONObj expectedRepresentativeExpr) { + assertRepresentativeShapeIsStable( + inputExpr, expectedRepresentativeExpr, [](const BSONObj& input) { + return makeGeoNearMatchExpression(input); + }); +} + +TEST(ExpressionGeoTest, RoundTripSerializeGeoExpressions) { + assertRepresentativeGeoShapeIsStable(fromjson("{$within: {$box: [{x: 4, y: 4}, [6, 6]]}}"), + fromjson("{$within: {$box: [[1, 1],[1, 1]]}}")); + + assertRepresentativeGeoShapeIsStable( + fromjson( + R"({$geoWithin: {$geometry: {type: "MultiPolygon", coordinates: [[[[20.0, 70.0],[30.0, 70.0],[30.0, 50.0],[20.0, 50.0],[20.0, 70.0]]]]}}})"), + fromjson( + R"({$geoWithin: {$geometry: {type: "MultiPolygon", coordinates: [[[[0, 0],[0, 1],[1, 1],[0, 0]]]]}}})")); + + assertRepresentativeGeoShapeIsStable(fromjson(R"({ + "$geoIntersects": { + "$geometry": { + "type": "MultiPolygon", + "coordinates": [[[ + [-20.0, -70.0], + [-30.0, -70.0], + [-30.0, -50.0], + [-20.0, -50.0], + [-20.0, -70.0] + ]]] + } + } + })"), + fromjson(R"({ + "$geoIntersects": { + "$geometry": { + "type": "MultiPolygon", + "coordinates": [[[[0, 0],[0, 1],[1, 1],[0, 0]]]] + } + } + })")); + + assertRepresentativeGeoShapeIsStable(fromjson(R"({$within: { + $geometry: { + type: 'Polygon', + coordinates: [[[0, 0], [3, 6], [6, 1], [0, 0]]] + } + }})"), + fromjson(R"({$within: { + $geometry: { + type: 'Polygon', + coordinates: [[[0, 0],[0, 1],[1, 1],[0, 0]]] + } + }})")); + + assertRepresentativeGeoNearShapeIsStable( + fromjson("{$near: {$maxDistance: 100, $geometry: {type: 'Point', coordinates: [0, 0]}}}"), + fromjson("{$near: {$maxDistance: 1, $geometry: {type: 'Point', coordinates: [1, 1]}}}")); + + assertRepresentativeGeoNearShapeIsStable( + fromjson("{$nearSphere: [0,0], $minDistance: 2, $maxDistance: 4 }"), + fromjson("{$nearSphere: [1,1], $minDistance: 1, $maxDistance: 1 }")); + + assertRepresentativeGeoNearShapeIsStable( + fromjson("{$minDistance: 2, $maxDistance: 4, $nearSphere: [0,0]}"), + fromjson("{$minDistance: 1, $maxDistance: 1, $nearSphere: [1,1]}")); + + assertRepresentativeGeoNearShapeIsStable(fromjson("{$near: [0, 0, 1]}"), + fromjson("{$near: [1, 1]}")); + + assertRepresentativeGeoNearShapeIsStable(fromjson("{$geoNear: [0, 0, 100]}"), + fromjson("{$geoNear: [1, 1]}")); + + assertRepresentativeGeoNearShapeIsStable(fromjson("{$geoNear: [0, 10], $maxDistance: 80 }"), + fromjson("{$geoNear: [1, 1], $maxDistance: 1}")); + + assertRepresentativeGeoShapeIsStable(fromjson("{$geoIntersects: {$geometry: [0, 0]}}"), + fromjson("{$geoIntersects: {$geometry: [1, 1]}}")); + // Test scenario with new $geometry query not specifying the geometry type. + assertRepresentativeGeoNearShapeIsStable( + fromjson("{$geoNear: { $geometry: {coordinates: [0, 10]}}}"), + fromjson("{$geoNear: { $geometry: {coordinates: [1, 1]}}}")); + + // Test scenario with new $geometry query specifying invalid type. + assertRepresentativeGeoNearShapeIsStable( + fromjson("{$geoNear: { $geometry: { type: 'b.c', coordinates: [0, 10]}}}"), + fromjson("{$geoNear: { $geometry: {type: 'b.c', coordinates: [1, 1]}}}")); + + // Test scenario with $nearSphere without $geometry and no type specified + assertRepresentativeGeoNearShapeIsStable(fromjson(R"({"$nearSphere":{"coordinates":[0,0]}})"), + fromjson(R"({"$nearSphere":{"coordinates":[1,1]}})")); + + // Test case with first field of $geometry as numeric field, arbitrary coordinate naming. + assertRepresentativeGeoShapeIsStable( + fromjson(R"({"$geoIntersects":{"$geometry":{"shardOptions":40,"y":5}}})"), + fromjson(R"({"$geoIntersects":{"$geometry":{"shardOptions":1,"y":1}}})")); + + assertRepresentativeGeoShapeIsStable(fromjson(R"({ + "$geoIntersects": { + "$geometry": { + "type": "MultiLineString", + "coordinates": [[ + [2, 0], + [2, 2] + ], [ + [0, 4], + [1, 4] + ]] + } + } + })"), + fromjson(R"({ + "$geoIntersects": { + "$geometry": { + "type": "MultiLineString", + "coordinates": [[[0, 0], [1, 1]],[[0, 0], [1, 1]]] + } + } + })")); + + assertRepresentativeGeoShapeIsStable(fromjson(R"({ + "$geoIntersects": { + "$geometry": { + "type": "LineString", + "coordinates": [ + [2, 0], + [2, 2] + ] + } + } + })"), + fromjson(R"({ + "$geoIntersects": { + "$geometry": { + "type": "LineString", + "coordinates": [[0, 0], [1, 1]] + } + } + })")); + + assertRepresentativeGeoShapeIsStable(fromjson(R"({ + "$geoIntersects": { + "$geometry": { + "type": "GeometryCollection", + "geometries": [{ + "type": "LineString", + "coordinates": [ + [2, 0], + [2, 2] + ] + }, { + type: 'Point', coordinates: [2, 2] + }] + } + } + })"), + fromjson(R"({ + "$geoIntersects": { + "$geometry": { + "type": "GeometryCollection", + "geometries": [{ + "type": "LineString", + "coordinates": [[0, 0], [1, 1]] + }, { + type: 'Point', coordinates: [1, 1] + }] + } + } + })")); +} + } // namespace mongo diff --git a/src/mongo/db/matcher/expression_internal_bucket_geo_within.cpp b/src/mongo/db/matcher/expression_internal_bucket_geo_within.cpp index 5bf75f8218f..e09f142a770 100644 --- a/src/mongo/db/matcher/expression_internal_bucket_geo_within.cpp +++ b/src/mongo/db/matcher/expression_internal_bucket_geo_within.cpp @@ -51,7 +51,7 @@ void InternalBucketGeoWithinMatchExpression::debugString(StringBuilder& debug, _debugAddSpace(debug, indentationLevel); BSONObjBuilder builder; - serialize(&builder, true); + serialize(&builder, {}); debug << builder.obj().toString() << "\n"; const auto* tag = getTag(); @@ -194,13 +194,17 @@ bool InternalBucketGeoWithinMatchExpression::_matchesBSONObj(const BSONObj& obj) } void InternalBucketGeoWithinMatchExpression::serialize(BSONObjBuilder* builder, + const SerializationOptions& opts, bool includePath) const { BSONObjBuilder bob(builder->subobjStart(InternalBucketGeoWithinMatchExpression::kName)); + // Serialize the geometry shape. BSONObjBuilder withinRegionBob( bob.subobjStart(InternalBucketGeoWithinMatchExpression::kWithinRegion)); - withinRegionBob.append(_geoContainer->getGeoElement()); + opts.appendLiteral(&withinRegionBob, _geoContainer->getGeoElement()); withinRegionBob.doneFast(); - bob.append(InternalBucketGeoWithinMatchExpression::kField, _field); + // Serialize the field which is being searched over. + bob.append(InternalBucketGeoWithinMatchExpression::kField, + opts.serializeFieldPathFromString(_field)); bob.doneFast(); } diff --git a/src/mongo/db/matcher/expression_internal_bucket_geo_within.h b/src/mongo/db/matcher/expression_internal_bucket_geo_within.h index d2a6202744f..0749e33adb4 100644 --- a/src/mongo/db/matcher/expression_internal_bucket_geo_within.h +++ b/src/mongo/db/matcher/expression_internal_bucket_geo_within.h @@ -95,7 +95,9 @@ public: return false; } - void serialize(BSONObjBuilder* builder, bool includePath) const final; + void serialize(BSONObjBuilder* builder, + const SerializationOptions& opts = {}, + bool includePath = true) const final; std::unique_ptr<MatchExpression> shallowClone() const final; diff --git a/src/mongo/db/matcher/expression_internal_expr_eq_test.cpp b/src/mongo/db/matcher/expression_internal_expr_eq_test.cpp index 0de1fbec00b..facc1f9f7a4 100644 --- a/src/mongo/db/matcher/expression_internal_expr_eq_test.cpp +++ b/src/mongo/db/matcher/expression_internal_expr_eq_test.cpp @@ -264,7 +264,7 @@ TEST(InternalExprEqMatchExpression, SerializesCorrectly) { operand.firstElement()); BSONObjBuilder bob; - eq.serialize(&bob, true); + eq.serialize(&bob, {}); ASSERT_BSONOBJ_EQ(BSON("x" << BSON("$_internalExprEq" << 5)), bob.obj()); } diff --git a/src/mongo/db/matcher/expression_leaf.cpp b/src/mongo/db/matcher/expression_leaf.cpp index ae5b3da8834..721b93c8da4 100644 --- a/src/mongo/db/matcher/expression_leaf.cpp +++ b/src/mongo/db/matcher/expression_leaf.cpp @@ -39,6 +39,7 @@ #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/config.h" +#include "mongo/db/exec/document_value/value.h" #include "mongo/db/field_ref.h" #include "mongo/db/jsobj.h" #include "mongo/db/matcher/expression_parser.h" @@ -93,8 +94,10 @@ void ComparisonMatchExpressionBase::debugString(StringBuilder& debug, int indent debug << "\n"; } -BSONObj ComparisonMatchExpressionBase::getSerializedRightHandSide() const { - return BSON(name() << _rhs); +void ComparisonMatchExpressionBase::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + opts.appendLiteral(bob, name(), _rhs); } ComparisonMatchExpression::ComparisonMatchExpression(MatchType type, @@ -290,15 +293,19 @@ void RegexMatchExpression::debugString(StringBuilder& debug, int indentationLeve debug << "\n"; } -BSONObj RegexMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder regexBuilder; - regexBuilder.append("$regex", _regex); +void RegexMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + // We need to be careful to generate a valid regex representative value, and the default string + // "?" is not valid. + opts.appendLiteral(bob, "$regex", _regex, Value("\\?"_sd)); if (!_flags.empty()) { - regexBuilder.append("$options", _flags); + // We need to make sure the $options value can be re-parsed as legal regex options, so + // we'll set the representative value in this case to be the string "i" rather than + // "?", which is the standard representative for string values. + opts.appendLiteral(bob, "$options", _flags, Value("i"_sd)); } - - return regexBuilder.obj(); } void RegexMatchExpression::serializeToBSONTypeRegex(BSONObjBuilder* out) const { @@ -371,8 +378,11 @@ void ModMatchExpression::debugString(StringBuilder& debug, int indentationLevel) debug << "\n"; } -BSONObj ModMatchExpression::getSerializedRightHandSide() const { - return BSON("$mod" << BSON_ARRAY(_divisor << _remainder)); +void ModMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + bob->append("$mod", + BSON_ARRAY(opts.serializeLiteral(_divisor) << opts.serializeLiteral(_remainder))); } bool ModMatchExpression::equivalent(const MatchExpression* other) const { @@ -407,8 +417,10 @@ void ExistsMatchExpression::debugString(StringBuilder& debug, int indentationLev debug << "\n"; } -BSONObj ExistsMatchExpression::getSerializedRightHandSide() const { - return BSON("$exists" << true); +void ExistsMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + opts.appendLiteral(bob, "$exists", true); } bool ExistsMatchExpression::equivalent(const MatchExpression* other) const { @@ -490,9 +502,45 @@ void InMatchExpression::debugString(StringBuilder& debug, int indentationLevel) debug << "\n"; } -BSONObj InMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder inBob; - BSONArrayBuilder arrBob(inBob.subarrayStart("$in")); +namespace { +/** + * Reduces the potentially large vector of elements to just the first of each "canonical" type. + * Different types of numbers are not considered distinct. + * + * For example, collapses [2, 4, NumberInt(3), "string", "another", 3, 5] into just [2, "string"]. + */ +std::vector<Value> justFirstOfEachType(std::vector<BSONElement> elems) { + stdx::unordered_set<int> seenTypes; + std::vector<Value> result; + for (auto&& elem : elems) { + bool inserted = seenTypes.insert(canonicalizeBSONType(elem.type())).second; + if (inserted) { + // A new type. + result.emplace_back(elem); + } + } + return result; +} +} // namespace + +void InMatchExpression::serializeToShape(BSONObjBuilder* bob, + const SerializationOptions& opts) const { + std::vector<Value> firstOfEachType = justFirstOfEachType(_equalitySet); + if (hasRegex()) { + firstOfEachType.emplace_back(BSONRegEx()); + } + opts.appendLiteral(bob, "$in", std::move(firstOfEachType)); +} + +void InMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + serializeToShape(bob, opts); + return; + } + + BSONArrayBuilder arrBob(bob->subarrayStart("$in")); for (auto&& _equality : _equalitySet) { arrBob.append(_equality); } @@ -502,7 +550,6 @@ BSONObj InMatchExpression::getSerializedRightHandSide() const { arrBob.append(regexBob.obj().firstElement()); } arrBob.doneFast(); - return inBob.obj(); } bool InMatchExpression::equivalent(const MatchExpression* other) const { @@ -861,7 +908,9 @@ void BitTestMatchExpression::debugString(StringBuilder& debug, int indentationLe } } -BSONObj BitTestMatchExpression::getSerializedRightHandSide() const { +void BitTestMatchExpression::appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts, + bool includePath) const { std::string opString = ""; switch (matchType()) { @@ -886,8 +935,10 @@ BSONObj BitTestMatchExpression::getSerializedRightHandSide() const { arrBob.append(static_cast<int32_t>(bitPosition)); } arrBob.doneFast(); - - return BSON(opString << arrBob.arr()); + // Unfortunately this cannot be done without copying the array into the BSONObjBuilder, since + // `opts.appendLiteral` may choose to append this actual array, a representative empty array, or + // a debug string. + opts.appendLiteral(bob, opString, arrBob.arr()); } bool BitTestMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/expression_leaf.h b/src/mongo/db/matcher/expression_leaf.h index a00ed9e31f1..5f4f2666a94 100644 --- a/src/mongo/db/matcher/expression_leaf.h +++ b/src/mongo/db/matcher/expression_leaf.h @@ -163,7 +163,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel = 0) const; - BSONObj getSerializedRightHandSide() const final; + virtual void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const; virtual bool equivalent(const MatchExpression* other) const; @@ -506,7 +508,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; void serializeToBSONTypeRegex(BSONObjBuilder* out) const; @@ -586,7 +590,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; virtual bool equivalent(const MatchExpression* other) const; @@ -651,7 +657,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; virtual bool equivalent(const MatchExpression* other) const; @@ -682,7 +690,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; virtual bool equivalent(const MatchExpression* other) const; @@ -750,6 +760,12 @@ public: private: ExpressionOptimizerFunc getOptimizer() const final; + /** + * A helper to serialize to something like {$in: "?array<?number>"} or similar, depending on + * 'opts' and whether we have a mixed-type $in or not. + */ + void serializeToShape(BSONObjBuilder* bob, const SerializationOptions& opts) const; + // Whether or not '_equalities' has a jstNULL element in it. bool _hasNull = false; @@ -818,7 +834,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel) const; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; virtual bool equivalent(const MatchExpression* other) const; diff --git a/src/mongo/db/matcher/expression_optimize_test.cpp b/src/mongo/db/matcher/expression_optimize_test.cpp index 8d62f53baa1..763d80fa2cb 100644 --- a/src/mongo/db/matcher/expression_optimize_test.cpp +++ b/src/mongo/db/matcher/expression_optimize_test.cpp @@ -75,7 +75,8 @@ Status isValid(const std::string& queryStr, const FindCommandRequest& findComman BSONObj queryObj = fromjson(queryStr); std::unique_ptr<MatchExpression> me(parseMatchExpression(queryObj)); me = MatchExpression::optimize(std::move(me)); - if (auto status = CanonicalQuery::isValid(me.get(), findCommand).getStatus(); !status.isOK()) { + if (auto status = parsed_find_command::isValid(me.get(), findCommand).getStatus(); + !status.isOK()) { return status; } return CanonicalQuery::isValidNormalized(me.get()); @@ -350,18 +351,14 @@ TEST(ExpressionOptimizeTest, AndWithAlwaysFalseChildOptimizesToAlwaysFalse) { BSONObj obj = fromjson("{$and: [{a: 1}, {$alwaysFalse: 1}]}"); std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{$alwaysFalse: 1}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{$alwaysFalse: 1}")); } TEST(ExpressionOptimizeTest, AndRemovesAlwaysTrueChildren) { BSONObj obj = fromjson("{$and: [{a: 1}, {$alwaysTrue: 1}]}"); std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{a: {$eq: 1}}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{a: {$eq: 1}}")); } TEST(ExpressionOptimizeTest, AndWithSingleChildAlwaysTrueOptimizesToEmptyAnd) { @@ -370,9 +367,7 @@ TEST(ExpressionOptimizeTest, AndWithSingleChildAlwaysTrueOptimizesToEmptyAnd) { auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); // TODO SERVER-34759 We want this to optimize to an AlwaysTrueMatchExpression. ASSERT_TRUE(dynamic_cast<AndMatchExpression*>(optimizedMatchExpression.get())); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{}")); } TEST(ExpressionOptimizeTest, AndWithEachChildAlwaysTrueOptimizesToEmptyAnd) { @@ -381,36 +376,28 @@ TEST(ExpressionOptimizeTest, AndWithEachChildAlwaysTrueOptimizesToEmptyAnd) { auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); // TODO SERVER-34759 We want this to optimize to an AlwaysTrueMatchExpression. ASSERT_TRUE(dynamic_cast<AndMatchExpression*>(optimizedMatchExpression.get())); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{}")); } TEST(ExpressionOptimizeTest, NestedAndWithAlwaysFalseOptimizesToAlwaysFalse) { BSONObj obj = fromjson("{$and: [{$and: [{$alwaysFalse: 1}, {a: 1}]}, {b: 1}]}"); std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{$alwaysFalse: 1}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{$alwaysFalse: 1}")); } TEST(ExpressionOptimizeTest, OrWithAlwaysTrueOptimizesToAlwaysTrue) { BSONObj obj = fromjson("{$or: [{a: 1}, {$alwaysTrue: 1}]}"); std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{$alwaysTrue: 1}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{$alwaysTrue: 1}")); } TEST(ExpressionOptimizeTest, OrRemovesAlwaysFalseChildren) { BSONObj obj = fromjson("{$or: [{a: 1}, {$alwaysFalse: 1}]}"); std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{a: {$eq: 1}}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{a: {$eq: 1}}")); } TEST(ExpressionOptimizeTest, OrPromotesSingleAlwaysFalseAfterOptimize) { @@ -419,9 +406,7 @@ TEST(ExpressionOptimizeTest, OrPromotesSingleAlwaysFalseAfterOptimize) { std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); ASSERT_TRUE(dynamic_cast<AlwaysFalseMatchExpression*>(optimizedMatchExpression.get())); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{$alwaysFalse: 1}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{$alwaysFalse: 1}")); } TEST(ExpressionOptimizeTest, OrPromotesSingleAlwaysFalse) { @@ -429,9 +414,7 @@ TEST(ExpressionOptimizeTest, OrPromotesSingleAlwaysFalse) { std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); ASSERT_TRUE(dynamic_cast<AlwaysFalseMatchExpression*>(optimizedMatchExpression.get())); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{$alwaysFalse: 1}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{$alwaysFalse: 1}")); } TEST(ExpressionOptimizeTest, OrPromotesMultipleAlwaysFalse) { @@ -439,18 +422,14 @@ TEST(ExpressionOptimizeTest, OrPromotesMultipleAlwaysFalse) { std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); ASSERT_TRUE(dynamic_cast<AlwaysFalseMatchExpression*>(optimizedMatchExpression.get())); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{$alwaysFalse: 1}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{$alwaysFalse: 1}")); } TEST(ExpressionOptimizeTest, NestedOrWithAlwaysTrueOptimizesToAlwaysTrue) { BSONObj obj = fromjson("{$or: [{$or: [{$alwaysTrue: 1}, {a: 1}]}, {b: 1}]}"); std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - ASSERT_BSONOBJ_EQ(bob.obj(), fromjson("{$alwaysTrue: 1}")); + ASSERT_BSONOBJ_EQ(optimizedMatchExpression->serialize(), fromjson("{$alwaysTrue: 1}")); } TEST(ExpressionOptimizeTest, OrRewrittenToIn) { @@ -477,9 +456,7 @@ TEST(ExpressionOptimizeTest, OrRewrittenToIn) { auto obj = fromjson(exprStr); std::unique_ptr<MatchExpression> matchExpression(parseMatchExpression(obj)); auto optimizedMatchExpression = MatchExpression::optimize(std::move(matchExpression)); - BSONObjBuilder bob; - optimizedMatchExpression->serialize(&bob, true); - return bob.obj(); + return optimizedMatchExpression->serialize(); }; ASSERT_BSONOBJ_EQ(optimizeExpr(queries[0].first), fromjson(queries[0].second)); diff --git a/src/mongo/db/matcher/expression_path.h b/src/mongo/db/matcher/expression_path.h index 16182705a62..f167177bdb4 100644 --- a/src/mongo/db/matcher/expression_path.h +++ b/src/mongo/db/matcher/expression_path.h @@ -122,21 +122,39 @@ public: } } - void serialize(BSONObjBuilder* out, bool includePath) const override { + void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const override { if (includePath) { - out->append(path(), getSerializedRightHandSide()); + BSONObjBuilder subObj(out->subobjStart(opts.serializeFieldPathFromString(path()))); + appendSerializedRightHandSide(&subObj, opts, includePath); + subObj.doneFast(); } else { - out->appendElements(getSerializedRightHandSide()); + appendSerializedRightHandSide(out, opts, includePath); } } /** - * Returns a BSONObj that represents the right-hand-side of a PathMatchExpression. Used for + * Constructs a BSONObj that represents the right-hand-side of a PathMatchExpression. Used for * serialization of PathMatchExpression in cases where we do not want to serialize the path in * line with the expression. For example {x: {$not: {$eq: 1}}}, where $eq is the * PathMatchExpression. + * + * Serialization options should be respected for any descendent expressions. Eg, if the + * 'literalPolicy' option is 'kToDebugTypeString', then any literal argument (like the number 1 + * in the example above), should be "shapified" (e.g. "?number"). 'literal' here is in contrast + * to another expression, if that is possible syntactically. */ - virtual BSONObj getSerializedRightHandSide() const = 0; + virtual void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const = 0; + + BSONObj getSerializedRightHandSide(const SerializationOptions& opts = {}, + bool includePath = true) const { + BSONObjBuilder bob; + appendSerializedRightHandSide(&bob, opts, includePath); + return bob.obj(); + } protected: void _doAddDependencies(DepsTracker* deps) const final { diff --git a/src/mongo/db/matcher/expression_serialization_test.cpp b/src/mongo/db/matcher/expression_serialization_test.cpp index 78517a52634..3b25631cd0f 100644 --- a/src/mongo/db/matcher/expression_serialization_test.cpp +++ b/src/mongo/db/matcher/expression_serialization_test.cpp @@ -37,7 +37,16 @@ #include "mongo/db/matcher/expression_parser.h" #include "mongo/db/matcher/extensions_callback_noop.h" #include "mongo/db/matcher/matcher.h" +#include "mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index.h" +#include "mongo/db/matcher/schema/expression_internal_schema_cond.h" +#include "mongo/db/matcher/schema/expression_internal_schema_eq.h" +#include "mongo/db/matcher/schema/expression_internal_schema_fmod.h" +#include "mongo/db/matcher/schema/expression_internal_schema_max_items.h" #include "mongo/db/matcher/schema/expression_internal_schema_max_length.h" +#include "mongo/db/matcher/schema/expression_internal_schema_max_properties.h" +#include "mongo/db/matcher/schema/expression_internal_schema_min_items.h" +#include "mongo/db/matcher/schema/expression_internal_schema_min_length.h" +#include "mongo/db/matcher/schema/expression_internal_schema_min_properties.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/unittest/unittest.h" @@ -49,9 +58,7 @@ using std::string; using std::unique_ptr; BSONObj serialize(MatchExpression* match) { - BSONObjBuilder bob; - match->serialize(&bob, true); - return bob.obj(); + return match->serialize(); } TEST(SerializeBasic, AndExpressionWithOneChildSerializesCorrectly) { @@ -355,7 +362,6 @@ TEST(SerializeBasic, ExpressionElemMatchValueWithTripleNotSerializesCorrectly) { ASSERT_EQ(original.matches(obj), reserialized.matches(obj)); } - TEST(SerializeBasic, ExpressionSizeSerializesCorrectly) { boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); Matcher original(fromjson("{x: {$size: 2}}"), @@ -630,6 +636,32 @@ TEST(SerializeBasic, ExpressionRegexWithValueAndOptionsSerializesCorrectly) { ASSERT_EQ(original.matches(obj), reserialized.matches(obj)); } +TEST(SerializeBasic, ExpressionRegexWithoutOptionsSerializesShapeCorrectly) { + auto query = fromjson(R"({x: {$regex: ".*"}})"); + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + auto objMatch = MatchExpressionParser::parse(query, expCtx); + ASSERT_OK(objMatch.getStatus()); + SerializationOptions opts; + opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue; + ASSERT_BSONOBJ_EQ(BSON("x" << BSON("$regex" + << "\\?")), + objMatch.getValue()->serialize(opts)); +} + +TEST(SerializeBasic, ExpressionRegexWithOptionsSerializesShapeCorrectly) { + auto query = fromjson(R"({x: {$regex: ".*", $options: "m"}})"); + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + auto objMatch = MatchExpressionParser::parse(query, expCtx); + ASSERT_OK(objMatch.getStatus()); + SerializationOptions opts; + opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue; + ASSERT_BSONOBJ_EQ(BSON("x" << BSON("$regex" + << "\\?" + << "$options" + << "i")), + objMatch.getValue()->serialize(opts)); +} + TEST(SerializeBasic, ExpressionRegexWithValueSerializesCorrectly) { boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); Matcher original(fromjson("{x: /a.b/}"), @@ -1093,7 +1125,7 @@ TEST(SerializeBasic, ExpressionNotWithDirectPathExpSerializesCorrectly) { // direct path expression child, instead creating a NOT -> AND -> path expression. This test // manually constructs such an expression in case it ever turns up, since that should still be // able to serialize. - auto originalBSON = fromjson("{a: {$not: {$eq: 2}}}}"); + auto originalBSON = fromjson("{a: {$not: {$eq: 2}}}"); auto equalityRHSElem = originalBSON["a"]["$not"]["$eq"]; auto equalityExpression = std::make_unique<EqualityMatchExpression>("a"_sd, equalityRHSElem); @@ -1679,7 +1711,7 @@ TEST(SerializeInternalSchema, ExpressionInternalSchemaMaxLengthSerializesCorrect TEST(SerializeInternalSchema, ExpressionInternalSchemaCondSerializesCorrectly) { boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); - Matcher original(fromjson("{$_internalSchemaCond: [{a: 1}, {b: 2}, {c: 3}]}}"), + Matcher original(fromjson("{$_internalSchemaCond: [{a: 1}, {b: 2}, {c: 3}]}"), expCtx, ExtensionsCallbackNoop(), MatchExpressionParser::kAllowAllSpecialFeatures); @@ -1690,7 +1722,7 @@ TEST(SerializeInternalSchema, ExpressionInternalSchemaCondSerializesCorrectly) { BSONObjBuilder builder; ASSERT_BSONOBJ_EQ( *reserialized.getQuery(), - fromjson("{$_internalSchemaCond: [{a: {$eq: 1}}, {b: {$eq: 2}}, {c: {$eq: 3}}]}}")); + fromjson("{$_internalSchemaCond: [{a: {$eq: 1}}, {b: {$eq: 2}}, {c: {$eq: 3}}]}")); ASSERT_BSONOBJ_EQ(*reserialized.getQuery(), serialize(reserialized.getMatchExpression())); } @@ -1848,5 +1880,311 @@ TEST(SerializeInternalBinDataSubType, ExpressionBinDataSubTypeSerializesCorrectl ASSERT_TRUE(original.matches(obj)); } +TEST(SerializeInternalSchema, AllowedPropertiesRedactsCorrectly) { + + auto query = fromjson( + "{$_internalSchemaAllowedProperties: {properties: ['a', 'b']," + "namePlaceholder: 'i', patternProperties: [], otherwise: {i: 0}}}"); + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + auto objMatch = MatchExpressionParser::parse(query, expCtx); + ASSERT_OK(objMatch.getStatus()); + + SerializationOptions opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSchemaAllowedProperties": { + "properties": "?array<?string>", + "namePlaceholder": "i", + "patternProperties": [], + "otherwise": { + "HASH<i>": { + "$eq": "?number" + } + } + } + })", + objMatch.getValue()->serialize(opts)); +} + +/** + * Helper function for parsing and creating MatchExpressions. + */ +std::unique_ptr<InternalSchemaCondMatchExpression> createCondMatchExpression(BSONObj condition, + BSONObj thenBranch, + BSONObj elseBranch) { + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + auto conditionExpr = MatchExpressionParser::parse(condition, expCtx); + ASSERT_OK(conditionExpr.getStatus()); + auto thenBranchExpr = MatchExpressionParser::parse(thenBranch, expCtx); + ASSERT_OK(thenBranchExpr.getStatus()); + auto elseBranchExpr = MatchExpressionParser::parse(elseBranch, expCtx); + + std::array<std::unique_ptr<MatchExpression>, 3> expressions = { + {std::move(conditionExpr.getValue()), + std::move(thenBranchExpr.getValue()), + std::move(elseBranchExpr.getValue())}}; + + auto cond = std::make_unique<InternalSchemaCondMatchExpression>(std::move(expressions)); + + return cond; +} + +TEST(SerializeInternalSchema, CondMatchRedactsCorrectly) { + auto opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto conditionQuery = BSON("age" << BSON("$lt" << 18)); + auto thenQuery = BSON("job" + << "student"); + auto elseQuery = BSON("job" + << "engineer"); + auto cond = createCondMatchExpression(conditionQuery, thenQuery, elseQuery); + BSONObjBuilder bob; + cond->serialize(&bob, opts); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSchemaCond": [ + { + "HASH<age>": { + "$lt": "?number" + } + }, + { + "HASH<job>": { + "$eq": "?string" + } + }, + { + "HASH<job>": { + "$eq": "?string" + } + } + ] + })", + bob.done()); +} + +TEST(SerializeInternalSchema, FmodMatchRedactsCorrectly) { + InternalSchemaFmodMatchExpression m("a"_sd, Decimal128(1.7), Decimal128(2)); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + BSONObjBuilder bob; + m.serialize(&bob, opts); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"a":{"$_internalSchemaFmod":["?number","?number"]}})", + bob.done()); +} + +TEST(SerializeInternalSchema, MatchArrayIndexRedactsCorrectly) { + auto query = fromjson( + "{foo: {$_internalSchemaMatchArrayIndex:" + "{index: 0, namePlaceholder: 'i', expression: {i: {$type: 'number'}}}}}"); + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + auto objMatch = MatchExpressionParser::parse(query, expCtx); + ASSERT_OK(objMatch.getStatus()); + + BSONObjBuilder bob; + auto opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + objMatch.getValue()->serialize(&bob, opts); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "HASH<foo>": { + "$_internalSchemaMatchArrayIndex": { + "index": "?number", + "namePlaceholder": "HASH<i>", + "expression": { + "HASH<i>": { + "$type": ['number'] + } + } + } + } + })", + bob.done()); +} + +TEST(SerializeInternalSchema, MaxItemsRedactsCorrectly) { + InternalSchemaMaxItemsMatchExpression maxItems("a.b"_sd, 2); + auto opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalSchemaMaxItems":"?number"})", + maxItems.getSerializedRightHandSide(opts)); +} + +TEST(SerializeInternalSchema, MaxLengthRedactsCorrectly) { + InternalSchemaMaxLengthMatchExpression maxLength("a"_sd, 2); + auto opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalSchemaMaxLength":"?number"})", + maxLength.getSerializedRightHandSide(opts)); +} + +TEST(SerializeInternalSchema, MinItemsRedactsCorrectly) { + InternalSchemaMinItemsMatchExpression minItems("a.b"_sd, 2); + auto opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalSchemaMinItems":"?number"})", + minItems.getSerializedRightHandSide(opts)); +} + +TEST(SerializeInternalSchema, MinLengthRedactsCorrectly) { + InternalSchemaMinLengthMatchExpression minLength("a"_sd, 2); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalSchemaMinLength":"?number"})", + minLength.getSerializedRightHandSide(opts)); +} + +TEST(SerializeInternalSchema, MinPropertiesRedactsCorrectly) { + InternalSchemaMinPropertiesMatchExpression minProperties(5); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + + BSONObjBuilder bob; + minProperties.serialize(&bob, opts); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalSchemaMinProperties":"?number"})", + bob.done()); +} + +TEST(SerializeInternalSchema, ObjectMatchRedactsCorrectly) { + auto opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto query = fromjson( + " {a: {$_internalSchemaObjectMatch: {" + " c: {$eq: 3}" + " }}}"); + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + auto objMatch = MatchExpressionParser::parse(query, expCtx); + ASSERT_OK(objMatch.getStatus()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"HASH<a>":{"$_internalSchemaObjectMatch":{"HASH<c>":{"$eq":"?number"}}}})", + objMatch.getValue()->serialize(opts)); +} + +TEST(SerializeInternalSchema, RootDocEqRedactsCorrectly) { + auto query = fromjson("{$_internalSchemaRootDocEq: {a:1, b: {c: 1, d: [1]}}}"); + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + auto opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto objMatch = MatchExpressionParser::parse(query, expCtx); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSchemaRootDocEq": { + "HASH<a>": "?number", + "HASH<b>": { + "HASH<c>": "?number", + "HASH<d>": [ + "?number" + ] + } + } + })", + objMatch.getValue()->serialize(opts)); +} + +TEST(SerializeInternalSchema, BinDataEncryptedTypeRedactsCorrectly) { + MatcherTypeSet typeSet; + typeSet.bsonTypes.insert(BSONType::String); + typeSet.bsonTypes.insert(BSONType::Date); + InternalSchemaBinDataEncryptedTypeExpression e("a"_sd, std::move(typeSet)); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalSchemaBinDataEncryptedType":[2,9]})", + e.getSerializedRightHandSide(opts)); +} + +TEST(SerializeInternalSchema, BinDataFLE2EncryptedTypeRedactsCorrectly) { + InternalSchemaBinDataFLE2EncryptedTypeExpression e("ssn"_sd, BSONType::String); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalSchemaBinDataFLE2EncryptedType":[2]})", + e.getSerializedRightHandSide(opts)); +} + +TEST(SerializesInternalSchema, MaxPropertiesRedactsCorrectly) { + InternalSchemaMaxPropertiesMatchExpression maxProperties(5); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + + BSONObjBuilder bob; + maxProperties.serialize(&bob, opts); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalSchemaMaxProperties":"?number"})", + bob.done()); +} + +TEST(SerializesInternalSchema, EqRedactsCorrectly) { + auto opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto query = fromjson("{$_internalSchemaEq: {a:1, b: {c: 1, d: [1]}}}"); + BSONObjBuilder bob; + InternalSchemaEqMatchExpression e("a"_sd, query.firstElement()); + e.serialize(&bob, opts); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "HASH<a>": { + "$_internalSchemaEq": { + "HASH<a>": "?number", + "HASH<b>": { + "HASH<c>": "?number", + "HASH<d>": [ + "?number" + ] + } + } + } + })", + bob.done()); +} + +TEST(InternalSchemaAllElemMatchFromIndexMatchExpression, RedactsExpressionCorrectly) { + auto query = fromjson("{a: {$_internalSchemaAllElemMatchFromIndex: [2, {a: {$lt: 5}}]}}"); + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + auto expr = MatchExpressionParser::parse(query, expCtx); + ASSERT_OK(expr.getStatus()); + auto elemMatchExpr = dynamic_cast<const InternalSchemaAllElemMatchFromIndexMatchExpression*>( + expr.getValue().get()); + + auto opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSchemaAllElemMatchFromIndex": [ + "?number", + { + "HASH<a>": { + "$lt": "?number" + } + } + ] + })", + elemMatchExpr->getSerializedRightHandSide(opts)); +} + +TEST(SerializeBasic, SerializesNestedElemMatchCorrectly) { + auto query = fromjson(R"({a: {$elemMatch: {$elemMatch: {b: {$lt: 6, $gt: 4}}}}})"); + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + auto objMatch = MatchExpressionParser::parse(query, expCtx); + ASSERT_OK(objMatch.getStatus()); + SerializationOptions opts; + opts.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + ASSERT_BSONOBJ_EQ_AUTO( + R"({"a": { + "$elemMatch": { + "$elemMatch": { + "$and": [ + { + "b": { + "$lt": "?number" + } + }, + { + "b": { + "$gt": "?number" + } + } + ] + } + } + } + })", + objMatch.getValue()->serialize(opts)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/matcher/expression_text_base.cpp b/src/mongo/db/matcher/expression_text_base.cpp index 56ad05af3d3..ce485de6fbc 100644 --- a/src/mongo/db/matcher/expression_text_base.cpp +++ b/src/mongo/db/matcher/expression_text_base.cpp @@ -56,12 +56,16 @@ void TextMatchExpressionBase::debugString(StringBuilder& debug, int indentationL debug << "\n"; } -void TextMatchExpressionBase::serialize(BSONObjBuilder* out, bool includePath) const { +void TextMatchExpressionBase::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, + bool includePath) const { const fts::FTSQuery& ftsQuery = getFTSQuery(); out->append("$text", - BSON("$search" << ftsQuery.getQuery() << "$language" << ftsQuery.getLanguage() - << "$caseSensitive" << ftsQuery.getCaseSensitive() - << "$diacriticSensitive" << ftsQuery.getDiacriticSensitive())); + BSON("$search" << opts.serializeLiteral(ftsQuery.getQuery()) << "$language" + << opts.serializeLiteral(ftsQuery.getLanguage()) << "$caseSensitive" + << opts.serializeLiteral(ftsQuery.getCaseSensitive()) + << "$diacriticSensitive" + << opts.serializeLiteral(ftsQuery.getDiacriticSensitive()))); } bool TextMatchExpressionBase::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/expression_text_base.h b/src/mongo/db/matcher/expression_text_base.h index 22fc581a2e2..622438e1500 100644 --- a/src/mongo/db/matcher/expression_text_base.h +++ b/src/mongo/db/matcher/expression_text_base.h @@ -60,7 +60,9 @@ public: */ virtual const fts::FTSQuery& getFTSQuery() const = 0; - BSONObj getSerializedRightHandSide() const final { + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final { MONGO_UNREACHABLE; } @@ -70,7 +72,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel = 0) const final; - void serialize(BSONObjBuilder* out, bool includePath) const final; + void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/expression_tree.cpp b/src/mongo/db/matcher/expression_tree.cpp index a7496729144..74765949ca5 100644 --- a/src/mongo/db/matcher/expression_tree.cpp +++ b/src/mongo/db/matcher/expression_tree.cpp @@ -40,16 +40,106 @@ #include "mongo/db/matcher/expression_text_base.h" namespace mongo { +namespace { + +PathMatchExpression* getEligiblePathMatchForNotSerialization(MatchExpression* expr) { + // Returns a pointer to a PathMatchExpression if 'expr' is such a pointer, otherwise returns + // nullptr. + // + // One exception: while TextMatchExpressionBase derives from PathMatchExpression, text match + // expressions cannot be serialized in the same manner as other PathMatchExpression derivatives. + // This is because the path for a TextMatchExpression is embedded within the $text object, + // whereas for other PathMatchExpressions it is on the left-hand-side, for example {x: {$eq: + // 1}}. + // + // Rather than the following dynamic_cast, we'll do a more performant, but also more verbose + // check. + // dynamic_cast<PathMatchExpression*>(expr) && !dynamic_cast<TextMatchExpressionBase*>(expr) + // + // This version below is less obviously exhaustive, but because this is just a legibility + // optimization, and this function also gets called on the query shape stats recording hot path, + // we think it is worth it. + switch (expr->matchType()) { + // leaf types + case MatchExpression::EQ: + case MatchExpression::LTE: + case MatchExpression::LT: + case MatchExpression::GT: + case MatchExpression::GTE: + case MatchExpression::REGEX: + case MatchExpression::MOD: + case MatchExpression::EXISTS: + case MatchExpression::MATCH_IN: + case MatchExpression::BITS_ALL_SET: + case MatchExpression::BITS_ALL_CLEAR: + case MatchExpression::BITS_ANY_SET: + case MatchExpression::BITS_ANY_CLEAR: + // array types + case MatchExpression::ELEM_MATCH_OBJECT: + case MatchExpression::ELEM_MATCH_VALUE: + case MatchExpression::SIZE: + // special types + case MatchExpression::TYPE_OPERATOR: + case MatchExpression::GEO: + case MatchExpression::GEO_NEAR: + // Internal subclasses of PathMatchExpression: + case MatchExpression::INTERNAL_SCHEMA_ALL_ELEM_MATCH_FROM_INDEX: + case MatchExpression::INTERNAL_SCHEMA_BIN_DATA_ENCRYPTED_TYPE: + case MatchExpression::INTERNAL_SCHEMA_BIN_DATA_FLE2_ENCRYPTED_TYPE: + case MatchExpression::INTERNAL_SCHEMA_BIN_DATA_SUBTYPE: + case MatchExpression::INTERNAL_SCHEMA_MATCH_ARRAY_INDEX: + case MatchExpression::INTERNAL_SCHEMA_MAX_ITEMS: + case MatchExpression::INTERNAL_SCHEMA_MAX_LENGTH: + case MatchExpression::INTERNAL_SCHEMA_MAX_PROPERTIES: + case MatchExpression::INTERNAL_SCHEMA_MIN_ITEMS: + case MatchExpression::INTERNAL_SCHEMA_MIN_LENGTH: + case MatchExpression::INTERNAL_SCHEMA_TYPE: + case MatchExpression::INTERNAL_SCHEMA_UNIQUE_ITEMS: + return static_cast<PathMatchExpression*>(expr); + // purposefully skip TEXT: + case MatchExpression::TEXT: + // Any other type is not considered a PathMatchExpression. + case MatchExpression::AND: + case MatchExpression::OR: + case MatchExpression::NOT: + case MatchExpression::NOR: + case MatchExpression::WHERE: + case MatchExpression::EXPRESSION: + case MatchExpression::ALWAYS_FALSE: + case MatchExpression::ALWAYS_TRUE: + case MatchExpression::INTERNAL_2D_POINT_IN_ANNULUS: + case MatchExpression::INTERNAL_BUCKET_GEO_WITHIN: + case MatchExpression::INTERNAL_EXPR_EQ: + case MatchExpression::INTERNAL_EXPR_GT: + case MatchExpression::INTERNAL_EXPR_GTE: + case MatchExpression::INTERNAL_EXPR_LT: + case MatchExpression::INTERNAL_EXPR_LTE: + case MatchExpression::INTERNAL_SCHEMA_ALLOWED_PROPERTIES: + case MatchExpression::INTERNAL_SCHEMA_COND: + case MatchExpression::INTERNAL_SCHEMA_EQ: + case MatchExpression::INTERNAL_SCHEMA_FMOD: + case MatchExpression::INTERNAL_SCHEMA_MIN_PROPERTIES: + case MatchExpression::INTERNAL_SCHEMA_OBJECT_MATCH: + case MatchExpression::INTERNAL_SCHEMA_ROOT_DOC_EQ: + case MatchExpression::INTERNAL_SCHEMA_XOR: + return nullptr; + default: + MONGO_UNREACHABLE_TASSERT(7800300); + } +}; +} // namespace void ListOfMatchExpression::_debugList(StringBuilder& debug, int indentationLevel) const { for (unsigned i = 0; i < _expressions.size(); i++) _expressions[i]->debugString(debug, indentationLevel + 1); } -void ListOfMatchExpression::_listToBSON(BSONArrayBuilder* out, bool includePath) const { +void ListOfMatchExpression::_listToBSON(BSONArrayBuilder* out, + const SerializationOptions& opts, + bool includePath) const { for (unsigned i = 0; i < _expressions.size(); i++) { BSONObjBuilder childBob(out->subobjStart()); - _expressions[i]->serialize(&childBob, includePath); + _expressions[i]->serialize(&childBob, opts, includePath); } out->doneFast(); } @@ -354,7 +444,9 @@ void AndMatchExpression::debugString(StringBuilder& debug, int indentationLevel) _debugList(debug, indentationLevel); } -void AndMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const { +void AndMatchExpression::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, + bool includePath) const { if (!numChildren()) { // It is possible for an AndMatchExpression to have no children, resulting in the serialized // expression {$and: []}, which is not a valid query object. @@ -362,7 +454,7 @@ void AndMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const } BSONArrayBuilder arrBob(out->subarrayStart("$and")); - _listToBSON(&arrBob, includePath); + _listToBSON(&arrBob, opts, includePath); arrBob.doneFast(); } @@ -398,7 +490,9 @@ void OrMatchExpression::debugString(StringBuilder& debug, int indentationLevel) _debugList(debug, indentationLevel); } -void OrMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const { +void OrMatchExpression::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, + bool includePath) const { if (!numChildren()) { // It is possible for an OrMatchExpression to have no children, resulting in the serialized // expression {$or: []}, which is not a valid query object. An empty $or is logically @@ -407,7 +501,7 @@ void OrMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const { return; } BSONArrayBuilder arrBob(out->subarrayStart("$or")); - _listToBSON(&arrBob, includePath); + _listToBSON(&arrBob, opts, includePath); } bool OrMatchExpression::isTriviallyFalse() const { @@ -440,9 +534,11 @@ void NorMatchExpression::debugString(StringBuilder& debug, int indentationLevel) _debugList(debug, indentationLevel); } -void NorMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const { +void NorMatchExpression::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, + bool includePath) const { BSONArrayBuilder arrBob(out->subarrayStart("$nor")); - _listToBSON(&arrBob, includePath); + _listToBSON(&arrBob, opts, includePath); } // ------- @@ -455,9 +551,10 @@ void NotMatchExpression::debugString(StringBuilder& debug, int indentationLevel) void NotMatchExpression::serializeNotExpressionToNor(MatchExpression* exp, BSONObjBuilder* out, + const SerializationOptions& opts, bool includePath) { BSONObjBuilder childBob; - exp->serialize(&childBob, includePath); + exp->serialize(&childBob, opts, includePath); BSONObj tempObj = childBob.obj(); BSONArrayBuilder tBob(out->subarrayStart("$nor")); @@ -465,9 +562,11 @@ void NotMatchExpression::serializeNotExpressionToNor(MatchExpression* exp, tBob.doneFast(); } -void NotMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const { +void NotMatchExpression::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, + bool includePath) const { if (_exp->matchType() == MatchType::AND && _exp->numChildren() == 0) { - out->append("$alwaysFalse", 1); + opts.appendLiteral(out, "$alwaysFalse", 1); return; } @@ -478,10 +577,10 @@ void NotMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const // internally, so we un-nest it here to be able to re-parse it. if (_exp->matchType() == MatchType::AND) { for (size_t x = 0; x < _exp->numChildren(); ++x) { - _exp->getChild(x)->serialize(¬Bob, includePath); + _exp->getChild(x)->serialize(¬Bob, opts, includePath); } } else { - _exp->serialize(¬Bob, includePath); + _exp->serialize(¬Bob, opts, includePath); } return; } @@ -494,20 +593,15 @@ void NotMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const // It is generally easier to be correct if we just always serialize to a $nor, since this will // delegate the path serialization to lower in the tree where we have the information on-hand. // However, for legibility we preserve a $not with a single path-accepting child as a $not. - // - // One exception: while TextMatchExpressionBase derives from PathMatchExpression, text match - // expressions cannot be serialized in the same manner as other PathMatchExpression derivatives. - // This is because the path for a TextMatchExpression is embedded within the $text object, - // whereas for other PathMatchExpressions it is on the left-hand-side, for example {x: {$eq: - // 1}}. - if (auto pathMatch = dynamic_cast<PathMatchExpression*>(expressionToNegate); - pathMatch && !dynamic_cast<TextMatchExpressionBase*>(expressionToNegate)) { - const auto path = pathMatch->path(); - BSONObjBuilder pathBob(out->subobjStart(path)); - pathBob.append("$not", pathMatch->getSerializedRightHandSide()); + if (auto pathMatch = getEligiblePathMatchForNotSerialization(expressionToNegate)) { + auto append = [&](StringData path) { + BSONObjBuilder pathBob(out->subobjStart(path)); + pathBob.append("$not", pathMatch->getSerializedRightHandSide(opts)); + }; + append(opts.serializeFieldPathFromString(pathMatch->path())); return; } - return serializeNotExpressionToNor(expressionToNegate, out, includePath); + return serializeNotExpressionToNor(expressionToNegate, out, opts); } bool NotMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/expression_tree.h b/src/mongo/db/matcher/expression_tree.h index f8b9179c003..b067c7b5f7e 100644 --- a/src/mongo/db/matcher/expression_tree.h +++ b/src/mongo/db/matcher/expression_tree.h @@ -98,7 +98,9 @@ public: protected: void _debugList(StringBuilder& debug, int indentationLevel) const; - void _listToBSON(BSONArrayBuilder* out, bool includePath) const; + void _listToBSON(BSONArrayBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const; private: ExpressionOptimizerFunc getOptimizer() const final; @@ -137,7 +139,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel = 0) const; - virtual void serialize(BSONObjBuilder* out, bool includePath) const; + virtual void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const; bool isTriviallyTrue() const final; @@ -181,7 +185,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel = 0) const; - virtual void serialize(BSONObjBuilder* out, bool includePath) const; + virtual void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const; bool isTriviallyFalse() const final; @@ -225,7 +231,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel = 0) const; - virtual void serialize(BSONObjBuilder* out, bool includePath) const; + virtual void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const; void acceptVisitor(MatchExpressionMutableVisitor* visitor) final { visitor->visit(this); @@ -265,7 +273,9 @@ public: virtual void debugString(StringBuilder& debug, int indentationLevel = 0) const; - virtual void serialize(BSONObjBuilder* out, bool includePath) const; + virtual void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const; bool equivalent(const MatchExpression* other) const final; @@ -306,7 +316,8 @@ public: private: static void serializeNotExpressionToNor(MatchExpression* exp, BSONObjBuilder* out, - bool includePath); + const SerializationOptions& opts = {}, + bool includePath = true); ExpressionOptimizerFunc getOptimizer() const final; diff --git a/src/mongo/db/matcher/expression_type.h b/src/mongo/db/matcher/expression_type.h index c81aade1701..07e575245c9 100644 --- a/src/mongo/db/matcher/expression_type.h +++ b/src/mongo/db/matcher/expression_type.h @@ -82,12 +82,10 @@ public: debug << "\n"; } - BSONObj getSerializedRightHandSide() const final { - BSONObjBuilder subBuilder; - BSONArrayBuilder arrBuilder(subBuilder.subarrayStart(name())); - _typeSet.toBSONArray(&arrBuilder); - arrBuilder.doneFast(); - return subBuilder.obj(); + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final { + bob->appendArray(name(), _typeSet.toBSONArray()); } bool equivalent(const MatchExpression* other) const final { @@ -256,10 +254,19 @@ public: debug << "\n"; } - BSONObj getSerializedRightHandSide() const final { - BSONObjBuilder bob; - bob.append(name(), _binDataSubType); - return bob.obj(); + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final { + if (opts.literalPolicy == LiteralSerializationPolicy::kUnchanged) { + bob->append(name(), _binDataSubType); + } else { + // There is some fancy serialization logic to get the above BSONObjBuilder append to + // work. We just want to make sure we're doing the same thing here. + static_assert(BSONObjAppendFormat<decltype(_binDataSubType)>::value == NumberInt, + "Expecting that the BinData sub type should be specified and serialized " + "as an int."); + opts.appendLiteral(bob, name(), static_cast<int>(_binDataSubType)); + } } bool equivalent(const MatchExpression* other) const final { diff --git a/src/mongo/db/matcher/expression_type_test.cpp b/src/mongo/db/matcher/expression_type_test.cpp index bf7d40e836a..573ab77860f 100644 --- a/src/mongo/db/matcher/expression_type_test.cpp +++ b/src/mongo/db/matcher/expression_type_test.cpp @@ -217,6 +217,14 @@ TEST(ExpressionTypeTest, InternalSchemaTypeExprWithMultipleTypesMatchesAllSuchTy ASSERT_FALSE(expr.matchesBSON(fromjson("{a: ['str']}"))); } +TEST(ExpressionTypeTest, RedactsTypesCorrectly) { + TypeMatchExpression type(""_sd, String); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$type":[2]})", + type.getSerializedRightHandSide(opts)); +} + TEST(ExpressionBinDataSubTypeTest, MatchesBinDataGeneral) { BSONObj match = BSON("a" << BSONBinData(nullptr, 0, BinDataType::BinDataGeneral)); BSONObj notMatch = BSON("a" << BSONBinData(nullptr, 0, BinDataType::bdtCustom)); @@ -265,6 +273,14 @@ TEST(ExpressionBinDataSubTypeTest, MatchesBinDataColumnType) { ASSERT_FALSE(type.matchesSingleElement(notMatch["a"])); } +TEST(ExpressionBinDataSubTypeTest, MatchesBinDataSensitiveType) { + BSONObj match = BSON("a" << BSONBinData(nullptr, 0, BinDataType::Sensitive)); + BSONObj notMatch = BSON("a" << BSONBinData(nullptr, 0, BinDataType::newUUID)); + InternalSchemaBinDataSubTypeExpression type(""_sd, BinDataType::Sensitive); + ASSERT_TRUE(type.matchesSingleElement(match["a"])); + ASSERT_FALSE(type.matchesSingleElement(notMatch["a"])); +} + TEST(ExpressionBinDataSubTypeTest, MatchesBinDataBdtCustom) { BSONObj match = BSON("a" << BSONBinData(nullptr, 0, BinDataType::bdtCustom)); BSONObj notMatch = BSON("a" << BSONBinData(nullptr, 0, BinDataType::Function)); @@ -300,6 +316,14 @@ TEST(ExpressionBinDataSubTypeTest, Equivalent) { ASSERT(!e1.equivalent(&e3)); } +TEST(ExpressionBinDataSubTypeTest, RedactsCorrectly) { + InternalSchemaBinDataSubTypeExpression e("b"_sd, BinDataType::newUUID); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalSchemaBinDataSubType":"?number"})", + e.getSerializedRightHandSide(opts)); +} + TEST(InternalSchemaBinDataEncryptedTypeTest, DoesNotTraverseLeafArrays) { MatcherTypeSet typeSet; typeSet.bsonTypes.insert(BSONType::String); diff --git a/src/mongo/db/matcher/expression_where_base.cpp b/src/mongo/db/matcher/expression_where_base.cpp index 30148f9a42b..b25a8febccf 100644 --- a/src/mongo/db/matcher/expression_where_base.cpp +++ b/src/mongo/db/matcher/expression_where_base.cpp @@ -46,8 +46,10 @@ void WhereMatchExpressionBase::debugString(StringBuilder& debug, int indentation debug << "code: " << getCode() << "\n"; } -void WhereMatchExpressionBase::serialize(BSONObjBuilder* out, bool includePath) const { - out->appendCode("$where", getCode()); +void WhereMatchExpressionBase::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, + bool includePath) const { + opts.appendLiteral(out, "$where", BSONCode(getCode())); } bool WhereMatchExpressionBase::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/expression_where_base.h b/src/mongo/db/matcher/expression_where_base.h index 550e94d1d37..91b6d0985cd 100644 --- a/src/mongo/db/matcher/expression_where_base.h +++ b/src/mongo/db/matcher/expression_where_base.h @@ -69,7 +69,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel = 0) const final; - void serialize(BSONObjBuilder* out, bool includePath) const final; + void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/parsed_match_expression_for_test.h b/src/mongo/db/matcher/parsed_match_expression_for_test.h new file mode 100644 index 00000000000..4c4d8a9dc43 --- /dev/null +++ b/src/mongo/db/matcher/parsed_match_expression_for_test.h @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/matcher/expression.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { + +/** + * A MatchExpression may store BSONElements as arguments for expressions, to avoid copying large + * values. A BSONElement is essentially a pointer into a BSONObj, so use + * ParsedMatchExpressionForTest to ensure that the BSONObj outlives the MatchExpression, and the + * BSONElement arguments remain pointing to allocated memory. + */ +class ParsedMatchExpressionForTest { +public: + ParsedMatchExpressionForTest(const std::string& str, + const CollatorInterface* collator = nullptr) + : _obj(fromjson(str)) { + _expCtx = make_intrusive<ExpressionContextForTest>(); + _expCtx->setCollator(CollatorInterface::cloneCollator(collator)); + StatusWithMatchExpression result = + MatchExpressionParser::parse(_obj, + _expCtx, + ExtensionsCallbackNoop(), + MatchExpressionParser::kDefaultSpecialFeatures | + MatchExpressionParser::AllowedFeatures::kJavascript); + ASSERT_OK(result.getStatus()); + _expr = std::move(result.getValue()); + } + + const MatchExpression* get() const { + return _expr.get(); + } + + /** + * Relinquishes ownership of the parsed expression and returns it as a unique_ptr to the caller. + * This 'ParsedMatchExpressionForTest' object still must outlive the returned value so that the + * BSONObj used to create it remains alive. + */ + std::unique_ptr<MatchExpression> release() { + return std::move(_expr); + } + + +private: + const BSONObj _obj; + std::unique_ptr<MatchExpression> _expr; + boost::intrusive_ptr<ExpressionContext> _expCtx; +}; +} // namespace mongo diff --git a/src/mongo/db/matcher/rewrite_expr.cpp b/src/mongo/db/matcher/rewrite_expr.cpp index 4edb9abd222..4f44cd269ef 100644 --- a/src/mongo/db/matcher/rewrite_expr.cpp +++ b/src/mongo/db/matcher/rewrite_expr.cpp @@ -45,7 +45,7 @@ using CmpOp = ExpressionCompare::CmpOp; RewriteExpr::RewriteResult RewriteExpr::rewrite(const boost::intrusive_ptr<Expression>& expression, const CollatorInterface* collator) { LOGV2_DEBUG( - 20725, 5, "Expression prior to rewrite", "expression"_attr = expression->serialize(false)); + 20725, 5, "Expression prior to rewrite", "expression"_attr = expression->serialize()); RewriteExpr rewriteExpr(collator); std::unique_ptr<MatchExpression> matchExpression; diff --git a/src/mongo/db/matcher/rewrite_expr_test.cpp b/src/mongo/db/matcher/rewrite_expr_test.cpp index 948add02c26..c4f1a4b7619 100644 --- a/src/mongo/db/matcher/rewrite_expr_test.cpp +++ b/src/mongo/db/matcher/rewrite_expr_test.cpp @@ -57,7 +57,7 @@ void testExprRewrite(BSONObj expr, BSONObj expectedMatch) { if (!expectedMatch.isEmpty()) { ASSERT(result.matchExpression()); BSONObjBuilder bob; - result.matchExpression()->serialize(&bob, true); + result.matchExpression()->serialize(&bob, {}); ASSERT_BSONOBJ_EQ(expectedMatch, bob.obj()); } else { ASSERT_FALSE(result.matchExpression()); diff --git a/src/mongo/db/matcher/schema/assert_serializes_to.h b/src/mongo/db/matcher/schema/assert_serializes_to.h index e62b5e62e1d..7f9b3d14292 100644 --- a/src/mongo/db/matcher/schema/assert_serializes_to.h +++ b/src/mongo/db/matcher/schema/assert_serializes_to.h @@ -34,11 +34,9 @@ namespace mongo { /** * Asserts that the given MatchExpression 'match' serializes to the BSONObj 'expected'. */ -#define ASSERT_SERIALIZES_TO(match, expected) \ - do { \ - BSONObjBuilder bob; \ - match->serialize(&bob, true); \ - ASSERT_BSONOBJ_EQ(bob.obj(), expected); \ +#define ASSERT_SERIALIZES_TO(match, expected) \ + do { \ + ASSERT_BSONOBJ_EQ(match->serialize(), expected); \ } while (false) } // namespace mongo diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index.cpp index 27b2adae68d..caf39e7fcc4 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index.cpp @@ -76,17 +76,11 @@ void InternalSchemaAllElemMatchFromIndexMatchExpression::debugString(StringBuild _expression->getFilter()->debugString(debug, indentationLevel + 1); } -BSONObj InternalSchemaAllElemMatchFromIndexMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder allElemMatchBob; - BSONArrayBuilder subArray(allElemMatchBob.subarrayStart(kName)); - subArray.append(_index); - { - BSONObjBuilder eBuilder(subArray.subobjStart()); - _expression->getFilter()->serialize(&eBuilder, true); - eBuilder.doneFast(); - } - subArray.doneFast(); - return allElemMatchBob.obj(); +void InternalSchemaAllElemMatchFromIndexMatchExpression::appendSerializedRightHandSide( + BSONObjBuilder* bob, const SerializationOptions& opts, bool includePath) const { + bob->append(kName, + BSON_ARRAY(opts.serializeLiteral(_index) + << _expression->getFilter()->serialize(opts, includePath))); } MatchExpression::ExpressionOptimizerFunc diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index.h b/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index.h index 0a613b23303..1d1ff14d6c0 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index.h @@ -77,7 +77,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel) const final; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index_test.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index_test.cpp index 49e1b3b0235..b3b3234778d 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index_test.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_all_elem_match_from_index_test.cpp @@ -135,6 +135,5 @@ DEATH_TEST_REGEX(InternalSchemaAllElemMatchFromIndexMatchExpression, objMatch.getValue()->getChild(1); } - } // namespace } // namespace mongo diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_allowed_properties.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_allowed_properties.cpp index 64b34aafc3a..5a6471e2155 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_allowed_properties.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_allowed_properties.cpp @@ -59,7 +59,7 @@ void InternalSchemaAllowedPropertiesMatchExpression::debugString(StringBuilder& _debugAddSpace(debug, indentationLevel); BSONObjBuilder builder; - serialize(&builder, true); + serialize(&builder, {}); debug << builder.obj().toString() << "\n"; const auto* tag = getTag(); @@ -128,29 +128,29 @@ bool InternalSchemaAllowedPropertiesMatchExpression::_matchesBSONObj(const BSONO } void InternalSchemaAllowedPropertiesMatchExpression::serialize(BSONObjBuilder* builder, + const SerializationOptions& opts, bool includePath) const { BSONObjBuilder expressionBuilder( builder->subobjStart(InternalSchemaAllowedPropertiesMatchExpression::kName)); std::vector<StringData> sortedProperties(_properties.begin(), _properties.end()); std::sort(sortedProperties.begin(), sortedProperties.end()); - expressionBuilder.append("properties", sortedProperties); - + opts.appendLiteral(&expressionBuilder, "properties", sortedProperties); + // This will be serialized to "i", which is the parser chosen namePlaceholder. Using this + // unmodified will have a similar effect to serializing to "?", however it preserves round trip + // parsing. expressionBuilder.append("namePlaceholder", _namePlaceholder); BSONArrayBuilder patternPropertiesBuilder(expressionBuilder.subarrayStart("patternProperties")); - for (auto&& item : _patternProperties) { - BSONObjBuilder itemBuilder(patternPropertiesBuilder.subobjStart()); - itemBuilder.appendRegex("regex", item.first.rawRegex); - - BSONObjBuilder subexpressionBuilder(itemBuilder.subobjStart("expression")); - item.second->getFilter()->serialize(&subexpressionBuilder, includePath); - subexpressionBuilder.doneFast(); + for (auto&& [pattern, expression] : _patternProperties) { + patternPropertiesBuilder << BSON( + "regex" << opts.serializeLiteral(BSONRegEx(pattern.rawRegex)) << "expression" + << expression->getFilter()->serialize(opts, includePath)); } patternPropertiesBuilder.doneFast(); BSONObjBuilder otherwiseBuilder(expressionBuilder.subobjStart("otherwise")); - _otherwise->getFilter()->serialize(&otherwiseBuilder, includePath); + _otherwise->getFilter()->serialize(&otherwiseBuilder, opts, includePath); otherwiseBuilder.doneFast(); expressionBuilder.doneFast(); } diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_allowed_properties.h b/src/mongo/db/matcher/schema/expression_internal_schema_allowed_properties.h index e95d0582d15..ae6de80a7f8 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_allowed_properties.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_allowed_properties.h @@ -136,7 +136,9 @@ public: bool matches(const MatchableDocument* doc, MatchDetails* details) const final; bool matchesSingleElement(const BSONElement& element, MatchDetails* details) const final; - void serialize(BSONObjBuilder* builder, bool includePath) const final; + void serialize(BSONObjBuilder* builder, + const SerializationOptions& opts = {}, + bool includePath = true) const final; std::unique_ptr<MatchExpression> shallowClone() const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_eq.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_eq.cpp index 263b23cc4e9..31f5812256f 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_eq.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_eq.cpp @@ -69,10 +69,16 @@ void InternalSchemaEqMatchExpression::debugString(StringBuilder& debug, debug << "\n"; } -BSONObj InternalSchemaEqMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder eqObj; - eqObj.appendAs(_rhsElem, kName); - return eqObj.obj(); +void InternalSchemaEqMatchExpression::appendSerializedRightHandSide( + BSONObjBuilder* bob, const SerializationOptions& opts, bool includePath) const { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged && _rhsElem.isABSONObj()) { + BSONObjBuilder exprSpec(bob->subobjStart(kName)); + opts.addHmacedObjToBuilder(&exprSpec, _rhsElem.Obj()); + exprSpec.doneFast(); + return; + } + // If the element is not an object it must be a literal. + opts.appendLiteral(bob, kName, _rhsElem); } bool InternalSchemaEqMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_eq.h b/src/mongo/db/matcher/schema/expression_internal_schema_eq.h index 3f604294f60..5a24d5787d2 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_eq.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_eq.h @@ -59,7 +59,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel) const final; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_fmod.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_fmod.cpp index e86e4d0240d..fbd267bcbbf 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_fmod.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_fmod.cpp @@ -76,13 +76,10 @@ void InternalSchemaFmodMatchExpression::debugString(StringBuilder& debug, debug << "\n"; } -BSONObj InternalSchemaFmodMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder objMatchBob; - BSONArrayBuilder arrBuilder(objMatchBob.subarrayStart("$_internalSchemaFmod")); - arrBuilder.append(_divisor); - arrBuilder.append(_remainder); - arrBuilder.doneFast(); - return objMatchBob.obj(); +void InternalSchemaFmodMatchExpression::appendSerializedRightHandSide( + BSONObjBuilder* bob, const SerializationOptions& opts, bool includePath) const { + bob->append("$_internalSchemaFmod"_sd, + BSON_ARRAY(opts.serializeLiteral(_divisor) << opts.serializeLiteral(_remainder))); } bool InternalSchemaFmodMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_fmod.h b/src/mongo/db/matcher/schema/expression_internal_schema_fmod.h index ae147f3a4fe..e808e079b27 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_fmod.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_fmod.h @@ -58,7 +58,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel) const final; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_match_array_index.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_match_array_index.cpp index a09ad27ff22..8af85dd6436 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_match_array_index.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_match_array_index.cpp @@ -51,7 +51,7 @@ void InternalSchemaMatchArrayIndexMatchExpression::debugString(StringBuilder& de _debugAddSpace(debug, indentationLevel); BSONObjBuilder builder; - serialize(&builder, true); + serialize(&builder, {}); debug << builder.obj().toString() << "\n"; const auto* tag = getTag(); @@ -72,20 +72,14 @@ bool InternalSchemaMatchArrayIndexMatchExpression::equivalent(const MatchExpress _expression->equivalent(other->_expression.get()); } -BSONObj InternalSchemaMatchArrayIndexMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder objBuilder; - { - BSONObjBuilder matchArrayElemSubobj(objBuilder.subobjStart(kName)); - matchArrayElemSubobj.append("index", _index); - matchArrayElemSubobj.append("namePlaceholder", _expression->getPlaceholder().value_or("")); - { - BSONObjBuilder subexprSubObj(matchArrayElemSubobj.subobjStart("expression")); - _expression->getFilter()->serialize(&subexprSubObj, true); - subexprSubObj.doneFast(); - } - matchArrayElemSubobj.doneFast(); - } - return objBuilder.obj(); +void InternalSchemaMatchArrayIndexMatchExpression::appendSerializedRightHandSide( + BSONObjBuilder* bob, const SerializationOptions& opts, bool includePath) const { + bob->append( + kName, + BSON( + "index" << opts.serializeLiteral(_index) << "namePlaceholder" + << opts.serializeFieldPathFromString(_expression->getPlaceholder().value_or("")) + << "expression" << _expression->getFilter()->serialize(opts, includePath))); } std::unique_ptr<MatchExpression> InternalSchemaMatchArrayIndexMatchExpression::shallowClone() diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_match_array_index.h b/src/mongo/db/matcher/schema/expression_internal_schema_match_array_index.h index 8fff1225bf3..cea0149a028 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_match_array_index.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_match_array_index.h @@ -73,7 +73,9 @@ public: return _expression->matchesBSONElement(element, details); } - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; std::unique_ptr<MatchExpression> shallowClone() const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_num_array_items.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_num_array_items.cpp index 6fde9a327cf..c6d20f63409 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_num_array_items.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_num_array_items.cpp @@ -56,10 +56,9 @@ void InternalSchemaNumArrayItemsMatchExpression::debugString(StringBuilder& debu debug << "\n"; } -BSONObj InternalSchemaNumArrayItemsMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder objBuilder; - objBuilder.append(_name, _numItems); - return objBuilder.obj(); +void InternalSchemaNumArrayItemsMatchExpression::appendSerializedRightHandSide( + BSONObjBuilder* bob, const SerializationOptions& opts, bool includePath) const { + opts.appendLiteral(bob, _name, _numItems); } bool InternalSchemaNumArrayItemsMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_num_array_items.h b/src/mongo/db/matcher/schema/expression_internal_schema_num_array_items.h index 99ed79ec5d6..e566692b642 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_num_array_items.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_num_array_items.h @@ -52,7 +52,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel) const final; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_num_properties.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_num_properties.cpp index e640a58ec71..91b764dd8f5 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_num_properties.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_num_properties.cpp @@ -37,13 +37,14 @@ void InternalSchemaNumPropertiesMatchExpression::debugString(StringBuilder& debu int indentationLevel) const { _debugAddSpace(debug, indentationLevel); BSONObjBuilder builder; - serialize(&builder, true); + serialize(&builder, {}); debug << builder.obj().toString() << "\n"; } void InternalSchemaNumPropertiesMatchExpression::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, bool includePath) const { - out->append(_name, _numProperties); + opts.appendLiteral(out, _name, _numProperties); } bool InternalSchemaNumPropertiesMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_num_properties.h b/src/mongo/db/matcher/schema/expression_internal_schema_num_properties.h index 29fd4fc6145..679aaaca609 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_num_properties.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_num_properties.h @@ -71,7 +71,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel) const final; - void serialize(BSONObjBuilder* out, bool includePath) const final; + void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_object_match.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_object_match.cpp index ec5a0943fad..317c316374d 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_object_match.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_object_match.cpp @@ -61,12 +61,9 @@ void InternalSchemaObjectMatchExpression::debugString(StringBuilder& debug, _sub->debugString(debug, indentationLevel + 1); } -BSONObj InternalSchemaObjectMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder objMatchBob; - BSONObjBuilder subBob(objMatchBob.subobjStart(kName)); - _sub->serialize(&subBob, true); - subBob.doneFast(); - return objMatchBob.obj(); +void InternalSchemaObjectMatchExpression::appendSerializedRightHandSide( + BSONObjBuilder* bob, const SerializationOptions& opts, bool includePath) const { + bob->append(kName, _sub->serialize(opts, includePath)); } bool InternalSchemaObjectMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_object_match.h b/src/mongo/db/matcher/schema/expression_internal_schema_object_match.h index 93b6e1b765e..a41190d9545 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_object_match.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_object_match.h @@ -49,7 +49,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel = 0) const final; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_root_doc_eq.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_root_doc_eq.cpp index c4b1e3f7aa2..300d1a49109 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_root_doc_eq.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_root_doc_eq.cpp @@ -55,9 +55,11 @@ void InternalSchemaRootDocEqMatchExpression::debugString(StringBuilder& debug, } void InternalSchemaRootDocEqMatchExpression::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, bool includePath) const { BSONObjBuilder subObj(out->subobjStart(kName)); - subObj.appendElements(_rhsObj); + SerializationOptions options = opts; + options.addHmacedObjToBuilder(&subObj, _rhsObj); subObj.doneFast(); } diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_root_doc_eq.h b/src/mongo/db/matcher/schema/expression_internal_schema_root_doc_eq.h index 62dfb6d66a2..4418e101d43 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_root_doc_eq.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_root_doc_eq.h @@ -72,7 +72,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel = 0) const final; - void serialize(BSONObjBuilder* out, bool includePath) const final; + void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_str_length.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_str_length.cpp index 87d6396fd0f..bdf2f5c2c4c 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_str_length.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_str_length.cpp @@ -56,10 +56,9 @@ void InternalSchemaStrLengthMatchExpression::debugString(StringBuilder& debug, debug << "\n"; } -BSONObj InternalSchemaStrLengthMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder objBuilder; - objBuilder.append(_name, _strLen); - return objBuilder.obj(); +void InternalSchemaStrLengthMatchExpression::appendSerializedRightHandSide( + BSONObjBuilder* bob, const SerializationOptions& opts, bool includePath) const { + opts.appendLiteral(bob, _name, _strLen); } bool InternalSchemaStrLengthMatchExpression::equivalent(const MatchExpression* other) const { diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_str_length.h b/src/mongo/db/matcher/schema/expression_internal_schema_str_length.h index 43483be1ce3..6796c532972 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_str_length.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_str_length.h @@ -62,7 +62,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel) const final; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; bool equivalent(const MatchExpression* other) const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_unique_items.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_unique_items.cpp index f78dca0c37f..afe8f8422a9 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_unique_items.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_unique_items.cpp @@ -39,7 +39,7 @@ void InternalSchemaUniqueItemsMatchExpression::debugString(StringBuilder& debug, _debugAddSpace(debug, indentationLevel); BSONObjBuilder builder; - serialize(&builder, true); + serialize(&builder, {}); debug << builder.obj().toString() << "\n"; const auto* tag = getTag(); @@ -59,10 +59,9 @@ bool InternalSchemaUniqueItemsMatchExpression::equivalent(const MatchExpression* return path() == other->path(); } -BSONObj InternalSchemaUniqueItemsMatchExpression::getSerializedRightHandSide() const { - BSONObjBuilder bob; - bob.append(kName, true); - return bob.obj(); +void InternalSchemaUniqueItemsMatchExpression::appendSerializedRightHandSide( + BSONObjBuilder* bob, const SerializationOptions& opts, bool includePath) const { + bob->append(kName, true); } std::unique_ptr<MatchExpression> InternalSchemaUniqueItemsMatchExpression::shallowClone() const { diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_unique_items.h b/src/mongo/db/matcher/schema/expression_internal_schema_unique_items.h index ddb86c89403..dc6e76af59f 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_unique_items.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_unique_items.h @@ -86,7 +86,9 @@ public: bool equivalent(const MatchExpression* other) const final; - BSONObj getSerializedRightHandSide() const final; + void appendSerializedRightHandSide(BSONObjBuilder* bob, + const SerializationOptions& opts = {}, + bool includePath = true) const final; std::unique_ptr<MatchExpression> shallowClone() const final; diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_xor.cpp b/src/mongo/db/matcher/schema/expression_internal_schema_xor.cpp index ad265266f52..04c5bbb045a 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_xor.cpp +++ b/src/mongo/db/matcher/schema/expression_internal_schema_xor.cpp @@ -73,8 +73,10 @@ void InternalSchemaXorMatchExpression::debugString(StringBuilder& debug, _debugList(debug, indentationLevel); } -void InternalSchemaXorMatchExpression::serialize(BSONObjBuilder* out, bool includePath) const { +void InternalSchemaXorMatchExpression::serialize(BSONObjBuilder* out, + const SerializationOptions& opts, + bool includePath) const { BSONArrayBuilder arrBob(out->subarrayStart(kName)); - _listToBSON(&arrBob, includePath); + _listToBSON(&arrBob, opts, includePath); } } // namespace mongo diff --git a/src/mongo/db/matcher/schema/expression_internal_schema_xor.h b/src/mongo/db/matcher/schema/expression_internal_schema_xor.h index 928a13413f1..57825bff060 100644 --- a/src/mongo/db/matcher/schema/expression_internal_schema_xor.h +++ b/src/mongo/db/matcher/schema/expression_internal_schema_xor.h @@ -69,7 +69,9 @@ public: void debugString(StringBuilder& debug, int indentationLevel = 0) const final; - void serialize(BSONObjBuilder* out, bool includePath) const final; + void serialize(BSONObjBuilder* out, + const SerializationOptions& opts = {}, + bool includePath = true) const final; void acceptVisitor(MatchExpressionMutableVisitor* visitor) final { visitor->visit(this); diff --git a/src/mongo/db/matcher/schema/object_keywords_test.cpp b/src/mongo/db/matcher/schema/object_keywords_test.cpp index 73bb4b7d7dd..468c0bb8cc9 100644 --- a/src/mongo/db/matcher/schema/object_keywords_test.cpp +++ b/src/mongo/db/matcher/schema/object_keywords_test.cpp @@ -256,11 +256,11 @@ TEST(JSONSchemaObjectKeywordTest, SharedJsonAndBsonTypeAliasesTranslateIdentical ASSERT_OK(bsonTypeResult.getStatus()); BSONObjBuilder typeBuilder; - MatchExpression::optimize(std::move(typeResult.getValue()))->serialize(&typeBuilder, true); + MatchExpression::optimize(std::move(typeResult.getValue()))->serialize(&typeBuilder, {}); BSONObjBuilder bsonTypeBuilder; MatchExpression::optimize(std::move(bsonTypeResult.getValue())) - ->serialize(&bsonTypeBuilder, true); + ->serialize(&bsonTypeBuilder, {}); ASSERT_BSONOBJ_EQ(typeBuilder.obj(), bsonTypeBuilder.obj()); } diff --git a/src/mongo/db/mirror_maestro.cpp b/src/mongo/db/mirror_maestro.cpp index b586b332b02..a47c50ef318 100644 --- a/src/mongo/db/mirror_maestro.cpp +++ b/src/mongo/db/mirror_maestro.cpp @@ -511,6 +511,7 @@ void MirrorMaestroImpl::shutdown() noexcept { if (_executor) { _executor->shutdown(); + _executor->join(); } // Set _initGuard.liveness to kShutdown diff --git a/src/mongo/db/mongod_main.cpp b/src/mongo/db/mongod_main.cpp index 1fbbb6f5a06..4f46f4ceb71 100644 --- a/src/mongo/db/mongod_main.cpp +++ b/src/mongo/db/mongod_main.cpp @@ -1279,7 +1279,7 @@ void logShutdownTimeElapsedStatistics(ServiceContext* serviceContext, elapsedInitAndListen.toString()); shutdownInfoBuilder->append("Statistics", shutdownTimeElapsedBuilder->obj()); LOGV2_INFO(8423404, - "shutdownTask complete", + "mongod shutdown complete", "Summary of time elapsed"_attr = shutdownInfoBuilder->obj()); } @@ -1471,7 +1471,10 @@ void shutdownTask(const ShutdownTaskArgs& shutdownArgs) { uniqueOpCtx = client->makeOperationContext(); opCtx = uniqueOpCtx.get(); } - opCtx->setIsExecutingShutdown(); + { + stdx::lock_guard lg(*client); + opCtx->setIsExecutingShutdown(); + } // This can wait a long time while we drain the secondary's apply queue, especially if // it is building an index. @@ -1647,7 +1650,7 @@ void shutdownTask(const ShutdownTaskArgs& shutdownArgs) { // Shutdown the Service Entry Point and its sessions and give it a grace period to complete. if (auto sep = serviceContext->getServiceEntryPoint()) { LOGV2_OPTIONS(4784923, {LogComponent::kCommand}, "Shutting down the ServiceEntryPoint"); - if (!sep->shutdown(Seconds(10))) { + if (!sep->shutdown(Seconds(30))) { LOGV2_OPTIONS(20563, {LogComponent::kNetwork}, "Service entry point did not shutdown within the time limit"); diff --git a/src/mongo/db/namespace_string.cpp b/src/mongo/db/namespace_string.cpp index 52e3b5a9717..374fe4e5795 100644 --- a/src/mongo/db/namespace_string.cpp +++ b/src/mongo/db/namespace_string.cpp @@ -35,6 +35,7 @@ #include "mongo/base/parse_number.h" #include "mongo/base/status.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/db/server_options.h" #include "mongo/util/str.h" @@ -278,6 +279,10 @@ std::string NamespaceString::getSisterNS(StringData local) const { return db().toString() + "." + local.toString(); } +std::string NamespaceString::toString(const SerializationOptions& opts) const { + return opts.serializeIdentifier(ns()); +} + void NamespaceString::serializeCollectionName(BSONObjBuilder* builder, StringData fieldName) const { if (isCollectionlessAggregateNS()) { builder->append(fieldName, 1); diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h index 1c4854f4ea9..0ff9b081497 100644 --- a/src/mongo/db/namespace_string.h +++ b/src/mongo/db/namespace_string.h @@ -45,6 +45,8 @@ namespace mongo { +struct SerializationOptions; + class NamespaceString { public: constexpr static size_t MaxDatabaseNameLen = @@ -308,6 +310,8 @@ public: return _ns; } + std::string toString(const SerializationOptions& opts) const; + const std::string& toString() const { return ns(); } @@ -348,6 +352,9 @@ public: bool isSystemDotViews() const { return coll() == kSystemDotViewsCollectionName; } + bool isSystemDotUsers() const { + return coll() == kSystemUsers; + } bool isServerConfigurationCollection() const { return (db() == kAdminDb) && (coll() == "system.version"); } @@ -664,6 +671,15 @@ public: void serialize(BSONObjBuilder* builder, StringData fieldName) const; + template <typename H> + friend H AbslHashValue(H h, const NamespaceStringOrUUID& nssOrUUID) { + if (nssOrUUID.nss().has_value()) { + return H::combine(std::move(h), *nssOrUUID.nss()); + } else { + return H::combine(std::move(h), *nssOrUUID.uuid()); + } + } + private: // At any given time exactly one of these optionals will be initialized. boost::optional<NamespaceString> _nss; diff --git a/src/mongo/db/op_observer/SConscript b/src/mongo/db/op_observer/SConscript index 21f062d96a3..19493f722bd 100644 --- a/src/mongo/db/op_observer/SConscript +++ b/src/mongo/db/op_observer/SConscript @@ -37,6 +37,7 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/catalog/catalog_helpers', '$BUILD_DIR/mongo/db/repl/oplog', ], ) diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp index d4f00d62f4c..56343465374 100644 --- a/src/mongo/db/op_observer_impl.cpp +++ b/src/mongo/db/op_observer_impl.cpp @@ -38,6 +38,7 @@ #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/batched_write_context.h" +#include "mongo/db/catalog/backwards_compatible_collection_options_util.h" #include "mongo/db/catalog/collection_options.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_holder.h" @@ -666,6 +667,19 @@ void OpObserverImpl::onInserts(OperationContext* opCtx, bucketsColl->setRequiresTimeseriesExtendedRangeSupport(opCtx); } } + + uassert( + ErrorCodes::CannotInsertTimeseriesBucketsWithMixedSchema, + "Cannot write time-series bucket containing mixed schema data, please ensure all nodes " + "are upgraded to the latest v6.0 release, run collMod with " + "timeseriesBucketsMayHaveMixedSchemaData, and retry your insert", + !opCtx->isEnforcingConstraints() || + bucketsColl->getTimeseriesBucketsMayHaveMixedSchemaData().value_or(false) || + std::none_of(first, last, [bucketsColl](auto&& insert) { + auto mixedSchema = + bucketsColl->doesTimeseriesBucketsDocContainMixedSchemaData(insert.doc); + return mixedSchema.isOK() && mixedSchema.getValue(); + })); } } @@ -865,6 +879,26 @@ void OpObserverImpl::onUpdate(OperationContext* opCtx, const OplogUpdateEntryArg opCtx, args.updateArgs->updatedDoc["_id"], args.updateArgs->updatedDoc); } else if (args.nss.isTimeseriesBucketsCollection()) { if (args.updateArgs->source != OperationSource::kTimeseriesInsert) { + invariant(opCtx->lockState()->isCollectionLockedForMode(args.nss, MODE_IX)); + auto bucketsColl = + CollectionCatalog::get(opCtx)->lookupCollectionByNamespaceForRead(opCtx, args.nss); + tassert(8453101, "Could not find collection for write", bucketsColl); + + auto mixedSchema = [&args, &bucketsColl] { + auto result = bucketsColl->doesTimeseriesBucketsDocContainMixedSchemaData( + args.updateArgs->updatedDoc); + return result.isOK() && result.getValue(); + }; + + uassert( + ErrorCodes::CannotInsertTimeseriesBucketsWithMixedSchema, + "Cannot write time-series bucket containing mixed schema data, please ensure all " + "nodes are upgraded to the latest v6.0 release, run collMod with " + "timeseriesBucketsMayHaveMixedSchemaData, and retry your update", + !opCtx->isEnforcingConstraints() || + bucketsColl->getTimeseriesBucketsMayHaveMixedSchemaData().value_or(false) || + !mixedSchema()); + auto& bucketCatalog = BucketCatalog::get(opCtx); bucketCatalog.clear(args.updateArgs->updatedDoc["_id"].OID()); } @@ -1121,6 +1155,8 @@ void OpObserverImpl::onCollMod(OperationContext* opCtx, const BSONObj& collModCmd, const CollectionOptions& oldCollOptions, boost::optional<IndexCollModInfo> indexInfo) { + const auto [collModOplogCmd, additionalO2Field] = + backwards_compatible_collection_options::getCollModCmdAndAdditionalO2Field(collModCmd); if (!nss.isSystemDotProfile()) { // do not replicate system.profile modifications @@ -1128,6 +1164,10 @@ void OpObserverImpl::onCollMod(OperationContext* opCtx, // Create the 'o2' field object. We save the old collection metadata and TTL expiration. BSONObjBuilder o2Builder; o2Builder.append("collectionOptions_old", oldCollOptions.toBSON()); + if (!additionalO2Field.isEmpty()) { + o2Builder.append(backwards_compatible_collection_options::additionalCollModO2Field, + additionalO2Field); + } if (indexInfo) { BSONObjBuilder oldIndexOptions; if (indexInfo->oldExpireAfterSeconds) { @@ -1150,7 +1190,7 @@ void OpObserverImpl::onCollMod(OperationContext* opCtx, oplogEntry.setOpType(repl::OpTypeEnum::kCommand); oplogEntry.setNss(nss.getCommandNS()); oplogEntry.setUuid(uuid); - oplogEntry.setObject(repl::makeCollModCmdObj(collModCmd, oldCollOptions, indexInfo)); + oplogEntry.setObject(repl::makeCollModCmdObj(collModOplogCmd, oldCollOptions, indexInfo)); oplogEntry.setObject2(o2Builder.done()); logOperation(opCtx, &oplogEntry); } diff --git a/src/mongo/db/operation_context.cpp b/src/mongo/db/operation_context.cpp index 66fbfad5b6b..f4272010fa5 100644 --- a/src/mongo/db/operation_context.cpp +++ b/src/mongo/db/operation_context.cpp @@ -400,7 +400,10 @@ void OperationContext::setIsExecutingShutdown() { _isExecutingShutdown = true; - pushIgnoreInterrupts(); + // The OperationContext executing shutdown is immune from interruption. + _hasArtificialDeadline = true; + setDeadlineByDate(Date_t::max(), ErrorCodes::ExceededTimeLimit); + _ignoreInterrupts = true; } void OperationContext::setLogicalSessionId(LogicalSessionId lsid) { diff --git a/src/mongo/db/operation_context.h b/src/mongo/db/operation_context.h index be234281d8b..2ebbf8f4d5d 100644 --- a/src/mongo/db/operation_context.h +++ b/src/mongo/db/operation_context.h @@ -29,7 +29,9 @@ #pragma once +#include "mongo/util/assert_util.h" #include <boost/optional.hpp> +#include <cstddef> #include <memory> #include "mongo/base/status.h" @@ -343,7 +345,8 @@ public: * global shutdown. * * This should only be called from the registered task of global shutdown and is not - * recoverable. + * recoverable. May only be called by the thread executing on behalf of this OperationContext, + * and only while it has the Client that owns this OperationContext locked. */ void setIsExecutingShutdown(); @@ -565,6 +568,10 @@ public: return _comment ? boost::optional<BSONElement>(_comment->firstElement()) : boost::none; } + boost::optional<BSONObj> getCommentOwnedCopy() const { + return _comment.has_value() ? boost::optional<BSONObj>{_comment->copy()} : boost::none; + } + /** * Sets whether this operation is an exhaust command. */ @@ -588,29 +595,50 @@ public: */ void restoreMaxTimeMS(); + /** + * Invokes the passed callback while ignoring interrupts. Note that this causes the deadline to + * be reset to Date_t::max(), but that it can also subsequently be reduced in size after the + * fact. Additionally handles the dance of try/catching the invocation and checking + * checkForInterrupt with the guard inactive (to allow a higher level timeout to override a + * lower level one, or for top level interruption to propagate). + * + * This should only be called from the thread executing on behalf of this OperationContext. + * The Client for this OperationContext should not be locked by the thread calling this + * function, as this function will acquire the lock internally to modify the OperationContext's + * interrupt state. + */ + template <typename Callback> + decltype(auto) runWithoutInterruptionExceptAtGlobalShutdown(Callback&& cb) { + try { + bool prevIgnoringInterrupts = _ignoreInterrupts; + DeadlineState prevDeadlineState{_deadline, _timeoutError, _hasArtificialDeadline}; + ScopeGuard guard([&] { + // Restore the original interruption and deadline state. + stdx::lock_guard lg(*_client); + _ignoreInterrupts = prevIgnoringInterrupts; + setDeadlineByDate(prevDeadlineState.deadline, prevDeadlineState.error); + _hasArtificialDeadline = prevDeadlineState.hasArtificialDeadline; + _markKilledIfDeadlineRequires(); + }); + // Ignore interrupts until the callback completes. + { + stdx::lock_guard lg(*_client); + _hasArtificialDeadline = true; + setDeadlineByDate(Date_t::max(), ErrorCodes::ExceededTimeLimit); + _ignoreInterrupts = true; + } + return std::forward<Callback>(cb)(); + } catch (const ExceptionForCat<ErrorCategory::ExceededTimeLimitError>&) { + // May throw replacement exception + checkForInterrupt(); + throw; + } + } + private: StatusWith<stdx::cv_status> waitForConditionOrInterruptNoAssertUntil( stdx::condition_variable& cv, BasicLockableAdapter m, Date_t deadline) noexcept override; - IgnoreInterruptsState pushIgnoreInterrupts() override { - IgnoreInterruptsState iis{_ignoreInterrupts, - {_deadline, _timeoutError, _hasArtificialDeadline}}; - _hasArtificialDeadline = true; - setDeadlineByDate(Date_t::max(), ErrorCodes::ExceededTimeLimit); - _ignoreInterrupts = true; - - return iis; - } - - void popIgnoreInterrupts(IgnoreInterruptsState iis) override { - _ignoreInterrupts = iis.ignoreInterrupts; - - setDeadlineByDate(iis.deadline.deadline, iis.deadline.error); - _hasArtificialDeadline = iis.deadline.hasArtificialDeadline; - - _markKilledIfDeadlineRequires(); - } - DeadlineState pushArtificialDeadline(Date_t deadline, ErrorCodes::Error error) override { DeadlineState ds{_deadline, _timeoutError, _hasArtificialDeadline}; diff --git a/src/mongo/db/ops/SConscript b/src/mongo/db/ops/SConscript index 3e89ae63e36..9ac3c43f9fd 100644 --- a/src/mongo/db/ops/SConscript +++ b/src/mongo/db/ops/SConscript @@ -15,9 +15,9 @@ env.Library( '$BUILD_DIR/mongo/db/catalog/collection_options', '$BUILD_DIR/mongo/db/catalog_raii', '$BUILD_DIR/mongo/db/concurrency/exception_util', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/curop_metrics', '$BUILD_DIR/mongo/db/dbhelpers', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/oplog', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', diff --git a/src/mongo/db/ops/insert.cpp b/src/mongo/db/ops/insert.cpp index 61d92fda6bf..8c949b333c7 100644 --- a/src/mongo/db/ops/insert.cpp +++ b/src/mongo/db/ops/insert.cpp @@ -87,6 +87,7 @@ Status validateDepth(const BSONObj& obj) { StatusWith<BSONObj> fixDocumentForInsert(OperationContext* opCtx, const BSONObj& doc, + bool bypassEmptyTsReplacement, bool* containsDotsAndDollarsField) { bool validationDisabled = DocumentValidationSettings::get(opCtx).isInternalValidationDisabled(); @@ -139,7 +140,8 @@ StatusWith<BSONObj> fixDocumentForInsert(OperationContext* opCtx, } if (!validationDisabled) { - if (e.type() == bsonTimestamp && e.timestampValue() == 0) { + if (!bypassEmptyTsReplacement && e.type() == bsonTimestamp && + e.timestampValue() == 0) { // we replace Timestamp(0,0) at the top level with a correct value // in the fast pass, we just mark that we want to swap hasTimestampToFix = true; @@ -187,7 +189,8 @@ StatusWith<BSONObj> fixDocumentForInsert(OperationContext* opCtx, BSONElement e = i.next(); if (hadId && e.fieldNameStringData() == "_id") { // no-op - } else if (e.type() == bsonTimestamp && e.timestampValue() == 0) { + } else if (!bypassEmptyTsReplacement && e.type() == bsonTimestamp && + e.timestampValue() == 0) { auto nextTime = VectorClockMutable::get(opCtx)->tickClusterTime(1); b.append(e.fieldName(), nextTime.asTimestamp()); } else { diff --git a/src/mongo/db/ops/insert.h b/src/mongo/db/ops/insert.h index faed6de5890..a523e499d02 100644 --- a/src/mongo/db/ops/insert.h +++ b/src/mongo/db/ops/insert.h @@ -47,6 +47,7 @@ class OperationContext; */ StatusWith<BSONObj> fixDocumentForInsert(OperationContext* opCtx, const BSONObj& doc, + bool bypassEmptyTsReplacement = false, bool* containsDotsOrDollarsField = nullptr); /** diff --git a/src/mongo/db/ops/parsed_update.cpp b/src/mongo/db/ops/parsed_update.cpp index b9557e4b2fe..a2badffa2aa 100644 --- a/src/mongo/db/ops/parsed_update.cpp +++ b/src/mongo/db/ops/parsed_update.cpp @@ -188,6 +188,7 @@ void ParsedUpdate::parseUpdate() { _driver.setCollator(_expCtx->getCollator()); _driver.setLogOp(true); _driver.setFromOplogApplication(_request->isFromOplogApplication()); + _driver.setBypassEmptyTsReplacement(static_cast<bool>(_request->getBypassEmptyTsReplacement())); // Time-series operations will not result in any documents with dots or dollars fields. if (auto source = _request->source(); source == OperationSource::kTimeseriesInsert || source == OperationSource::kTimeseriesUpdate) { diff --git a/src/mongo/db/ops/update_request.h b/src/mongo/db/ops/update_request.h index 3db331defa9..2a086d8a4ab 100644 --- a/src/mongo/db/ops/update_request.h +++ b/src/mongo/db/ops/update_request.h @@ -209,6 +209,14 @@ public: return _fromOplogApplication; } + void setBypassEmptyTsReplacement(OptionalBool bypassEmptyTsReplacement) { + _bypassEmptyTsReplacement = bypassEmptyTsReplacement; + } + + OptionalBool getBypassEmptyTsReplacement() const { + return _bypassEmptyTsReplacement; + } + void setExplain(boost::optional<ExplainOptions::Verbosity> verbosity) { _explain = verbosity; } @@ -314,6 +322,8 @@ private: // The statement ids of this request. std::vector<StmtId> _stmtIds = {kUninitializedStmtId}; + OptionalBool _bypassEmptyTsReplacement; + // Flags controlling the update. // God bypasses _id checking and index generation. It is only used on behalf of system diff --git a/src/mongo/db/ops/write_ops.cpp b/src/mongo/db/ops/write_ops.cpp index de7df85e564..14cf48fca37 100644 --- a/src/mongo/db/ops/write_ops.cpp +++ b/src/mongo/db/ops/write_ops.cpp @@ -131,6 +131,12 @@ int getWriteCommandRequestBaseSize(const WriteCommandRequestBase& base) { encryptionInfo->toBSON().objsize() + kPerElementOverhead; } + if (auto bypassEmptyTsReplacement = base.getBypassEmptyTsReplacement(); + bypassEmptyTsReplacement.has_value()) { + estSize += write_ops::WriteCommandRequestBase::kBypassEmptyTsReplacementFieldName.size() + + kBoolSize + kPerElementOverhead; + } + return estSize; } diff --git a/src/mongo/db/ops/write_ops.idl b/src/mongo/db/ops/write_ops.idl index ca606f3cfad..e41af05e808 100644 --- a/src/mongo/db/ops/write_ops.idl +++ b/src/mongo/db/ops/write_ops.idl @@ -205,6 +205,12 @@ structs: type: EncryptionInformation optional: true unstable: true + bypassEmptyTsReplacement: + description: "Only applicable for inserts and replacement updates. If set to true, + any empty timestamps (Timestamp(0,0)) in 'documents' or 'u' will not + be replaced by the current time and instead will be preserved as-is." + type: optionalBool + unstable: true UpdateOpEntry: description: "Parser for the entries in the 'updates' array of an update command." @@ -539,3 +545,9 @@ commands: type: EncryptionInformation optional: true unstable: true + bypassEmptyTsReplacement: + description: "Only applicable when 'update' is a replacement update. If set, any + empty timestamps (Timestamp(0, 0)) in the update will not be replaced + by the current time and instead will be preserved as-is." + type: optionalBool + unstable: true diff --git a/src/mongo/db/ops/write_ops_exec.cpp b/src/mongo/db/ops/write_ops_exec.cpp index 05e78ecfca6..e58146ac5cd 100644 --- a/src/mongo/db/ops/write_ops_exec.cpp +++ b/src/mongo/db/ops/write_ops_exec.cpp @@ -96,46 +96,6 @@ #include "mongo/util/scopeguard.h" namespace mongo::write_ops_exec { -class Atomic64Metric; -} // namespace mongo::write_ops_exec - -namespace mongo { -template <> -struct BSONObjAppendFormat<write_ops_exec::Atomic64Metric> : FormatKind<NumberLong> {}; -} // namespace mongo - - -namespace mongo::write_ops_exec { - -/** - * Atomic wrapper for long long type for Metrics. - */ -class Atomic64Metric { -public: - /** Set _value to the max of the current or newMax. */ - void setIfMax(long long newMax) { - /* Note: compareAndSwap will load into val most recent value. */ - for (long long val = _value.load(); val < newMax && !_value.compareAndSwap(&val, newMax);) { - } - } - - /** store val into value. */ - void set(long long val) { - _value.store(val); - } - - /** Return the current value. */ - long long get() const { - return _value.load(); - } - - operator long long() const { - return get(); - } - -private: - mongo::AtomicWord<long long> _value; -}; // Convention in this file: generic helpers go in the anonymous namespace. Helpers that are for a // single type of operation are static functions defined above their caller. @@ -214,7 +174,7 @@ void finishCurOp(OperationContext* opCtx, CurOp* curOp) { try { curOp->done(); auto executionTimeMicros = duration_cast<Microseconds>(curOp->elapsedTimeExcludingPauses()); - curOp->debug().executionTime = executionTimeMicros; + curOp->debug().additiveMetrics.executionTime = executionTimeMicros; recordCurOpMetrics(opCtx); Top::get(opCtx->getServiceContext()) @@ -713,7 +673,6 @@ bool getFleCrudProcessed(OperationContext* opCtx, WriteResult performInserts(OperationContext* opCtx, const write_ops::InsertCommandRequest& wholeOp, OperationSource source) { - // Insert performs its own retries, so we should only be within a WriteUnitOfWork when run in a // transaction. auto txnParticipant = TransactionParticipant::get(opCtx); @@ -772,11 +731,20 @@ WriteResult performInserts(OperationContext* opCtx, const size_t maxBatchBytes = write_ops::insertVectorMaxBytes; batch.reserve(std::min(wholeOp.getDocuments().size(), maxBatchSize)); + // If 'wholeOp.getBypassEmptyTsReplacement()' is true or if 'source' is 'kFromMigrate', set + // "bypassEmptyTsReplacement=true" for fixDocumentForInsert(). + const bool bypassEmptyTsReplacement = (source == OperationSource::kFromMigrate) || + static_cast<bool>(wholeOp.getBypassEmptyTsReplacement()); + for (auto&& doc : wholeOp.getDocuments()) { const bool isLastDoc = (&doc == &wholeOp.getDocuments().back()); bool containsDotsAndDollarsField = false; - auto fixedDoc = fixDocumentForInsert(opCtx, doc, &containsDotsAndDollarsField); + + auto fixedDoc = fixDocumentForInsert( + opCtx, doc, bypassEmptyTsReplacement, &containsDotsAndDollarsField); + const StmtId stmtId = getStmtIdForWriteOp(opCtx, wholeOp, stmtIdIndex++); + const bool wasAlreadyExecuted = opCtx->isRetryableWrite() && txnParticipant.checkStatementExecutedNoOplogEntryFetch(opCtx, stmtId); @@ -1018,6 +986,7 @@ static SingleWriteResult performSingleUpdateOpWithDupKeyRetry( const write_ops::UpdateOpEntry& op, LegacyRuntimeConstants runtimeConstants, const boost::optional<BSONObj>& letParams, + const OptionalBool& bypassEmptyTsReplacement, OperationSource source, bool forgoOpCounterIncrements) { globalOpCounters.gotUpdate(); @@ -1046,6 +1015,7 @@ static SingleWriteResult performSingleUpdateOpWithDupKeyRetry( if (letParams) { request.setLetParameters(std::move(letParams)); } + request.setBypassEmptyTsReplacement(bypassEmptyTsReplacement); request.setStmtIds(stmtIds); request.setYieldPolicy(PlanYieldPolicy::YieldPolicy::YIELD_AUTO); request.setSource(source); @@ -1187,6 +1157,7 @@ WriteResult performUpdates(OperationContext* opCtx, singleOp, runtimeConstants, wholeOp.getLet(), + wholeOp.getBypassEmptyTsReplacement(), source, forgoOpCounterIncrements); out.results.emplace_back(reply); diff --git a/src/mongo/db/periodic_runner_job_abort_expired_transactions.cpp b/src/mongo/db/periodic_runner_job_abort_expired_transactions.cpp index 52568382913..7e10ea049ff 100644 --- a/src/mongo/db/periodic_runner_job_abort_expired_transactions.cpp +++ b/src/mongo/db/periodic_runner_job_abort_expired_transactions.cpp @@ -61,6 +61,12 @@ Milliseconds getPeriod(const Argument& transactionLifetimeLimitSeconds) { } // namespace +// Tracks the number of passes the "abortExpiredTransactions" thread makes to abort expired +// transactions. +static Counter64 abortExpiredTransactionsPasses; +static ServerStatusMetricField<Counter64> dAbortExpiredTransactionsPasses( + "abortExpiredTransactions.passes", &abortExpiredTransactionsPasses); + auto PeriodicThreadToAbortExpiredTransactions::get(ServiceContext* serviceContext) -> PeriodicThreadToAbortExpiredTransactions& { auto& jobContainer = _serviceDecoration(serviceContext); @@ -109,6 +115,7 @@ void PeriodicThreadToAbortExpiredTransactions::_init(ServiceContext* serviceCont try { killAllExpiredTransactions(opCtx.get()); + abortExpiredTransactionsPasses.increment(); } catch (ExceptionForCat<ErrorCategory::CancellationError>& ex) { LOGV2_DEBUG(4684101, 2, "Periodic job canceled", "{reason}"_attr = ex.reason()); } diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index f5ad3d7cdb7..f3b54ed67a3 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -63,12 +63,10 @@ env.Library( ) env.Library( - target='aggregation_request_helper', - source=[ - 'aggregation_request_helper.cpp', + target='aggregation_request_helper', source=[ 'aggregate_command.idl', - ], - LIBDEPS=[ + 'aggregation_request_helper.cpp', + ], LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/exec/document_value/document_value', '$BUILD_DIR/mongo/db/namespace_string', @@ -80,8 +78,7 @@ env.Library( '$BUILD_DIR/mongo/db/write_concern_options', '$BUILD_DIR/mongo/idl/idl_parser', 'document_sources_idl', - ] -) + ]) env.Library( target='variable_validation', @@ -95,6 +92,7 @@ env.Library( env.Library( target='expression_context', source=[ + '$BUILD_DIR/mongo/db/query/query_shape/cmd_with_let_shape.cpp', 'expression.cpp', 'expression_context.cpp', 'expression_function.cpp', @@ -115,6 +113,7 @@ env.Library( '$BUILD_DIR/mongo/db/query/collation/collator_factory_interface', '$BUILD_DIR/mongo/db/query/datetime/date_time_support', '$BUILD_DIR/mongo/db/query/query_knobs', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape_common', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/stats/counters', @@ -274,6 +273,8 @@ pipelineEnv.Library( 'abt/field_map_builder.cpp', 'abt/match_expression_visitor.cpp', 'abt/utils.cpp', + '$BUILD_DIR/mongo/db/query/query_shape/agg_cmd_shape.cpp', + '$BUILD_DIR/mongo/db/query/query_stats/agg_key.cpp', 'document_source.cpp', 'document_source_add_fields.cpp', 'document_source_bucket.cpp', @@ -298,6 +299,7 @@ pipelineEnv.Library( 'document_source_internal_inhibit_optimization.cpp', 'document_source_internal_shard_filter.cpp', 'document_source_internal_shred_documents.cpp', + 'document_source_internal_shardserver_info.cpp', 'document_source_internal_split_pipeline.cpp', 'document_source_internal_unpack_bucket.cpp', 'document_source_limit.cpp', @@ -312,6 +314,7 @@ pipelineEnv.Library( 'document_source_out.cpp', 'document_source_plan_cache_stats.cpp', 'document_source_project.cpp', + 'document_source_query_stats.cpp', 'document_source_queue.cpp', 'document_source_redact.cpp', 'document_source_replace_root.cpp', @@ -350,7 +353,6 @@ pipelineEnv.Library( '$BUILD_DIR/mongo/client/clientdriver_minimal', '$BUILD_DIR/mongo/db/auth/auth', '$BUILD_DIR/mongo/db/bson/dotted_path_support', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/curop_failpoint_helpers', '$BUILD_DIR/mongo/db/exec/bucket_unpacker', '$BUILD_DIR/mongo/db/exec/document_value/document_value', @@ -483,11 +485,14 @@ env.Library( 'document_source_merge_modes.idl', 'document_source_merge_spec.cpp', 'document_source_parsing_validators.cpp', + 'document_source_query_stats.idl', + 'document_source_query_stats_validators.cpp', 'document_source_replace_root.idl', 'document_source_set_window_fields.idl', 'document_source_union_with.idl', 'document_source_set_variable_from_subpipeline.idl', 'exchange_spec.idl', + 'resume_token.idl', 'resume_token.cpp', 'storage_stats_spec.idl', 'value.idl', @@ -495,6 +500,8 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/exec/document_value/document_value', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats_parse', '$BUILD_DIR/mongo/db/storage/key_string', '$BUILD_DIR/mongo/idl/idl_parser', '$BUILD_DIR/mongo/s/common_s', @@ -580,9 +587,11 @@ env.CppUnitTest( 'document_source_change_stream_add_post_image_test.cpp', 'document_source_change_stream_test.cpp', 'document_source_check_resume_token_test.cpp', + 'document_source_coll_stats_test.cpp', 'document_source_count_test.cpp', 'document_source_current_op_test.cpp', 'document_source_densify_test.cpp', + 'document_source_documents_test.cpp', 'document_source_exchange_test.cpp', 'document_source_facet_test.cpp', 'document_source_find_and_modify_image_lookup_test.cpp', @@ -611,8 +620,11 @@ env.CppUnitTest( 'document_source_skip_test.cpp', 'document_source_sort_by_count_test.cpp', 'document_source_sort_test.cpp', + 'document_source_query_stats_test.cpp', 'document_source_union_with_test.cpp', + 'document_source_internal_all_collection_stats_test.cpp', 'document_source_internal_compute_geo_near_distance_test.cpp', + 'document_source_internal_convert_bucket_index_stats_test.cpp', 'document_source_internal_unpack_bucket_test/extract_or_build_project_to_internalize_test.cpp', 'document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp', 'document_source_internal_unpack_bucket_test/extract_project_for_pushdown_test.cpp', @@ -633,7 +645,9 @@ env.CppUnitTest( 'expression_date_test.cpp', 'expression_field_path_test.cpp', 'expression_find_internal_test.cpp', + 'expression_function_test.cpp', 'expression_javascript_test.cpp' if get_option('js-engine') != 'none' else [], + 'expression_let_test.cpp', 'expression_nary_test.cpp', 'expression_object_test.cpp', 'expression_or_test.cpp', diff --git a/src/mongo/db/pipeline/abt/abt_document_source_visitor.cpp b/src/mongo/db/pipeline/abt/abt_document_source_visitor.cpp index 1d136a098ce..a2bf9a28656 100644 --- a/src/mongo/db/pipeline/abt/abt_document_source_visitor.cpp +++ b/src/mongo/db/pipeline/abt/abt_document_source_visitor.cpp @@ -59,6 +59,7 @@ #include "mongo/db/pipeline/document_source_operation_metrics.h" #include "mongo/db/pipeline/document_source_out.h" #include "mongo/db/pipeline/document_source_plan_cache_stats.h" +#include "mongo/db/pipeline/document_source_query_stats.h" #include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/document_source_redact.h" #include "mongo/db/pipeline/document_source_replace_root.h" @@ -606,6 +607,10 @@ public: unsupportedStage(source); } + void visit(const DocumentSourceQueryStats* source) override { + unsupportedStage(source); + } + void visit(const DocumentSourceUnionWith* source) override { auto entry = _ctx.getNode(); ProjectionName unionProjName = entry._rootProjection; diff --git a/src/mongo/db/pipeline/accumulator.h b/src/mongo/db/pipeline/accumulator.h index 114ccc260ce..0fbc3c60baf 100644 --- a/src/mongo/db/pipeline/accumulator.h +++ b/src/mongo/db/pipeline/accumulator.h @@ -42,6 +42,7 @@ #include "mongo/db/exec/document_value/value_comparator.h" #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/stdx/unordered_set.h" #include "mongo/util/summation.h" @@ -134,12 +135,12 @@ public: */ virtual Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options = {}) const { ExpressionConstant const* ec = dynamic_cast<ExpressionConstant const*>(initializer.get()); invariant(ec); invariant(ec->getValue().nullish()); - return DOC(getOpName() << argument->serialize(explain)); + return DOC(getOpName() << argument->serialize(options)); } virtual AccumulatorDocumentsNeeded documentsNeeded() const { diff --git a/src/mongo/db/pipeline/accumulator_js_reduce.cpp b/src/mongo/db/pipeline/accumulator_js_reduce.cpp index 78058d5913d..bd8c5ccf882 100644 --- a/src/mongo/db/pipeline/accumulator_js_reduce.cpp +++ b/src/mongo/db/pipeline/accumulator_js_reduce.cpp @@ -196,8 +196,9 @@ void AccumulatorInternalJsReduce::reset() { // Returns this accumulator serialized as a Value along with the reduce function. Document AccumulatorInternalJsReduce::serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { - return DOC(kName << DOC("data" << argument->serialize(explain) << "eval" << _funcSource)); + const SerializationOptions& options) const { + return DOC(kName << DOC("data" << argument->serialize(options) << "eval" + << options.serializeLiteral(_funcSource))); } REGISTER_ACCUMULATOR(accumulator, AccumulatorJs::parse); @@ -235,15 +236,16 @@ std::string parseFunction(StringData fieldName, Document AccumulatorJs::serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options) const { MutableDocument args; - args.addField("init", Value(_init)); - args.addField("initArgs", Value(initializer->serialize(explain))); - args.addField("accumulate", Value(_accumulate)); - args.addField("accumulateArgs", Value(argument->serialize(explain))); - args.addField("merge", Value(_merge)); + + args.addField("init", options.serializeLiteral(_init)); + args.addField("initArgs", initializer->serialize(options)); + args.addField("accumulate", options.serializeLiteral(_accumulate)); + args.addField("accumulateArgs", argument->serialize(options)); + args.addField("merge", options.serializeLiteral(_merge)); if (_finalize) { - args.addField("finalize", Value(*_finalize)); + args.addField("finalize", options.serializeLiteral(*_finalize)); } args.addField("lang", Value("js"_sd)); return DOC(kName << args.freeze()); diff --git a/src/mongo/db/pipeline/accumulator_js_reduce.h b/src/mongo/db/pipeline/accumulator_js_reduce.h index af025132ee0..9e3a6e63d40 100644 --- a/src/mongo/db/pipeline/accumulator_js_reduce.h +++ b/src/mongo/db/pipeline/accumulator_js_reduce.h @@ -66,7 +66,7 @@ public: virtual Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const override; + const SerializationOptions& options = {}) const override; private: static std::string parseReduceFunction(BSONElement func); @@ -101,10 +101,9 @@ public: void processInternal(const Value& input, bool merging) final; void reduceMemoryConsumptionIfAble() final; - Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const final; + const SerializationOptions& options = {}) const final; void startNewGroup(Value const& input) final; private: diff --git a/src/mongo/db/pipeline/accumulator_multi.cpp b/src/mongo/db/pipeline/accumulator_multi.cpp index d488bd8789c..26cfa570d91 100644 --- a/src/mongo/db/pipeline/accumulator_multi.cpp +++ b/src/mongo/db/pipeline/accumulator_multi.cpp @@ -178,9 +178,9 @@ const char* AccumulatorMinMaxN::getOpName() const { Document AccumulatorMinMaxN::serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options) const { MutableDocument args; - AccumulatorN::serializeHelper(initializer, argument, explain, args); + AccumulatorN::serializeHelper(initializer, argument, options, args); return DOC(getOpName() << args.freeze()); } @@ -230,10 +230,10 @@ void AccumulatorN::updateAndCheckMemUsage(size_t memAdded) { void AccumulatorN::serializeHelper(const boost::intrusive_ptr<Expression>& initializer, const boost::intrusive_ptr<Expression>& argument, - bool explain, + const SerializationOptions& options, MutableDocument& md) { - md.addField(kFieldNameN, Value(initializer->serialize(explain))); - md.addField(kFieldNameInput, Value(argument->serialize(explain))); + md.addField(kFieldNameN, Value(initializer->serialize(options))); + md.addField(kFieldNameInput, Value(argument->serialize(options))); } template <MinMaxSense s> @@ -385,9 +385,9 @@ const char* AccumulatorFirstLastN::getOpName() const { Document AccumulatorFirstLastN::serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options) const { MutableDocument args; - AccumulatorN::serializeHelper(initializer, argument, explain, args); + AccumulatorN::serializeHelper(initializer, argument, options, args); return DOC(getOpName() << args.freeze()); } @@ -531,32 +531,46 @@ template <TopBottomSense sense, bool single> Document AccumulatorTopBottomN<sense, single>::serialize( boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options) const { MutableDocument args; if constexpr (!single) { - args.addField(kFieldNameN, Value(initializer->serialize(explain))); + args.addField(kFieldNameN, Value(initializer->serialize(options))); } - auto serializedArg = argument->serialize(explain); - - // If 'argument' contains a field named 'output', this means that we are serializing the - // accumulator's original output expression under the field name 'output'. Otherwise, we are - // serializing a custom argument under the field name 'output'. For instance, a merging $group - // will provide an argument that merges multiple partial groups. - if (auto output = serializedArg[kFieldNameOutput]; !output.missing()) { - args.addField(kFieldNameOutput, Value(output)); + + // If 'argument' is either an ExpressionObject or an ExpressionConstant of object type, then + // we are serializing the original expression under the 'output' field of the object. Otherwise, + // we're serializing a custom expression for merging group. + if (auto argObj = dynamic_cast<ExpressionObject*>(argument.get())) { + bool foundOutputField = false; + for (auto& child : argObj->getChildExpressions()) { + if (child.first == kFieldNameOutput) { + auto output = child.second->serialize(options); + args.addField(kFieldNameOutput, output); + foundOutputField = true; + break; + } + } + tassert(7773700, "'output' field should be present.", foundOutputField); + } else if (auto argConst = dynamic_cast<ExpressionConstant*>(argument.get())) { + auto output = argConst->getValue().getDocument()[kFieldNameOutput]; + tassert(7773701, "'output' field should be present.", !output.missing()); + args.addField(kFieldNameOutput, output); } else { + auto serializedArg = argument->serialize(options); args.addField(kFieldNameOutput, serializedArg); } + args.addField(kFieldNameSortBy, Value(_sortPattern.serialize( - SortPattern::SortKeySerialization::kForPipelineSerialization))); + SortPattern::SortKeySerialization::kForPipelineSerialization, options))); return DOC(getOpName() << args.freeze()); } template <TopBottomSense sense> std::pair<SortPattern, BSONArray> parseAccumulatorTopBottomNSortBy(ExpressionContext* const expCtx, BSONObj sortBy) { + SortPattern sortPattern(sortBy, expCtx); BSONArrayBuilder sortFieldsExpBab; BSONObjIterator sortByBoi(sortBy); @@ -568,7 +582,7 @@ std::pair<SortPattern, BSONArray> parseAccumulatorTopBottomNSortBy(ExpressionCon // since the evaluated argument wouldn't have the same metadata as the original // document. Instead we use [{$meta: "textScore"}] as the sortFields expression so the // sortFields array contains the data we need for sorting. - const auto serialized = part.expression->serialize(false); + const auto serialized = part.expression->serialize(); sortFieldsExpBab.append(serialized.getDocument().toBson()); } else { sortFieldsExpBab.append((StringBuilder() << "$" << fieldName).str()); @@ -581,10 +595,8 @@ template <TopBottomSense sense, bool single> AccumulationExpression AccumulatorTopBottomN<sense, single>::parseTopBottomN( ExpressionContext* const expCtx, BSONElement elem, VariablesParseState vps) { auto name = AccumulatorTopBottomN<sense, single>::getName(); - const auto [n, output, sortBy] = accumulatorNParseArgs<single>(expCtx, elem, name.rawData(), true, vps); - auto [sortPattern, sortFieldsExp] = parseAccumulatorTopBottomNSortBy<sense>(expCtx, *sortBy); expCtx->sbeGroupCompatible = false; diff --git a/src/mongo/db/pipeline/accumulator_multi.h b/src/mongo/db/pipeline/accumulator_multi.h index 13e7971ee0c..32ced5b8c4e 100644 --- a/src/mongo/db/pipeline/accumulator_multi.h +++ b/src/mongo/db/pipeline/accumulator_multi.h @@ -34,6 +34,7 @@ #include "mongo/db/exec/sort_key_comparator.h" #include "mongo/db/index/sort_key_generator.h" #include "mongo/db/pipeline/accumulation_statement.h" +#include "mongo/db/query/query_shape/serialization_options.h" namespace mongo { @@ -85,7 +86,7 @@ public: */ static void serializeHelper(const boost::intrusive_ptr<Expression>& initializer, const boost::intrusive_ptr<Expression>& argument, - bool explain, + const SerializationOptions& options, MutableDocument& md); protected: @@ -136,7 +137,7 @@ public: Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const final; + const SerializationOptions& options = {}) const final; void reset() final; @@ -207,7 +208,7 @@ public: Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const final; + const SerializationOptions& options = {}) const final; void reset() final; @@ -322,7 +323,7 @@ public: Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const final; + const SerializationOptions& options = {}) const final; void reset() final; diff --git a/src/mongo/db/pipeline/accumulator_test.cpp b/src/mongo/db/pipeline/accumulator_test.cpp index b4665b6739c..3f3a4ba9385 100644 --- a/src/mongo/db/pipeline/accumulator_test.cpp +++ b/src/mongo/db/pipeline/accumulator_test.cpp @@ -39,10 +39,12 @@ #include "mongo/db/pipeline/accumulation_statement.h" #include "mongo/db/pipeline/accumulator.h" #include "mongo/db/pipeline/accumulator_for_window_functions.h" +#include "mongo/db/pipeline/accumulator_js_reduce.h" #include "mongo/db/pipeline/accumulator_multi.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/collation/collator_interface_mock.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/dbtests/dbtests.h" #include "mongo/idl/server_parameter_test_util.h" #include "mongo/logv2/log.h" @@ -1869,6 +1871,199 @@ TEST(Accumulators, CovarianceWithRandomVariables) { assertCovariance<AccumulatorCovarianceSamp>(&expCtx, randomVariables, boost::none); } +Value parseAndSerializeAccumExpr( + const BSONObj& obj, + std::function<boost::intrusive_ptr<Expression>( + ExpressionContext* expCtx, BSONElement, const VariablesParseState&)> func) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto expr = func(expCtx.get(), obj.firstElement(), expCtx->variablesParseState); + return expr->serialize(options); +} + +Document parseAndSerializeAccum( + const BSONElement elem, + std::function<AccumulationExpression( + ExpressionContext* const expCtx, BSONElement, VariablesParseState)> func) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto expCtx = make_intrusive<ExpressionContextForTest>(); + VariablesParseState vps = expCtx->variablesParseState; + + auto expr = func(expCtx.get(), elem, vps); + auto accum = expr.factory(); + return accum->serialize(expr.initializer, expr.argument, options); +} + +Document parseAndSerializeAccumRepresentative( + const BSONElement elem, + std::function<AccumulationExpression( + ExpressionContext* const expCtx, BSONElement, VariablesParseState)> func) { + SerializationOptions options = SerializationOptions::kRepresentativeQueryShapeSerializeOptions; + auto expCtx = make_intrusive<ExpressionContextForTest>(); + VariablesParseState vps = expCtx->variablesParseState; + + auto expr = func(expCtx.get(), elem, vps); + auto accum = expr.factory(); + return accum->serialize(expr.initializer, expr.argument, options); +} + +TEST(Accumulators, SerializeWithRedaction) { + auto jsReduce = + BSON("$accumulator" << BSON("init" + << "function() {}" + << "accumulateArgs" + << BSON_ARRAY("$a" + << "$b") + << "accumulate" + << "function(state, str1, str2) {return str1 + str2;}" + << "merge" + << "function(s1, s2) {return s1 || s2;}" + << "lang" + << "js")); + auto actual = parseAndSerializeAccum(jsReduce.firstElement(), &AccumulatorJs::parse); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$accumulator": { + "init": "?string", + "initArgs": "[]", + "accumulate": "?string", + "accumulateArgs": [ + "$HASH<a>", + "$HASH<b>" + ], + "merge": "?string", + "lang": "js" + } + })", + actual); + + auto topN = BSON("$topN" << BSON("n" << 3 << "output" + << "$output" + << "sortBy" << BSON("sortKey" << 1))); + actual = parseAndSerializeAccum( + topN.firstElement(), &AccumulatorTopBottomN<TopBottomSense::kTop, false>::parseTopBottomN); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$topN": { + "n": "?number", + "output": "$HASH<output>", + "sortBy": { + "HASH<sortKey>": 1 + } + } + })", + actual); + + auto addToSet = BSON("$addToSet" << BSON("a" << 5)); + actual = parseAndSerializeAccum(addToSet.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorAddToSet>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$addToSet":"?object"})", + actual); + + auto sum = BSON("$sum" << BSON_ARRAY(4 << 6)); + actual = parseAndSerializeAccum(sum.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorSum>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$sum": "?array<?number>"})", + actual); + + sum = BSON("$sum" << BSON_ARRAY("$a" << 5 << 3 << BSON("$sum" << BSON_ARRAY(4 << 6)))); + actual = parseAndSerializeAccum(sum.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorSum>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$sum":["$HASH<a>","?number","?number",{"$sum":"?array<?number>"}]})", + actual); + + auto mergeObjs = BSON("$mergeObjects" << BSON_ARRAY("$a" << BSON("b" + << "null"))); + actual = + parseAndSerializeAccum(mergeObjs.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorMergeObjects>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$mergeObjects":["$HASH<a>","?object"]})", + actual); + + auto push = BSON("$push" << BSON("$eq" << BSON_ARRAY("$str" + << "str2"))); + actual = parseAndSerializeAccum(push.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorPush>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$push":{"$eq":["$HASH<str>","?string"]}})", + actual); + + auto top = BSON("$top" << BSON("output" + << "$b" + << "sortBy" << BSON("sales" << 1))); + actual = parseAndSerializeAccum( + top.firstElement(), &AccumulatorTopBottomN<TopBottomSense::kTop, true>::parseTopBottomN); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$top": { + "output": "$HASH<b>", + "sortBy": { + "HASH<sales>": 1 + } + } + })", + actual); + + auto max = BSON("$max" << BSON_ARRAY( + "$a" << 2 << 3 << BSON("$max" << BSON_ARRAY(BSON_ARRAY("$b" << 4 << 5))))); + actual = parseAndSerializeAccum(max.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorMax>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$max": [ + "$HASH<a>", + "?number", + "?number", + { + "$max": [ + [ + "$HASH<b>", + "?number", + "?number" + ] + ] + } + ] + })", + actual); + + auto internalJsReduce = BSON( + "$_internalJsReduce" << BSON("data" + << "$emits" + << "eval" + << "function(key, values) {\n return Array.sum(values);\n")); + actual = parseAndSerializeAccum(internalJsReduce.firstElement(), + &AccumulatorInternalJsReduce::parseInternalJsReduce); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$_internalJsReduce":{"data":"$HASH<emits>","eval":"?string"}})", + actual); +} + +TEST(AccumulatorsToExpression, SerializeWithRedaction) { + auto maxN = BSON("$maxN" << BSON("n" << 3 << "input" << BSON_ARRAY(19 << 7 << 28 << 3 << 5))); + using Sense = AccumulatorMinMax::Sense; + auto actual = + parseAndSerializeAccumExpr(maxN, &AccumulatorMinMaxN::parseExpression<Sense::kMax>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$maxN":{"n":"?number","input":"?array<?number>"}})", + actual.getDocument()); + + auto firstN = BSON("$firstN" << BSON("input" + << "$sales" + << "n" + << "\'string\'")); + using FirstLastSense = AccumulatorFirstLastN::Sense; + actual = parseAndSerializeAccumExpr( + firstN, &AccumulatorFirstLastN::parseExpression<FirstLastSense::kFirst>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$firstN":{"n":"?string","input":"$HASH<sales>"}})", + actual.getDocument()); +} + /* ------------------------- AccumulatorMergeObjects -------------------------- */ TEST(AccumulatorMergeObjects, MergingZeroObjectsShouldReturnEmptyDocument) { @@ -1920,4 +2115,21 @@ TEST(AccumulatorMergeObjects, MergingWithEmptyDocumentShouldIgnore) { assertExpectedResults<AccumulatorMergeObjects>(&expCtx, {{{first, second}, expected}}); } +TEST(AccumulatorMergeObjects, RoundTripSerializationLiteral) { + auto mergeObjs = BSON("$mergeObjects" << BSON("$literal" << BSON_ARRAY(5 << true))); + auto actual = parseAndSerializeAccumRepresentative( + mergeObjs.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorMergeObjects>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$mergeObjects":{"$const":[2,"or more types"]}})", + actual); + + auto roundTrip = parseAndSerializeAccumRepresentative( + actual.toBson().firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorMergeObjects>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$mergeObjects":{"$const":[2,"or more types"]}})", + roundTrip); +} + } // namespace AccumulatorTests diff --git a/src/mongo/db/pipeline/aggregate_command.idl b/src/mongo/db/pipeline/aggregate_command.idl index f577931ff01..67c997dcd5d 100644 --- a/src/mongo/db/pipeline/aggregate_command.idl +++ b/src/mongo/db/pipeline/aggregate_command.idl @@ -66,11 +66,13 @@ types: structs: PassthroughToShardOptions: description: "options for commands requesting a per shard cursor" + query_shape_component: true fields: shard: description: "id of the shard to passthrough to" type: string unstable: true + query_shape: anonymize commands: aggregate: @@ -94,6 +96,14 @@ commands: - privilege: # $planCacheStats resource_pattern: exact_namespace action_type: planCacheRead + - privilege: # $queryStats + agg_stage: queryStats + resource_pattern: cluster + action_type: queryStatsRead + - privilege: # $queryStats + agg_stage: queryStats + resource_pattern: cluster + action_type: queryStatsReadTransformed - privilege: # $changeStream resource_pattern: exact_namespace action_type: changeStream diff --git a/src/mongo/db/pipeline/aggregation_context_fixture.h b/src/mongo/db/pipeline/aggregation_context_fixture.h index a4dfcfeb769..cce0283314c 100644 --- a/src/mongo/db/pipeline/aggregation_context_fixture.h +++ b/src/mongo/db/pipeline/aggregation_context_fixture.h @@ -31,8 +31,10 @@ #include <boost/intrusive_ptr.hpp> #include <memory> +#include <vector> #include "mongo/db/concurrency/locker_noop_client_observer.h" +#include "mongo/db/pipeline/document_source.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/service_context_test_fixture.h" #include "mongo/unittest/temp_dir.h" @@ -71,6 +73,42 @@ public: return _opCtx.get(); } + /* + * Serialize and redact a document source. + */ + BSONObj redact(const DocumentSource& docSource, + bool performRedaction = true, + boost::optional<ExplainOptions::Verbosity> verbosity = boost::none) { + SerializationOptions options; + options.verbosity = verbosity; + if (performRedaction) { + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + options.transformIdentifiersCallback = [](StringData s) -> std::string { + return str::stream() << "HASH<" << s << ">"; + }; + options.transformIdentifiers = true; + } + std::vector<Value> serialized; + docSource.serializeToArray(serialized, options); + ASSERT_EQ(1, serialized.size()); + return serialized[0].getDocument().toBson().getOwned(); + } + + std::vector<Value> redactToArray(const DocumentSource& docSource, + bool performRedaction = true) { + SerializationOptions options; + if (performRedaction) { + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + options.transformIdentifiersCallback = [](StringData s) -> std::string { + return str::stream() << "HASH<" << s << ">"; + }; + options.transformIdentifiers = true; + } + std::vector<Value> serialized; + docSource.serializeToArray(serialized, options); + return serialized; + } + private: ServiceContext::UniqueOperationContext _opCtx; boost::intrusive_ptr<ExpressionContextForTest> _expCtx; diff --git a/src/mongo/db/pipeline/change_stream_constants.h b/src/mongo/db/pipeline/change_stream_constants.h index 94fe6487b52..7b5104e93cd 100644 --- a/src/mongo/db/pipeline/change_stream_constants.h +++ b/src/mongo/db/pipeline/change_stream_constants.h @@ -34,8 +34,17 @@ namespace mongo { namespace change_stream_constants { +namespace stage_names { +static constexpr StringData kEnsureResumeTokenPresent = + "$_internalChangeStreamEnsureResumeTokenPresent"_sd; +static constexpr StringData kHandleTopologyChange = "$_internalChangeStreamHandleTopologyChange"_sd; +} // namespace stage_names -const BSONObj kSortSpec = BSON("_id._data" << 1); +static const BSONObj kSortSpec = BSON("_id._data" << 1); + +// Internal change stream stages that can appear in a router (mongoS) pipeline. +static const StringDataSet kChangeStreamRouterPipelineStages = { + stage_names::kEnsureResumeTokenPresent, stage_names::kHandleTopologyChange}; } // namespace change_stream_constants } // namespace mongo diff --git a/src/mongo/db/pipeline/change_stream_rewrite_helpers.cpp b/src/mongo/db/pipeline/change_stream_rewrite_helpers.cpp index f7703354560..ce4a9234c6a 100644 --- a/src/mongo/db/pipeline/change_stream_rewrite_helpers.cpp +++ b/src/mongo/db/pipeline/change_stream_rewrite_helpers.cpp @@ -227,7 +227,7 @@ boost::intrusive_ptr<Expression> exprRewriteOperationType( opCases.push_back(fromjson("{case: {$ne: ['$o.collMod', '$$REMOVE']}, then: 'modify'}")); // The default case, if nothing matches. - auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(false); + auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(); // Build the final expression object... BSONObjBuilder exprBuilder; @@ -417,7 +417,7 @@ boost::intrusive_ptr<Expression> exprRewriteDocumentKey( fromjson("{case: {$eq: ['$op', 'u']}, then: '" + updateAndReplacePath + "'}")); // The default case, if nothing matches. - auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(false); + auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(); // Build the expression BSON object. BSONObjBuilder exprBuilder; @@ -1159,7 +1159,7 @@ boost::intrusive_ptr<Expression> exprRewriteNs( collCases.push_back(fromjson("{case: {$ne: ['$o.collMod', '$$REMOVE']}, then: '$o.collMod'}")); // The default case, if nothing matches. - auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(false); + auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(); // Build the collection expression object... BSONObjBuilder collExprBuilder; @@ -1500,7 +1500,8 @@ std::unique_ptr<MatchExpression> rewriteMatchExpressionTree( case MatchExpression::EXPRESSION: { // Agg expressions are rewritten in-place, so we must clone the expression tree. auto origExprVal = - static_cast<const ExprMatchExpression*>(root)->getExpression()->serialize(false); + static_cast<const ExprMatchExpression*>(root)->getExpression()->serialize( + SerializationOptions{}); auto clonedExpr = Expression::parseOperand( expCtx.get(), BSON("" << origExprVal).firstElement(), expCtx->variablesParseState); @@ -1513,7 +1514,11 @@ std::unique_ptr<MatchExpression> rewriteMatchExpressionTree( } default: { if (auto pathME = dynamic_cast<const PathMatchExpression*>(root)) { - tassert(5687201, "Unexpected empty path", !pathME->path().empty()); + // Only attempt to rewrite non-empty paths. + if (pathME->path().empty()) { + return nullptr; + } + auto firstPath = pathME->fieldRef()->getPart(0).toString(); // Only attempt to rewrite paths that begin with one of the caller-requested fields. diff --git a/src/mongo/db/pipeline/document_path_support_test.cpp b/src/mongo/db/pipeline/document_path_support_test.cpp index 3df55d54e90..489575ba6aa 100644 --- a/src/mongo/db/pipeline/document_path_support_test.cpp +++ b/src/mongo/db/pipeline/document_path_support_test.cpp @@ -42,15 +42,6 @@ #include "mongo/db/pipeline/field_path.h" #include "mongo/unittest/unittest.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString(); \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace mongo { namespace document_path_support { diff --git a/src/mongo/db/pipeline/document_source.cpp b/src/mongo/db/pipeline/document_source.cpp index 55d400af60b..ea1cf479480 100644 --- a/src/mongo/db/pipeline/document_source.cpp +++ b/src/mongo/db/pipeline/document_source.cpp @@ -36,6 +36,7 @@ #include "mongo/db/commands/feature_compatibility_version_documentation.h" #include "mongo/db/exec/document_value/value.h" #include "mongo/db/matcher/expression_algo.h" +#include "mongo/db/pipeline/change_stream_constants.h" #include "mongo/db/pipeline/document_source_add_fields.h" #include "mongo/db/pipeline/document_source_group.h" #include "mongo/db/pipeline/document_source_internal_shard_filter.h" @@ -171,45 +172,77 @@ bool groupMatchSwapVerified(const DocumentSourceMatch& nextMatch, return !expression::hasExistencePredicateOnPath(*(nextMatch.getMatchExpression()), "_id"_sd); } +/** + * Returns 'true' if the given stage is an internal change stream stage that can appear in a router + * (mongoS) pipeline, or 'false' otherwise. + */ +bool isChangeStreamRouterPipelineStage(StringData stageName) { + return change_stream_constants::kChangeStreamRouterPipelineStages.contains(stageName); +} } // namespace bool DocumentSource::pushMatchBefore(Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { - auto nextMatch = dynamic_cast<DocumentSourceMatch*>((*std::next(itr)).get()); - auto thisGroup = dynamic_cast<DocumentSourceGroup*>(this); - if (constraints().canSwapWithMatch && nextMatch && !nextMatch->isTextQuery() && - (!thisGroup || groupMatchSwapVerified(*nextMatch, *thisGroup))) { - // We're allowed to swap with a $match and the stage after us is a $match. Furthermore, the - // $match does not contain a text search predicate, which we do not attempt to optimize - // because such a $match must already be the first stage in the pipeline. We can attempt to - // swap the $match or part of the $match before ourselves. - auto splitMatch = - DocumentSourceMatch::splitMatchByModifiedFields(nextMatch, getModifiedPaths()); - invariant(splitMatch.first || splitMatch.second); - - if (splitMatch.first) { - // At least part of the $match can be moved before this stage. Erase the original $match - // and put the independent part before this stage. If splitMatch.second is not null, - // then there is a new $match stage to insert after ourselves which is dependent on the - // modified fields. - LOGV2_DEBUG( - 5943503, + if (!constraints().canSwapWithMatch) { + return false; + } + + auto nextStageAsMatch = dynamic_cast<DocumentSourceMatch*>((*std::next(itr)).get()); + if (!nextStageAsMatch || nextStageAsMatch->isTextQuery()) { + // We do not need to attempt this optimization if the $match contains a text search + // predicate because, in that scenario, $match is already required to be the first stage in + // the pipeline. + return false; + } + + // At this point: + // 1) The next stage after 'this' is $match. + // 2) The $match stage does not contain a text search predicate. + + // TODO SERVER-55492: Remove the following workaround when there are rename checks for 'other' + // match expressions. + if (isChangeStreamRouterPipelineStage(this->getSourceName())) { + // Always move the $match stage ahead of internal change stream stages appearing in the + // router (mongoS) pipeline, because they do not access or modify any paths in the input + // document. + container->splice(itr, *container, std::next(itr)); + return true; + } + + auto thisStageAsGroup = dynamic_cast<DocumentSourceGroup*>(this); + if (thisStageAsGroup && !groupMatchSwapVerified(*nextStageAsMatch, *thisStageAsGroup)) { + return false; + } + + auto [renameableMatchPart, nonRenameableMatchPart] = + DocumentSourceMatch::splitMatchByModifiedFields(nextStageAsMatch, getModifiedPaths()); + invariant(renameableMatchPart || nonRenameableMatchPart); + if (!renameableMatchPart) { + return false; + } + + LOGV2_DEBUG(5943503, 5, "Swapping all or part of a $match stage in front of another stage: ", - "matchMovingBefore"_attr = redact(splitMatch.first->serializeToBSONForDebug()), + "matchMovingBefore"_attr = redact(renameableMatchPart->serializeToBSONForDebug()), "thisStage"_attr = redact(serializeToBSONForDebug()), "matchLeftAfter"_attr = redact( - splitMatch.second ? splitMatch.second->serializeToBSONForDebug() : BSONObj())); - container->erase(std::next(itr)); - container->insert(itr, std::move(splitMatch.first)); - if (splitMatch.second) { - container->insert(std::next(itr), std::move(splitMatch.second)); - } - - return true; - } + nonRenameableMatchPart ? nonRenameableMatchPart->serializeToBSONForDebug() + : BSONObj())); + + // At this point we know that at least part of the $match expression can be moved ahead of + // 'this'. So, we erase the original $match and move that renameable part ahead of 'this' stage. + container->erase(std::next(itr)); + container->insert(itr, std::move(renameableMatchPart)); + + // If 'nonRenameableMatchPart' is not null, the 'renameableMatchPart' of the $match expression + // was only one component of the original $match. So, we need to create a new $match stage for + // the remaining 'nonRenameableMatchPart' and insert it after 'this' - effectively keeping it in + // its original position in the pipeline. + if (nonRenameableMatchPart) { + container->insert(std::next(itr), std::move(nonRenameableMatchPart)); } - return false; + return true; } bool DocumentSource::pushSampleBefore(Pipeline::SourceContainer::iterator itr, @@ -227,7 +260,9 @@ bool DocumentSource::pushSampleBefore(Pipeline::SourceContainer::iterator itr, BSONObj DocumentSource::serializeToBSONForDebug() const { std::vector<Value> serialized; - serializeToArray(serialized, ExplainOptions::Verbosity::kQueryPlanner); + auto opts = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; + serializeToArray(serialized, opts); if (serialized.empty()) { LOGV2_DEBUG(5943501, 5, @@ -278,8 +313,8 @@ Pipeline::SourceContainer::iterator DocumentSource::optimizeAt( } void DocumentSource::serializeToArray(vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain) const { - Value entry = serialize(explain); + const SerializationOptions& opts) const { + Value entry = serialize(opts); if (!entry.missing()) { array.push_back(entry); } diff --git a/src/mongo/db/pipeline/document_source.h b/src/mongo/db/pipeline/document_source.h index faec2961599..4e4a5bfdadd 100644 --- a/src/mongo/db/pipeline/document_source.h +++ b/src/mongo/db/pipeline/document_source.h @@ -437,13 +437,9 @@ public: * * A subclass may choose to overwrite this, rather than serialize, if it should output multiple * stages (eg, $sort sometimes also outputs a $limit). - * - * The 'explain' parameter indicates the explain verbosity mode, or is equal boost::none if no - * explain is requested. */ - virtual void serializeToArray( - std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const; + virtual void serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts = SerializationOptions{}) const; /** * Shortcut method to get a BSONObj for debugging. Often useful in log messages, but is not @@ -794,12 +790,8 @@ private: * This is used by the default implementation of serializeToArray() to add this object * to a pipeline being serialized. Returning a missing() Value results in no entry * being added to the array for this stage (DocumentSource). - * - * The 'explain' parameter indicates the explain verbosity mode, or is equal boost::none if no - * explain is requested. */ - virtual Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const = 0; + virtual Value serialize(const SerializationOptions& opts = SerializationOptions{}) const = 0; }; /** diff --git a/src/mongo/db/pipeline/document_source_bucket.cpp b/src/mongo/db/pipeline/document_source_bucket.cpp index 4de8daee491..acf56f771b9 100644 --- a/src/mongo/db/pipeline/document_source_bucket.cpp +++ b/src/mongo/db/pipeline/document_source_bucket.cpp @@ -71,6 +71,12 @@ list<intrusive_ptr<DocumentSource>> DocumentSourceBucket::createFromBson( BSONElement groupByField; Value defaultValue; + // Validating the input boundaries and default value requires a correct collation. We may not + // have this at parse time if we are not parsing for the purpose of executing the queries (see + // SERVER-87166), so we should skip the validation for that case. + const bool shouldValidateInputRanges = + pExpCtx->mongoProcessInterface->isExpectedToExecuteQueries(); + bool outputFieldSpecified = false; for (auto&& argument : bucketObj) { const auto argName = argument.fieldNameStringData(); @@ -78,10 +84,10 @@ list<intrusive_ptr<DocumentSource>> DocumentSourceBucket::createFromBson( groupByField = argument; const bool groupByIsExpressionInObject = groupByField.type() == BSONType::Object && - groupByField.embeddedObject().firstElementFieldName()[0] == '$'; + groupByField.embeddedObject().firstElementFieldNameStringData().startsWith("$"); - const bool groupByIsPrefixedPath = - groupByField.type() == BSONType::String && groupByField.valueStringData()[0] == '$'; + const bool groupByIsPrefixedPath = groupByField.type() == BSONType::String && + groupByField.valueStringData().startsWith("$"); uassert(40202, str::stream() << "The $bucket 'groupBy' field must be defined as a $-prefixed " "path or an expression, but found: " @@ -129,7 +135,8 @@ list<intrusive_ptr<DocumentSource>> DocumentSourceBucket::createFromBson( << "The 'boundaries' option to $bucket must be sorted, but elements " << i - 1 << " and " << i << " are not in ascending order (" << lower.toString() << " is not less than " << upper.toString() << ").", - pExpCtx->getValueComparator().evaluate(lower < upper)); + !shouldValidateInputRanges || + pExpCtx->getValueComparator().evaluate(lower < upper)); } } else if ("default" == argName) { // If there is a default, make sure that it parses to a constant expression then add @@ -166,8 +173,9 @@ list<intrusive_ptr<DocumentSource>> DocumentSourceBucket::createFromBson( Value lowerValue = boundaryValues.front(); Value upperValue = boundaryValues.back(); - if (canonicalizeBSONType(defaultValue.getType()) == - canonicalizeBSONType(lowerValue.getType())) { + if (shouldValidateInputRanges && + canonicalizeBSONType(defaultValue.getType()) == + canonicalizeBSONType(lowerValue.getType())) { // If the default has the same canonical type as the bucket's boundaries, then make sure the // default is less than the lowest boundary or greater than or equal to the highest // boundary. diff --git a/src/mongo/db/pipeline/document_source_bucket_auto.cpp b/src/mongo/db/pipeline/document_source_bucket_auto.cpp index e3a00af3bd9..cec0be7b847 100644 --- a/src/mongo/db/pipeline/document_source_bucket_auto.cpp +++ b/src/mongo/db/pipeline/document_source_bucket_auto.cpp @@ -372,24 +372,25 @@ void DocumentSourceBucketAuto::doDispose() { _sortedInput.reset(); } -Value DocumentSourceBucketAuto::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceBucketAuto::serialize(const SerializationOptions& opts) const { MutableDocument insides; - insides["groupBy"] = _groupByExpression->serialize(static_cast<bool>(explain)); - insides["buckets"] = Value(_nBuckets); + insides["groupBy"] = _groupByExpression->serialize(opts); + insides["buckets"] = opts.serializeLiteral(_nBuckets); if (_granularityRounder) { - insides["granularity"] = Value(_granularityRounder->getName()); + //"granularity" only supports some strings, so a specific representative value is used if + // necessary. + insides["granularity"] = + opts.serializeLiteral(_granularityRounder->getName(), Value("R5"_sd)); } MutableDocument outputSpec(_accumulatedFields.size()); for (auto&& accumulatedField : _accumulatedFields) { intrusive_ptr<AccumulatorState> accum = accumulatedField.makeAccumulator(); - outputSpec[accumulatedField.fieldName] = - Value(accum->serialize(accumulatedField.expr.initializer, - accumulatedField.expr.argument, - static_cast<bool>(explain))); + outputSpec[opts.serializeFieldPathFromString(accumulatedField.fieldName)] = + Value(accum->serialize( + accumulatedField.expr.initializer, accumulatedField.expr.argument, opts)); } insides["output"] = outputSpec.freezeToValue(); diff --git a/src/mongo/db/pipeline/document_source_bucket_auto.h b/src/mongo/db/pipeline/document_source_bucket_auto.h index 6a85e21c642..6d8cd09089e 100644 --- a/src/mongo/db/pipeline/document_source_bucket_auto.h +++ b/src/mongo/db/pipeline/document_source_bucket_auto.h @@ -45,7 +45,8 @@ namespace mongo { class DocumentSourceBucketAuto final : public DocumentSource { public: static constexpr StringData kStageName = "$bucketAuto"_sd; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + DepsTracker::State getDependencies(DepsTracker* deps) const final; const char* getSourceName() const final; diff --git a/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp b/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp index 8e0be33e786..4bc3cb0e11a 100644 --- a/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp +++ b/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp @@ -89,8 +89,9 @@ public: assertBucketAutoType(bucketAutoStage); vector<Value> explainedStages; - bucketAutoStage->serializeToArray(explainedStages, - ExplainOptions::Verbosity::kQueryPlanner); + bucketAutoStage->serializeToArray( + explainedStages, + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); ASSERT_EQUALS(explainedStages.size(), 1UL); Value expectedExplain = Value(expectedObj); @@ -868,5 +869,109 @@ TEST_F(BucketAutoTests, ShouldFailOnNegativeNumbersWhenGranularitySpecified) { AssertionException, 40260); } + +TEST_F(BucketAutoTests, RedactionWithoutOutputField) { + auto spec = fromjson(R"({ + $bucketAuto: { + groupBy: '$_id', + buckets: 5, + granularity: "R5" + } + })"); + auto docSource = DocumentSourceBucketAuto::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$bucketAuto": { + "groupBy": "$HASH<_id>", + "buckets": "?number", + "granularity": "?string", + "output": { + "HASH<count>": { + "$sum": "?number" + } + } + } + })", + redact(*docSource)); +} + +TEST_F(BucketAutoTests, RedactionWithOutputField) { + auto spec = fromjson(R"({ + $bucketAuto: { + groupBy: '$year', + buckets: 3, + output: { + count: { $sum: 1 }, + years: { $push: '$year' } + } + }})"); + auto docSource = DocumentSourceBucketAuto::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$bucketAuto": { + "groupBy": "$HASH<year>", + "buckets": "?number", + "output": { + "HASH<count>": { + "$sum": "?number" + }, + "HASH<years>": { + "$push": "$HASH<year>" + } + } + } + })", + redact(*docSource)); +} + +TEST_F(BucketAutoTests, QueryShapeReParseSerializedStage) { + auto expCtx = getExpCtx(); + auto spec = fromjson(R"({ + $bucketAuto: { + groupBy: '$year', + buckets: 3, + granularity: "E192", + output: { + count: { $sum: 1 }, + years: { $push: '$year' } + } + }})"); + + auto docSource = DocumentSourceBucketAuto::createFromBson(spec.firstElement(), expCtx); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + std::vector<Value> serialized; + docSource->serializeToArray(serialized, opts); + auto serializedDocSource = serialized[0].getDocument().toBson(); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$bucketAuto": { + "groupBy": "$year", + "buckets": 1, + "granularity": "R5", + "output": { + "count": { + "$sum": { + "$const":1 + } + }, + "years": { + "$push": "$year" + } + } + } + })", + serializedDocSource); + auto docSourceFromQueryShape = + DocumentSourceBucketAuto::createFromBson(serializedDocSource.firstElement(), expCtx); + + vector<Value> newSerialization; + docSourceFromQueryShape->serializeToArray(newSerialization, opts); + auto newSerializedDocSource = newSerialization[0].getDocument().toBson(); + ASSERT_BSONOBJ_EQ(serializedDocSource, newSerializedDocSource); +} + + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_bucket_test.cpp b/src/mongo/db/pipeline/document_source_bucket_test.cpp index 1036c456ef3..e7702d58d28 100644 --- a/src/mongo/db/pipeline/document_source_bucket_test.cpp +++ b/src/mongo/db/pipeline/document_source_bucket_test.cpp @@ -79,7 +79,8 @@ public: // Serialize the DocumentSourceGroup and DocumentSourceSort from $bucket so that we can // check the explain output to make sure $group and $sort have the correct fields. - auto explain = ExplainOptions::Verbosity::kQueryPlanner; + auto explain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; vector<Value> explainedStages; groupStage->serializeToArray(explainedStages, explain); sortStage->serializeToArray(explainedStages, explain); @@ -227,9 +228,35 @@ TEST_F(BucketReturnsGroupAndSort, BucketSucceedsWithMultipleBoundaryValues) { testCreateFromBsonResult(spec, expectedGroupExplain); } +TEST_F(BucketReturnsGroupAndSort, BucketWithEmptyGroupByStrDoesNotAccessPastEndOfString) { + // Verify that {groupBy: ''} is rejected _without_ attempting to read past the end of the empty + // string. + const auto spec = + fromjson("{$bucket : {groupBy : '', boundaries : [ 1, 5, 8 ], default : 'other'}}"); + + // Under a debug build, this would previously fail if an empty str for groupBy led to access + // past the end of the string, with pos() > size() in StringData::operator[]. + // Verify that this reaches the intended uassert, rejecting the empty string, _without_ first + // trying to read past the end of the string. + ASSERT_THROWS_CODE(DocumentSourceBucket::createFromBson(spec.firstElement(), getExpCtx()), + AssertionException, + 40202); +} + +/* + * Override the stub interface to allow full execution in these tests. + */ +class ExecutableStubMongoProcessInterface : public StubMongoProcessInterface { + bool isExpectedToExecuteQueries() override { + return true; + } +}; + class InvalidBucketSpec : public AggregationContextFixture { public: list<intrusive_ptr<DocumentSource>> createBucket(BSONObj bucketSpec) { + getExpCtx()->mongoProcessInterface = + std::make_unique<ExecutableStubMongoProcessInterface>(); auto sources = DocumentSourceBucket::createFromBson(bucketSpec.firstElement(), getExpCtx()); return sources; } diff --git a/src/mongo/db/pipeline/document_source_change_stream.cpp b/src/mongo/db/pipeline/document_source_change_stream.cpp index 3f19d4ac4ee..7602e29417e 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream.cpp @@ -352,7 +352,7 @@ std::list<boost::intrusive_ptr<DocumentSource>> DocumentSourceChangeStream::_bui // If 'showExpandedEvents' is NOT set, add a filter that returns only classic change events. if (!spec.getShowExpandedEvents()) { - stages.push_back(DocumentSourceMatch::create( + stages.push_back(DocumentSourceInternalChangeStreamMatch::create( change_stream_filter::getMatchFilterForClassicOperationTypes(), expCtx)); } return stages; @@ -369,6 +369,12 @@ void DocumentSourceChangeStream::assertIsLegalSpecification( (replCoord && replCoord->getReplicationMode() == repl::ReplicationCoordinator::Mode::modeReplSet)); + // We will not validate user specified options when we are not expecting to execute queries, + // such as during $queryStats. + if (!expCtx->mongoProcessInterface->isExpectedToExecuteQueries()) { + return; + } + // If 'allChangesForCluster' is true, the stream must be opened on the 'admin' database with // {aggregate: 1}. uassert(ErrorCodes::InvalidOptions, diff --git a/src/mongo/db/pipeline/document_source_change_stream.h b/src/mongo/db/pipeline/document_source_change_stream.h index 5a76a5205e2..1a75d5b4300 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.h +++ b/src/mongo/db/pipeline/document_source_change_stream.h @@ -336,4 +336,29 @@ public: } }; +/** + * A DocumentSource class for all internal change stream stages. This class is useful for + * shared logic between all of the internal change stream stages. For internally created match + * stages see 'DocumentSourceInternalChangeStreamMatch'. + */ +class DocumentSourceInternalChangeStreamStage : public DocumentSource { +public: + DocumentSourceInternalChangeStreamStage(StringData stageName, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSource(stageName, expCtx) {} + + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const override { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || + opts.transformIdentifiers) { + // Stages made internally by 'DocumentSourceChangeStream' should not be serialized for + // query stats. For query stats we will serialize only the user specified $changeStream + // stage. + return Value(); + } + return doSerialize(opts); + } + + virtual Value doSerialize(const SerializationOptions& opts) const = 0; +}; + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream.idl b/src/mongo/db/pipeline/document_source_change_stream.idl index e1b8227a0fc..de5473afb58 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.idl +++ b/src/mongo/db/pipeline/document_source_change_stream.idl @@ -30,22 +30,10 @@ global: cpp_namespace: "mongo" - cpp_includes: - - "mongo/db/pipeline/resume_token.h" imports: - "mongo/idl/basic_types.idl" - -types: - # A resume token is an opaque document we return to the user that contains all the information - # needed to resume a stream where they left off. It also provides the ordering of streams - # from multiple shards. - resumeToken: - bson_serialization_type: object - description: An object representing a resume token for a change stream - cpp_type: ResumeToken - serializer: ResumeToken::toBSON - deserializer: ResumeToken::parse + - "mongo/db/pipeline/resume_token.idl" enums: FullDocumentBeforeChangeMode: @@ -86,6 +74,7 @@ structs: description: A document used to specify the $changeStream stage of an aggregation pipeline. The same spec is also used by the $_internalChangeStreamTransform stage. + query_shape_component: true fields: resumeAfter: cpp_name: resumeAfter @@ -94,6 +83,7 @@ structs: description: An object representing the point at which we should resume reporting changes from. Only one of resumeAfter, startAfter, and startAtOperationTime should be specified. + query_shape: custom startAfter: cpp_name: startAfter @@ -103,6 +93,7 @@ structs: changes from. This is allowed to be a token from an invalidating command. Only one of resumeAfter, startAfter, and startAtOperationTime should be specified. + query_shape: custom startAtOperationTime: cpp_name: startAtOperationTime @@ -111,6 +102,7 @@ structs: description: The operation time after which we should start reporting changes. Only one of resumeAfter, startAfter, and startAtOperationTime should be specified. + query_shape: literal fullDocument: cpp_name: fullDocument @@ -118,6 +110,7 @@ structs: default: kDefault description: A string '"updateLookup"' or '"default"', indicating whether or not we should return a full document or just changes for an update. + query_shape: parameter fullDocumentBeforeChange: cpp_name: fullDocumentBeforeChange @@ -132,6 +125,7 @@ structs: set to "required", then the "fullDocumentBeforeChange" field is always populated and an exception is thrown if the pre-image is not available. + query_shape: parameter allChangesForCluster: cpp_name: allChangesForCluster @@ -139,6 +133,7 @@ structs: description: A flag indicating whether the stream should report all changes that occur on the deployment, aside from those on internal databases or collections. + query_shape: literal showMigrationEvents: cpp_name: showMigrationEvents @@ -149,12 +144,14 @@ structs: deletes may appear that do not reflect actual deletions or insertions of data. Instead they reflect this data moving from one shard to another. + query_shape: literal showSystemEvents: cpp_name: showSystemEvents type: optionalBool description: A flag indicating whether the stream should report events on system collections. + query_shape: literal allowToRunOnConfigDB: cpp_name: allowToRunOnConfigDB @@ -163,17 +160,20 @@ structs: 'config' database, which is usually banned. This flag is used internally to allow mongoS to open a stream on 'config.shards', in order to monitor for the addition of new shards to the cluster. + query_shape: literal allowToRunOnSystemNS: cpp_name: allowToRunOnSystemNS type: optionalBool description: An internal flag indicating whether the change stream may be opened on a system collection. + query_shape: literal showExpandedEvents: type: optionalBool description: A flag indicating whether the change stream output should include the expanded events and parameters that may not be part of the stable API. + query_shape: literal showRawUpdateDescription: cpp_name: showRawUpdateDescription @@ -183,6 +183,7 @@ structs: raw update description from the oplog entry for the corresponding update command. When this flag is set to true, the standard "updateDescription" field in update events will be omitted. + query_shape: literal DocumentSourceChangeStreamOplogMatchSpec: strict: true @@ -206,7 +207,7 @@ structs: startAfterInvalidate: type: resumeToken optional: true - + DocumentSourceChangeStreamCheckResumabilitySpec: strict: true description: A document used to specify the $_internalChangeStreamCheckResumability stage of @@ -231,3 +232,12 @@ structs: fields: fullDocument: type: FullDocumentMode + + DocumentSourceChangeStreamHandleTopologyChangeSpec: + strict: true + description: A document used to specify the $_internalChangeStreamHandleTopologyChange stage of + an aggregation pipeline. + fields: + originalAggregateCommand: + type: object + optional: true diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp index aa663dd131f..437ec96b386 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp @@ -217,9 +217,8 @@ boost::optional<Document> DocumentSourceChangeStreamAddPostImage::lookupLatestPo pExpCtx, nss, *resumeTokenData.uuid, documentKey, std::move(readConcern)); } -Value DocumentSourceChangeStreamAddPostImage::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return explain +Value DocumentSourceChangeStreamAddPostImage::doSerialize(const SerializationOptions& opts) const { + return opts.verbosity ? Value(Document{ {DocumentSourceChangeStream::kStageName, Document{{"stage"_sd, kStageName}, @@ -227,5 +226,4 @@ Value DocumentSourceChangeStreamAddPostImage::serialize( : Value(Document{{kStageName, DocumentSourceChangeStreamAddPostImageSpec(_fullDocumentMode).toBSON()}}); } - } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h index d92994064e2..03c5459d15d 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h +++ b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h @@ -38,7 +38,8 @@ namespace mongo { * Part of the change stream API machinery used to look up the post-image of a document. Uses the * "documentKey" field of the input to look up the new version of the document. */ -class DocumentSourceChangeStreamAddPostImage final : public DocumentSource { +class DocumentSourceChangeStreamAddPostImage final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamAddPostImage"_sd; static constexpr StringData kFullDocumentFieldName = @@ -114,7 +115,7 @@ public: return DepsTracker::State::SEE_NEXT; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; const char* getSourceName() const final { return kStageName.rawData(); @@ -123,7 +124,8 @@ public: private: DocumentSourceChangeStreamAddPostImage(const boost::intrusive_ptr<ExpressionContext>& expCtx, const FullDocumentModeEnum fullDocumentMode) - : DocumentSource(kStageName, expCtx), _fullDocumentMode(fullDocumentMode) { + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx), + _fullDocumentMode(fullDocumentMode) { tassert(5842300, "the 'fullDocument' field cannot be 'default'", _fullDocumentMode != FullDocumentModeEnum::kDefault); diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_post_image_test.cpp b/src/mongo/db/pipeline/document_source_change_stream_add_post_image_test.cpp index 25ca17bbbfa..c97a393d4d7 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_post_image_test.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_add_post_image_test.cpp @@ -106,7 +106,9 @@ TEST_F(DocumentSourceChangeStreamAddPostImageTest, ShouldSerializeAsExpectedForE Document{{"stage"_sd, DocumentSourceChangeStreamAddPostImage::kStageName}, {"fullDocument"_sd, "updateLookup"_sd}}}}); - ASSERT_VALUE_EQ(stage->serialize({ExplainOptions::Verbosity::kQueryPlanner}), expectedOutput); + ASSERT_VALUE_EQ(stage->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedOutput); } TEST_F(DocumentSourceChangeStreamAddPostImageTest, ShouldSerializeAsExpectedForDispatch) { diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp index 42b0cc20bfb..f3c47c58d9a 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp @@ -152,9 +152,8 @@ boost::optional<Document> DocumentSourceChangeStreamAddPreImage::lookupPreImage( return preImageField.getDocument().getOwned(); } -Value DocumentSourceChangeStreamAddPreImage::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return explain +Value DocumentSourceChangeStreamAddPreImage::doSerialize(const SerializationOptions& opts) const { + return opts.verbosity ? Value(Document{ {DocumentSourceChangeStream::kStageName, Document{{"stage"_sd, "internalAddPreImage"_sd}, diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h index e04ac9a30ef..2c4918ac34b 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h +++ b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h @@ -41,7 +41,7 @@ namespace mongo { * its "fullDocumentBeforeChange" field shall be the optime of the noop oplog entry containing the * pre-image. This stage replaces that field with the actual pre-image document. */ -class DocumentSourceChangeStreamAddPreImage final : public DocumentSource { +class DocumentSourceChangeStreamAddPreImage final : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamAddPreImage"_sd; static constexpr StringData kFullDocumentBeforeChangeFieldName = @@ -68,7 +68,8 @@ public: DocumentSourceChangeStreamAddPreImage(const boost::intrusive_ptr<ExpressionContext>& expCtx, FullDocumentBeforeChangeModeEnum mode) - : DocumentSource(kStageName, expCtx), _fullDocumentBeforeChangeMode(mode) { + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx), + _fullDocumentBeforeChangeMode(mode) { // This stage should never be created with FullDocumentBeforeChangeMode::kOff. invariant(_fullDocumentBeforeChangeMode != FullDocumentBeforeChangeModeEnum::kOff); } @@ -108,7 +109,7 @@ public: return DepsTracker::State::SEE_NEXT; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; const char* getSourceName() const final { return kStageName.rawData(); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp index 4e85815b049..935983e8ba5 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp @@ -180,18 +180,20 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamCheckInvalidate::doGetNe return nextInput; } -Value DocumentSourceChangeStreamCheckInvalidate::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { - return Value(Document{{DocumentSourceChangeStream::kStageName, - Document{{"stage"_sd, "internalCheckInvalidate"_sd}}}}); +Value DocumentSourceChangeStreamCheckInvalidate::doSerialize( + const SerializationOptions& opts) const { + BSONObjBuilder builder; + if (opts.verbosity) { + BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); + sub.append("stage"_sd, kStageName); + sub.done(); } - DocumentSourceChangeStreamCheckInvalidateSpec spec; if (_startAfterInvalidate) { spec.setStartAfterInvalidate(ResumeToken(*_startAfterInvalidate)); } - return Value(Document{{DocumentSourceChangeStreamCheckInvalidate::kStageName, spec.toBSON()}}); + builder.append(DocumentSourceChangeStreamCheckInvalidate::kStageName, spec.toBSON()); + return Value(builder.obj()); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h index 166604c1d10..764e8531ccd 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h +++ b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h @@ -39,7 +39,8 @@ namespace mongo { * "invalidate" entry for commands that should invalidate the change stream (e.g. collection drop * for a single-collection change stream). It is not intended to be created by the user. */ -class DocumentSourceChangeStreamCheckInvalidate final : public DocumentSource { +class DocumentSourceChangeStreamCheckInvalidate final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamCheckInvalidate"_sd; @@ -64,7 +65,7 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; static boost::intrusive_ptr<DocumentSourceChangeStreamCheckInvalidate> createFromBson( BSONElement spec, const boost::intrusive_ptr<ExpressionContext>& expCtx); @@ -79,7 +80,7 @@ private: */ DocumentSourceChangeStreamCheckInvalidate(const boost::intrusive_ptr<ExpressionContext>& expCtx, boost::optional<ResumeTokenData> startAfterInvalidate) - : DocumentSource(kStageName, expCtx), + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx), _startAfterInvalidate(std::move(startAfterInvalidate)) { invariant(!_startAfterInvalidate || _startAfterInvalidate->fromInvalidate == ResumeTokenData::kFromInvalidate); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp index e7e05dcd756..5d704b00c84 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp @@ -129,7 +129,8 @@ DocumentSourceChangeStreamCheckResumability::compareAgainstClientResumeToken( DocumentSourceChangeStreamCheckResumability::DocumentSourceChangeStreamCheckResumability( const intrusive_ptr<ExpressionContext>& expCtx, ResumeTokenData token) - : DocumentSource(getSourceName(), expCtx), _tokenFromClient(std::move(token)) {} + : DocumentSourceInternalChangeStreamStage(getSourceName(), expCtx), + _tokenFromClient(std::move(token)) {} intrusive_ptr<DocumentSourceChangeStreamCheckResumability> DocumentSourceChangeStreamCheckResumability::create(const intrusive_ptr<ExpressionContext>& expCtx, @@ -205,17 +206,21 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamCheckResumability::doGet MONGO_UNREACHABLE; } -Value DocumentSourceChangeStreamCheckResumability::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return explain - ? Value(DOC(DocumentSourceChangeStream::kStageName - << DOC("stage" - << "internalCheckResumability"_sd - << "resumeToken" << ResumeToken(_tokenFromClient).toDocument()))) - : Value(Document{ - {DocumentSourceChangeStreamCheckResumability::kStageName, - DocumentSourceChangeStreamCheckResumabilitySpec(ResumeToken(_tokenFromClient)) - .toBSON()}}); +Value DocumentSourceChangeStreamCheckResumability::doSerialize( + const SerializationOptions& opts) const { + BSONObjBuilder builder; + if (opts.verbosity) { + BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); + sub.append("stage"_sd, kStageName); + sub << "resumeToken"_sd << Value(ResumeToken(_tokenFromClient).toDocument(opts)); + sub.done(); + } else { + builder.append( + kStageName, + DocumentSourceChangeStreamCheckResumabilitySpec(ResumeToken(_tokenFromClient)) + .toBSON()); + } + return Value(builder.obj()); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h index 59946310ab1..d0409198d22 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h +++ b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h @@ -59,7 +59,7 @@ namespace mongo { * - Otherwise we cannot resume, as we do not know if there were any events between the resume token * and the first matching document in the oplog. */ -class DocumentSourceChangeStreamCheckResumability : public DocumentSource { +class DocumentSourceChangeStreamCheckResumability : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamCheckResumability"_sd; @@ -90,7 +90,7 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const override; static boost::intrusive_ptr<DocumentSourceChangeStreamCheckResumability> createFromBson( BSONElement spec, const boost::intrusive_ptr<ExpressionContext>& expCtx); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp index 26cdcb77181..912c691d8f2 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp @@ -86,9 +86,9 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamCheckTopologyChange::doG return nextInput; } -Value DocumentSourceChangeStreamCheckTopologyChange::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +Value DocumentSourceChangeStreamCheckTopologyChange::doSerialize( + const SerializationOptions& opts) const { + if (opts.verbosity) { return Value(DOC(DocumentSourceChangeStream::kStageName << DOC("stage" << "internalCheckTopologyChange"_sd))); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h index 7629e5d8063..9bf5b698a7c 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h +++ b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h @@ -45,7 +45,8 @@ namespace mongo { * that previously may not have held any data for the collection being watched, and they contain the * information necessary for the mongoS to include the new shard in the merged change stream. */ -class DocumentSourceChangeStreamCheckTopologyChange final : public DocumentSource { +class DocumentSourceChangeStreamCheckTopologyChange final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamCheckTopologyChange"_sd; @@ -67,12 +68,12 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; private: DocumentSourceChangeStreamCheckTopologyChange( const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSource(kStageName, expCtx) {} + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx) {} GetNextResult doGetNext() final; }; diff --git a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp index a6c9a3fee68..b5c24a36af8 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp @@ -153,16 +153,20 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamEnsureResumeTokenPresent } } -Value DocumentSourceChangeStreamEnsureResumeTokenPresent::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - // We only serialize this stage in the context of explain. - if (explain) { - return Value(DOC(DocumentSourceChangeStream::kStageName - << DOC("stage" - << "internalEnsureResumeTokenPresent"_sd - << "resumeToken" << ResumeToken(_tokenFromClient).toDocument()))); +Value DocumentSourceChangeStreamEnsureResumeTokenPresent::doSerialize( + const SerializationOptions& opts) const { + BSONObjBuilder builder; + if (opts.verbosity) { + BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); + sub.append("stage"_sd, kStageName); + sub << "resumeToken"_sd << Value(ResumeToken(_tokenFromClient).toDocument(opts)); + sub.done(); + } else { + BSONObjBuilder sub(builder.subobjStart(kStageName)); + sub << "resumeToken"_sd << Value(ResumeToken(_tokenFromClient).toDocument(opts)); + sub.done(); } - MONGO_UNREACHABLE_TASSERT(5467611); + return Value(builder.obj()); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h index 6fcabf4c0b6..0482e128379 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h +++ b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h @@ -39,7 +39,8 @@ namespace mongo { class DocumentSourceChangeStreamEnsureResumeTokenPresent final : public DocumentSourceChangeStreamCheckResumability { public: - static constexpr StringData kStageName = "$_internalChangeStreamEnsureResumeTokenPresent"_sd; + static constexpr StringData kStageName = + change_stream_constants::stage_names::kEnsureResumeTokenPresent; const char* getSourceName() const final; @@ -54,7 +55,7 @@ public: const boost::intrusive_ptr<ExpressionContext>& expCtx, const DocumentSourceChangeStreamSpec& spec); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; private: /** diff --git a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp index 9f83f3df203..4edb7f0c26a 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp @@ -46,6 +46,11 @@ namespace mongo { namespace { +REGISTER_INTERNAL_DOCUMENT_SOURCE(_internalChangeStreamHandleTopologyChange, + LiteParsedDocumentSourceChangeStreamInternal::parse, + DocumentSourceChangeStreamHandleTopologyChange::createFromBson, + true); + // Failpoint to throw an exception when the 'kNewShardDetected' event is observed. MONGO_FAIL_POINT_DEFINE(throwChangeStreamTopologyChangeExceptionToClient); @@ -110,6 +115,15 @@ bool isShardConfigEvent(const Document& eventDoc) { } // namespace boost::intrusive_ptr<DocumentSourceChangeStreamHandleTopologyChange> +DocumentSourceChangeStreamHandleTopologyChange::createFromBson( + const BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx) { + uassert(8131300, + str::stream() << "the '" << kStageName << "' spec must be an empty object", + elem.type() == Object && elem.Obj().isEmpty()); + return new DocumentSourceChangeStreamHandleTopologyChange(expCtx); +} + +boost::intrusive_ptr<DocumentSourceChangeStreamHandleTopologyChange> DocumentSourceChangeStreamHandleTopologyChange::create( const boost::intrusive_ptr<ExpressionContext>& expCtx) { return new DocumentSourceChangeStreamHandleTopologyChange(expCtx); @@ -117,7 +131,7 @@ DocumentSourceChangeStreamHandleTopologyChange::create( DocumentSourceChangeStreamHandleTopologyChange::DocumentSourceChangeStreamHandleTopologyChange( const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSource(kStageName, expCtx) {} + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx) {} StageConstraints DocumentSourceChangeStreamHandleTopologyChange::constraints( Pipeline::SplitState) const { @@ -257,9 +271,9 @@ BSONObj DocumentSourceChangeStreamHandleTopologyChange::replaceResumeTokenInComm return newCmd.freeze().toBson(); } -Value DocumentSourceChangeStreamHandleTopologyChange::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +Value DocumentSourceChangeStreamHandleTopologyChange::doSerialize( + const SerializationOptions& opts) const { + if (opts.verbosity) { return Value(DOC(DocumentSourceChangeStream::kStageName << DOC("stage" << "internalHandleTopologyChange"_sd))); diff --git a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h index 4c07368aff9..b88c50d75ce 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h +++ b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h @@ -46,10 +46,14 @@ namespace mongo { * the first time. When this event is detected, this stage will establish a new cursor on that * shard and add it to the cursors being merged. */ -class DocumentSourceChangeStreamHandleTopologyChange final : public DocumentSource { +class DocumentSourceChangeStreamHandleTopologyChange final + : public DocumentSourceInternalChangeStreamStage { public: - static constexpr StringData kStageName = "$_internalChangeStreamHandleTopologyChange"_sd; + static constexpr StringData kStageName = + change_stream_constants::stage_names::kHandleTopologyChange; + static boost::intrusive_ptr<DocumentSourceChangeStreamHandleTopologyChange> createFromBson( + BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx); /** * Creates a new stage which will establish a new cursor and add it to the cursors being merged * by 'mergeCursorsStage' whenever a new shard is detected by a change stream. @@ -61,7 +65,7 @@ public: return kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; StageConstraints constraints(Pipeline::SplitState) const final; diff --git a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp index 658861a9a3f..8656ad68b04 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp @@ -96,8 +96,8 @@ std::unique_ptr<MatchExpression> buildOplogMatchFilter( DocumentSourceChangeStreamOplogMatch::DocumentSourceChangeStreamOplogMatch( Timestamp clusterTime, const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSourceMatch(change_stream_filter::buildOplogMatchFilter(expCtx, clusterTime), - expCtx) { + : DocumentSourceInternalChangeStreamMatch( + change_stream_filter::buildOplogMatchFilter(expCtx, clusterTime), expCtx) { _clusterTime = clusterTime; expCtx->tailableMode = TailableModeEnum::kTailableAndAwaitData; } @@ -204,16 +204,23 @@ Pipeline::SourceContainer::iterator DocumentSourceChangeStreamOplogMatch::doOpti return nextChangeStreamStageItr; } -Value DocumentSourceChangeStreamOplogMatch::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { - return Value( - Document{{DocumentSourceChangeStream::kStageName, - Document{{"stage"_sd, "internalOplogMatch"_sd}, {"filter"_sd, _predicate}}}}); +Value DocumentSourceChangeStreamOplogMatch::doSerialize(const SerializationOptions& opts) const { + BSONObjBuilder builder; + if (opts.verbosity) { + BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); + sub.append("stage"_sd, kStageName); + sub.append(DocumentSourceChangeStreamOplogMatchSpec::kFilterFieldName, + getMatchExpression()->serialize(opts)); + sub.done(); + } else { + BSONObjBuilder sub(builder.subobjStart(kStageName)); + + // 'SerializationOptions' are not required here, since serialization for explain and query + // stats occur before this function call. + DocumentSourceChangeStreamOplogMatchSpec(_predicate).serialize(&sub); + sub.done(); } - - DocumentSourceChangeStreamOplogMatchSpec spec(_predicate); - return Value(Document{{DocumentSourceChangeStreamOplogMatch::kStageName, spec.toBSON()}}); + return Value(builder.obj()); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h index 54d15f363e6..6138fdaae31 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h +++ b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h @@ -36,7 +36,7 @@ namespace mongo { * A custom subclass of DocumentSourceMatch which is used to generate a $match stage to be applied * on the oplog. The stage requires itself to be the first stage in the pipeline. */ -class DocumentSourceChangeStreamOplogMatch final : public DocumentSourceMatch { +class DocumentSourceChangeStreamOplogMatch final : public DocumentSourceInternalChangeStreamMatch { public: static constexpr StringData kStageName = "$_internalChangeStreamOplogMatch"_sd; @@ -45,7 +45,7 @@ public: DocumentSourceChangeStreamOplogMatch(const DocumentSourceChangeStreamOplogMatch& other, const boost::intrusive_ptr<ExpressionContext>& newExpCtx) - : DocumentSourceMatch(other, newExpCtx) { + : DocumentSourceInternalChangeStreamMatch(other, newExpCtx) { _clusterTime = other._clusterTime; _optimizedEndOfPipeline = other._optimizedEndOfPipeline; } @@ -74,7 +74,7 @@ public: StageConstraints constraints(Pipeline::SplitState pipeState) const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts) const final; protected: Pipeline::SourceContainer::iterator doOptimizeAt(Pipeline::SourceContainer::iterator itr, @@ -88,7 +88,8 @@ private: */ DocumentSourceChangeStreamOplogMatch(BSONObj filter, const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSourceMatch(std::move(filter), expCtx), _optimizedEndOfPipeline(true) { + : DocumentSourceInternalChangeStreamMatch(std::move(filter), expCtx), + _optimizedEndOfPipeline(true) { expCtx->tailableMode = TailableModeEnum::kTailableAndAwaitData; } diff --git a/src/mongo/db/pipeline/document_source_change_stream_split_large_event.cpp b/src/mongo/db/pipeline/document_source_change_stream_split_large_event.cpp index b2ac5353325..1cf2ef0ed58 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_split_large_event.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_split_large_event.cpp @@ -31,8 +31,7 @@ #include "mongo/db/pipeline/change_stream_helpers.h" #include "mongo/db/pipeline/change_stream_split_event_helpers.h" -#include "mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h" -#include "mongo/db/pipeline/document_source_change_stream_handle_topology_change.h" +#include "mongo/db/pipeline/document_source_change_stream_check_resumability.h" namespace mongo { namespace { @@ -82,8 +81,7 @@ DocumentSourceChangeStreamSplitLargeEvent::DocumentSourceChangeStreamSplitLargeE !_resumeAfterSplit || _resumeAfterSplit->fragmentNum); } -Value DocumentSourceChangeStreamSplitLargeEvent::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceChangeStreamSplitLargeEvent::serialize(const SerializationOptions& opts) const { return Value(Document{{DocumentSourceChangeStreamSplitLargeEvent::kStageName, Document{}}}); } @@ -190,24 +188,19 @@ size_t DocumentSourceChangeStreamSplitLargeEvent::_handleResumeAfterSplit(const return fragmentNum; } -namespace { -// During pipeline optimization, the split stage must move ahead of these change stream stages. -static const std::set<StringData> kStagesToMoveAheadOf = { - DocumentSourceChangeStreamEnsureResumeTokenPresent::kStageName, - DocumentSourceChangeStreamHandleTopologyChange::kStageName}; -} // namespace - Pipeline::SourceContainer::iterator DocumentSourceChangeStreamSplitLargeEvent::doOptimizeAt( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { // Helper to determine whether the iterator has reached its final position in the pipeline. // Checks whether $changeStreamSplitLargeEvent should move ahead of the given stage. auto shouldMoveAheadOf = [](const auto& stagePtr) { - return kStagesToMoveAheadOf.count(stagePtr->getSourceName()); + return change_stream_constants::kChangeStreamRouterPipelineStages.contains( + stagePtr->getSourceName()); }; // Find the point in the pipeline that the stage should move to. - for (auto it = itr; it != container->begin() && shouldMoveAheadOf(*std::prev(it)); --it) { - std::swap(*it, *std::prev(it)); + for (auto it = itr; it != container->begin() && shouldMoveAheadOf(*std::prev(it));) { + // Swap 'it' with the previous stage. + container->splice(std::prev(it), *container, it); } // Return an iterator pointing to the next stage to be optimized. @@ -218,6 +211,7 @@ void DocumentSourceChangeStreamSplitLargeEvent::validatePipelinePosition( bool alreadyOptimized, Pipeline::SourceContainer::const_iterator pos, const Pipeline::SourceContainer& container) const { + // The $changeStreamSplitLargeEvent stage must be the final stage in the pipeline before // optimization. uassert(7182802, @@ -230,7 +224,8 @@ void DocumentSourceChangeStreamSplitLargeEvent::validatePipelinePosition( str::stream() << getSourceName() << " is at the wrong position in the pipeline after optimization", !alreadyOptimized || std::none_of(container.begin(), pos, [](const auto& stage) { - return kStagesToMoveAheadOf.count(stage->getSourceName()); + return change_stream_constants::kChangeStreamRouterPipelineStages.contains( + stage->getSourceName()); })); }; } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_split_large_event.h b/src/mongo/db/pipeline/document_source_change_stream_split_large_event.h index 1dfab7b85a1..44ed8b41955 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_split_large_event.h +++ b/src/mongo/db/pipeline/document_source_change_stream_split_large_event.h @@ -78,10 +78,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - // This stage does not reference any user or system variables. - void addVariableRefs(std::set<Variables::Id>* refs) const {} - - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/pipeline/document_source_change_stream_test.cpp b/src/mongo/db/pipeline/document_source_change_stream_test.cpp index 50e0f0cdcac..8bcdb7e7f9d 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_test.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_test.cpp @@ -27,6 +27,7 @@ * it in the license file. */ +#include "mongo/bson/bsontypes.h" #include "mongo/platform/basic.h" #include <boost/intrusive_ptr.hpp> @@ -52,8 +53,11 @@ #include "mongo/db/pipeline/document_source_change_stream_add_pre_image.h" #include "mongo/db/pipeline/document_source_change_stream_check_invalidate.h" #include "mongo/db/pipeline/document_source_change_stream_check_resumability.h" +#include "mongo/db/pipeline/document_source_change_stream_check_topology_change.h" #include "mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h" +#include "mongo/db/pipeline/document_source_change_stream_handle_topology_change.h" #include "mongo/db/pipeline/document_source_change_stream_oplog_match.h" +#include "mongo/db/pipeline/document_source_change_stream_split_large_event.h" #include "mongo/db/pipeline/document_source_change_stream_transform.h" #include "mongo/db/pipeline/document_source_change_stream_unwind_transaction.h" #include "mongo/db/pipeline/document_source_limit.h" @@ -88,15 +92,26 @@ using V = Value; using DSChangeStream = DocumentSourceChangeStream; +// Deterministic values used for testing +const UUID testConstUuid = UUID::parse("6948DF80-14BD-4E04-8842-7668D9C001F5").getValue(); + +class ExecutableStubMongoProcessInterface : public StubMongoProcessInterface { + bool isExpectedToExecuteQueries() override { + return true; + } +}; + class ChangeStreamStageTestNoSetup : public AggregationContextFixture { public: ChangeStreamStageTestNoSetup() : ChangeStreamStageTestNoSetup(nss) {} explicit ChangeStreamStageTestNoSetup(NamespaceString nsString) - : AggregationContextFixture(nsString) {} + : AggregationContextFixture(nsString) { + getExpCtx()->mongoProcessInterface = + std::make_unique<ExecutableStubMongoProcessInterface>(); + }; }; -struct MockMongoInterface final : public StubMongoProcessInterface { - +struct MockMongoInterface final : public ExecutableStubMongoProcessInterface { // Used by operations which need to obtain the oplog's UUID. static const UUID& oplogUuid() { static const UUID* oplog_uuid = new UUID(UUID::gen()); @@ -4648,5 +4663,308 @@ TEST_F(MultiTokenFormatVersionTest, CanResumeFromV2HighWaterMark) { next = lastStage->getNext(); ASSERT_FALSE(next.isAdvanced()); } + +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamAddPostImageEmptyForQueryStats) { + auto spec = DocumentSourceChangeStreamSpec(); + spec.setFullDocument(FullDocumentModeEnum::kUpdateLookup); + + auto docSource = DocumentSourceChangeStreamAddPostImage::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalChangeStreamAddPostImage":{"fullDocument":"updateLookup"}})", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamAddPreImageEmptyForQueryStats) { + auto docSource = DocumentSourceChangeStreamAddPreImage{ + getExpCtx(), FullDocumentBeforeChangeModeEnum::kWhenAvailable}; + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamAddPreImage": { + "fullDocumentBeforeChange": "whenAvailable" + } + })", + docSource.serialize().getDocument().toBson()); + + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource.serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamCheckInvalidateEmptyForQueryStats) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse(makeResumeToken(Timestamp(), + testConstUuid, + BSON("_id" << 1 << "x" << 2), + ResumeTokenData::kFromInvalidate))); + + auto docSource = DocumentSourceChangeStreamCheckInvalidate::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamCheckInvalidate": { + "startAfterInvalidate": { + "_data": "8200000000000000002B022C0100296F5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + } + } + })", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamCheckResumabilityEmptyForQueryStats) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamCheckResumability::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamCheckResumability": { + "resumeToken": { + "_data": "8200000000000000002B022C0100296E5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + } + } + })", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamCheckTopologyChangeEmptyForQueryStats) { + auto docSource = DocumentSourceChangeStreamCheckTopologyChange::create(getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalChangeStreamCheckTopologyChange":{}})", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamEnsureResumeTokenPresentEmptyForQueryStats) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamEnsureResumeTokenPresent::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamEnsureResumeTokenPresent": { + "resumeToken": { + "_data": "8200000000000000002B022C0100296E5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + } + } + })", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamHandleTopologyChangeEmptyForQueryStats) { + auto docSource = DocumentSourceChangeStreamHandleTopologyChange::create(getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalChangeStreamHandleTopologyChange":{}})", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamSplitLargeEvent) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamSplitLargeEvent::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$changeStreamSplitLargeEvent":{}})", + docSource->serialize().getDocument().toBson()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$changeStreamSplitLargeEvent":{}})", + redact(*docSource)); +} + +TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamTransform) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamTransform::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamTransform": { + "resumeAfter": { + "_data": "8200000000000000002B022C0100296E5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + }, + "fullDocument": "default", + "fullDocumentBeforeChange": "off" + } + })", + docSource->serialize().getDocument().toBson()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "resumeAfter": { + "_data": "?string" + }, + "fullDocument": "default", + "fullDocumentBeforeChange": "off" + } + })", + redact(*docSource)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "resumeAfter": { + "_data": "8200000000000000002B0229296E04" + }, + "fullDocument": "default", + "fullDocumentBeforeChange": "off" + } + })", + docSource + ->serialize( + SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}) + .getDocument() + .toBson()); +} + + +TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamTransformMoreFields) { + DocumentSourceChangeStreamSpec spec; + spec.setStartAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + spec.setFullDocument(FullDocumentModeEnum::kRequired); + spec.setFullDocumentBeforeChange(FullDocumentBeforeChangeModeEnum::kWhenAvailable); + spec.setShowExpandedEvents(true); + + auto docSource = DocumentSourceChangeStreamTransform::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamTransform": { + "startAfter": { + "_data": "8200000000000000002B022C0100296E5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + }, + "fullDocument": "required", + "fullDocumentBeforeChange": "whenAvailable", + "showExpandedEvents": true + } + })", + docSource->serialize().getDocument().toBson()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "startAfter": { + "_data": "?string" + }, + "fullDocument": "required", + "fullDocumentBeforeChange": "whenAvailable", + "showExpandedEvents": true + } + })", + redact(*docSource)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "startAfter": { + "_data": "8200000000000000002B0229296E04" + }, + "fullDocument": "required", + "fullDocumentBeforeChange": "whenAvailable", + "showExpandedEvents": true + } + })", + docSource + ->serialize( + SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}) + .getDocument() + .toBson()); +} + +// For DocumentSource types which contain an arbitrarily internal +// MatchExpression, we don't want match the entire structure. This +// assertion allows us to check some basic structure. +void assertRedactedMatchExpressionContainsOperatorsAndRedactedFieldPaths(BSONElement el) { + // Walk the redacted BSON and assert that we have some ops and + // redacted field paths. + auto opCount = 0; + auto redactedFieldPaths = 0; + while (true) { + if (el.type() == mongo::Array) { + auto array = el.Array(); + if (array.empty()) { + break; + } + el = array[0]; + } else if (el.type() == mongo::Object) { + auto obj = el.Obj(); + if (obj.begin() == obj.end()) { + break; + } + el = obj.firstElement(); + + // Field name should be an operator or a redacted field path. + if (el.fieldName()[0] == '$') { + opCount++; + } else if (!strcmp(el.fieldName(), "$regularExpression")) { + opCount++; + // Skip $regularExpression. + continue; + } else { + if (strstr(el.fieldName(), "HASH<") != el.fieldName()) { + FAIL(std::string("Expected redacted field path: ") + el.fieldName()); + } + redactedFieldPaths++; + } + } else { + break; + } + } + + ASSERT(opCount > 0); + ASSERT(redactedFieldPaths > 0); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamUnwindTransactionEmptyForQueryStats) { + auto docSource = DocumentSourceChangeStreamUnwindTransaction::create(getExpCtx()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamOplogMatchEmptyForQueryStats) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamOplogMatch::create(getExpCtx(), spec); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_transform.cpp b/src/mongo/db/pipeline/document_source_change_stream_transform.cpp index 7434190a609..3ea1312f51c 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_transform.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_transform.cpp @@ -73,7 +73,8 @@ DocumentSourceChangeStreamTransform::createFromBson( DocumentSourceChangeStreamTransform::DocumentSourceChangeStreamTransform( const boost::intrusive_ptr<ExpressionContext>& expCtx, DocumentSourceChangeStreamSpec spec) - : DocumentSource(DocumentSourceChangeStreamTransform::kStageName, expCtx), + : DocumentSourceInternalChangeStreamStage(DocumentSourceChangeStreamTransform::kStageName, + expCtx), _changeStreamSpec(std::move(spec)), _transformer(expCtx, _changeStreamSpec), _isIndependentOfAnyCollection(expCtx->ns.isCollectionlessAggregateNS()) { @@ -103,16 +104,115 @@ StageConstraints DocumentSourceChangeStreamTransform::constraints( return constraints; } -Value DocumentSourceChangeStreamTransform::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +namespace { + +template <typename T> +void serializeSpecField(BSONObjBuilder* builder, + const SerializationOptions& opts, + const StringData& fieldName, + const boost::optional<T>& value) { + if (value) { + opts.serializeLiteral((*value).toBSON()).addToBsonObj(builder, fieldName); + } +} + +template <> +void serializeSpecField(BSONObjBuilder* builder, + const SerializationOptions& opts, + const StringData& fieldName, + const boost::optional<Timestamp>& value) { + if (value) { + opts.serializeLiteral(*value).addToBsonObj(builder, fieldName); + } +} + +template <typename T> +void serializeSpecField(BSONObjBuilder* builder, + const SerializationOptions& opts, + const StringData& fieldName, + const T& value) { + opts.appendLiteral(builder, fieldName, value); +} + +template <> +void serializeSpecField(BSONObjBuilder* builder, + const SerializationOptions& opts, + const StringData& fieldName, + const mongo::OptionalBool& value) { + if (value.has_value()) { + opts.appendLiteral(builder, fieldName, value.value_or(true)); + } +} + +void serializeSpec(const DocumentSourceChangeStreamSpec& spec, + const SerializationOptions& opts, + BSONObjBuilder* builder) { + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kResumeAfterFieldName, + spec.getResumeAfter()); + serializeSpecField( + builder, opts, DocumentSourceChangeStreamSpec::kStartAfterFieldName, spec.getStartAfter()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kStartAtOperationTimeFieldName, + spec.getStartAtOperationTime()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kFullDocumentFieldName, + ::mongo::FullDocumentMode_serializer(spec.getFullDocument())); + serializeSpecField( + builder, + opts, + DocumentSourceChangeStreamSpec::kFullDocumentBeforeChangeFieldName, + ::mongo::FullDocumentBeforeChangeMode_serializer(spec.getFullDocumentBeforeChange())); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kAllChangesForClusterFieldName, + spec.getAllChangesForCluster()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kShowMigrationEventsFieldName, + spec.getShowMigrationEvents()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kShowSystemEventsFieldName, + spec.getShowSystemEvents()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kAllowToRunOnConfigDBFieldName, + spec.getAllowToRunOnConfigDB()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kAllowToRunOnSystemNSFieldName, + spec.getAllowToRunOnSystemNS()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName, + spec.getShowExpandedEvents()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kShowRawUpdateDescriptionFieldName, + spec.getShowRawUpdateDescription()); +} + +} // namespace + +Value DocumentSourceChangeStreamTransform::serialize(const SerializationOptions& opts) const { + if (opts.verbosity) { return Value(Document{{DocumentSourceChangeStream::kStageName, Document{{"stage"_sd, "internalTransform"_sd}, - {"options"_sd, _changeStreamSpec.toBSON()}}}}); + {"options"_sd, _changeStreamSpec.toBSON(opts)}}}}); } - return Value( - Document{{DocumentSourceChangeStreamTransform::kStageName, _changeStreamSpec.toBSON()}}); + // Internal change stream stages are not serialized for query stats. Query stats uses this stage + // to serialize the user specified stage, and therefore if serializing for query stats, we + // should use the '$changeStream' stage name. + auto stageName = + (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || opts.transformIdentifiers) + ? DocumentSourceChangeStream::kStageName + : DocumentSourceChangeStreamTransform::kStageName; + return Value(Document{{stageName, _changeStreamSpec.toBSON(opts)}}); } DepsTracker::State DocumentSourceChangeStreamTransform::getDependencies(DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/document_source_change_stream_transform.h b/src/mongo/db/pipeline/document_source_change_stream_transform.h index 71941800da9..52284954d70 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_transform.h +++ b/src/mongo/db/pipeline/document_source_change_stream_transform.h @@ -34,7 +34,7 @@ namespace mongo { -class DocumentSourceChangeStreamTransform : public DocumentSource { +class DocumentSourceChangeStreamTransform : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamTransform"_sd; @@ -54,7 +54,14 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + + /** + * This function should never be called, since this DocumentSource has its own serialize method. + */ + Value doSerialize(const SerializationOptions& opts) const final { + MONGO_UNREACHABLE; + } StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp index ae65c696d65..6f4877f878c 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp @@ -104,7 +104,7 @@ DocumentSourceChangeStreamUnwindTransaction::createFromBson( DocumentSourceChangeStreamUnwindTransaction::DocumentSourceChangeStreamUnwindTransaction( const BSONObj& filter, const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSource(kStageName, expCtx) { + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx) { rebuild(filter); } @@ -126,19 +126,23 @@ StageConstraints DocumentSourceChangeStreamUnwindTransaction::constraints( ChangeStreamRequirement::kChangeStreamStage); } -Value DocumentSourceChangeStreamUnwindTransaction::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - tassert(5467604, "expression has not been initialized", _expression); +Value DocumentSourceChangeStreamUnwindTransaction::doSerialize( + const SerializationOptions& opts) const { + tassert(7481400, "expression has not been initialized", _expression); - if (explain) { - return Value( - DOC(DocumentSourceChangeStream::kStageName << DOC("stage" - << "internalUnwindTransaction"_sd - << "filter" << _filter))); + if (opts.verbosity) { + BSONObjBuilder builder; + builder.append("stage"_sd, "internalUnwindTransaction"_sd); + builder.append(DocumentSourceChangeStreamUnwindTransactionSpec::kFilterFieldName, + _expression->serialize(opts)); + + return Value(DOC(DocumentSourceChangeStream::kStageName << builder.obj())); } - DocumentSourceChangeStreamUnwindTransactionSpec spec(_filter); - return Value(Document{{kStageName, Value(spec.toBSON())}}); + // 'SerializationOptions' are not required here, since serialization for explain and query + // stats occur before this function call. + return Value(Document{ + {kStageName, Value{DocumentSourceChangeStreamUnwindTransactionSpec{_filter}.toBSON()}}}); } DepsTracker::State DocumentSourceChangeStreamUnwindTransaction::getDependencies( diff --git a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h index ce7278b3b3e..d9e794f5551 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h +++ b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h @@ -41,7 +41,7 @@ namespace mongo { * output, but all other entries pass through unmodified. Note that the namespace filter applies * only to unwound transaction operations, not to any other entries. */ -class DocumentSourceChangeStreamUnwindTransaction : public DocumentSource { +class DocumentSourceChangeStreamUnwindTransaction : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamUnwindTransaction"_sd; @@ -55,7 +55,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/pipeline/document_source_coll_stats.cpp b/src/mongo/db/pipeline/document_source_coll_stats.cpp index 2881425d5e4..a947f1589e6 100644 --- a/src/mongo/db/pipeline/document_source_coll_stats.cpp +++ b/src/mongo/db/pipeline/document_source_coll_stats.cpp @@ -129,8 +129,8 @@ DocumentSource::GetNextResult DocumentSourceCollStats::doGetNext() { return {Document(makeStatsForNs(pExpCtx, pExpCtx->ns, _collStatsSpec))}; } -Value DocumentSourceCollStats::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{getSourceName(), _collStatsSpec.toBSON()}}); +Value DocumentSourceCollStats::serialize(const SerializationOptions& opts) const { + return Value(Document{{getSourceName(), _collStatsSpec.toBSON(opts)}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_coll_stats.h b/src/mongo/db/pipeline/document_source_coll_stats.h index 074683367e1..5c5f3d5482b 100644 --- a/src/mongo/db/pipeline/document_source_coll_stats.h +++ b/src/mongo/db/pipeline/document_source_coll_stats.h @@ -113,7 +113,7 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; static boost::intrusive_ptr<DocumentSource> createFromBson( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); diff --git a/src/mongo/db/pipeline/document_source_coll_stats.idl b/src/mongo/db/pipeline/document_source_coll_stats.idl index 264a4004a5c..b432912392a 100644 --- a/src/mongo/db/pipeline/document_source_coll_stats.idl +++ b/src/mongo/db/pipeline/document_source_coll_stats.idl @@ -39,36 +39,50 @@ structs: LatencyStatsSpec: description: Represents the 'latencyStats' argument to the $collStats stage. strict: true - fields: + query_shape_component: true + fields: histograms: description: Adds latency histogram information to the embedded documents in latencyStats if true. type: optionalBool + # Do not abstract this literal, since it is parameterizing the stage like an enum rather than representing + # real user input. + query_shape: parameter DocumentSourceCollStatsSpec: description: Specification for a $collStats stage. strict: true + query_shape_component: true fields: latencyStats: description: A request to include latency stats in the $collStats output. type: LatencyStatsSpec optional: true + query_shape: literal storageStats: description: Adds storage statistics to the return document. type: StorageStatsSpec optional: true + query_shape: literal count: description: Adds the total number of documents in the collection to the return document. type: object validator: callback: validateObjectIsEmpty optional: true + # Treat this value as a parameter (meaning it won't be abstracted or transformed) because + # the _only_ accepted value is the empty object. There's no transformation needed. + query_shape: parameter queryExecStats: description: Adds query execution statistics to the return document. type: object validator: callback: validateObjectIsEmpty optional: true + # Treat this value as a parameter (meaning it won't be abstracted or transformed) because + # the _only_ accepted value is the empty object. There's no transformation needed. + query_shape: parameter $_requestOnTimeseriesView: description: When set to true, $collStats stage requests statistics from the view namespace. When set to false, $collStats stage requests statistics from the underlying collection. cpp_name: requestOnTimeseriesView type: optionalBool + query_shape: parameter diff --git a/src/mongo/db/pipeline/document_source_coll_stats_test.cpp b/src/mongo/db/pipeline/document_source_coll_stats_test.cpp new file mode 100644 index 00000000000..f958307c8be --- /dev/null +++ b/src/mongo/db/pipeline/document_source_coll_stats_test.cpp @@ -0,0 +1,129 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_coll_stats.h" +#include "mongo/db/pipeline/document_source_coll_stats_gen.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/unittest/bson_test_util.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { +using DocumentSourceCollStatsTest = AggregationContextFixture; + +auto representativeShape(const DocumentSourceCollStats& collStatsStage) { + SerializationOptions opts{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + return collStatsStage.serialize(opts).getDocument().toBson(); +} + +TEST_F(DocumentSourceCollStatsTest, QueryShape) { + auto spec = DocumentSourceCollStatsSpec(); + + auto stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$collStats":{}})", + redact(*stage)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$collStats":{}})", + representativeShape(*stage)); + + spec.setCount(BSONObj()); + spec.setQueryExecStats(BSONObj()); + stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$collStats":{"count":{},"queryExecStats":{}}})", + redact(*stage)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$collStats":{"count":{},"queryExecStats":{}}})", + representativeShape(*stage)); + + auto latencyStats = LatencyStatsSpec(); + latencyStats.setHistograms(true); + spec.setLatencyStats(latencyStats); + stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$collStats": { + "latencyStats": { + "histograms": true + }, + "count": {}, + "queryExecStats": {} + } + })", + redact(*stage)); + + auto storageStats = StorageStatsSpec(); + storageStats.setScale(2); + storageStats.setVerbose(true); + spec.setStorageStats(storageStats); + stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$collStats": { + "latencyStats": { + "histograms": true + }, + "storageStats": { + "scale": "?number", + "verbose": true, + "waitForLock": true, + "numericOnly": false + }, + "count": {}, + "queryExecStats": {} + } + })", + redact(*stage)); + + storageStats.setWaitForLock(false); + storageStats.setNumericOnly(false); + spec.setStorageStats(storageStats); + stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$collStats": { + "latencyStats": { + "histograms": true + }, + "storageStats": { + "scale": "?number", + "verbose": true, + "waitForLock": false, + "numericOnly": false + }, + "count": {}, + "queryExecStats": {} + } + })", + redact(*stage)); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_count_test.cpp b/src/mongo/db/pipeline/document_source_count_test.cpp index 9ffd8a6d120..399086b0b52 100644 --- a/src/mongo/db/pipeline/document_source_count_test.cpp +++ b/src/mongo/db/pipeline/document_source_count_test.cpp @@ -65,10 +65,13 @@ public: dynamic_cast<DocumentSourceSingleDocumentTransformation*>(result.back().get()); ASSERT(projectStage); - auto explain = ExplainOptions::Verbosity::kQueryPlanner; vector<Value> explainedStages; - groupStage->serializeToArray(explainedStages, explain); - projectStage->serializeToArray(explainedStages, explain); + groupStage->serializeToArray( + explainedStages, + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); + projectStage->serializeToArray( + explainedStages, + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); ASSERT_EQUALS(explainedStages.size(), 2UL); StringData countName = countSpec.firstElement().valueStringData(); diff --git a/src/mongo/db/pipeline/document_source_current_op.cpp b/src/mongo/db/pipeline/document_source_current_op.cpp index 6ef6e2c762e..fc5c8b32a1f 100644 --- a/src/mongo/db/pipeline/document_source_current_op.cpp +++ b/src/mongo/db/pipeline/document_source_current_op.cpp @@ -70,8 +70,8 @@ std::unique_ptr<DocumentSourceCurrentOp::LiteParsed> DocumentSourceCurrentOp::Li << typeName(spec.type())); } - auto allUsers = UserMode::kExcludeOthers; - auto localOps = LocalOpsMode::kRemoteShardOps; + auto allUsers = kDefaultUserMode; + auto localOps = kDefaultLocalOpsMode; // Check the spec for all fields named 'allUsers'. If any of them are 'true', we require // the 'inprog' privilege. This avoids the possibility that a spec with multiple @@ -113,13 +113,14 @@ const char* DocumentSourceCurrentOp::getSourceName() const { DocumentSource::GetNextResult DocumentSourceCurrentOp::doGetNext() { if (_ops.empty()) { - _ops = pExpCtx->mongoProcessInterface->getCurrentOps(pExpCtx, - _includeIdleConnections, - _includeIdleSessions, - _includeOpsFromAllUsers, - _truncateOps, - _idleCursors, - _backtrace); + _ops = pExpCtx->mongoProcessInterface->getCurrentOps( + pExpCtx, + _includeIdleConnections.value_or(kDefaultConnMode), + _includeIdleSessions.value_or(kDefaultSessionMode), + _includeOpsFromAllUsers.value_or(kDefaultUserMode), + _truncateOps.value_or(kDefaultTruncationMode), + _idleCursors.value_or(kDefaultCursorMode), + _backtrace.value_or(kDefaultBacktraceMode)); _opsIter = _ops.begin(); @@ -191,13 +192,13 @@ intrusive_ptr<DocumentSource> DocumentSourceCurrentOp::createFromBson( "$currentOp must be run against the 'admin' database with {aggregate: 1}", nss.db() == NamespaceString::kAdminDb && nss.isCollectionlessAggregateNS()); - ConnMode includeIdleConnections = ConnMode::kExcludeIdle; - SessionMode includeIdleSessions = SessionMode::kIncludeIdle; - UserMode includeOpsFromAllUsers = UserMode::kExcludeOthers; - LocalOpsMode showLocalOpsOnMongoS = LocalOpsMode::kRemoteShardOps; - TruncationMode truncateOps = TruncationMode::kNoTruncation; - CursorMode idleCursors = CursorMode::kExcludeCursors; - BacktraceMode backtrace = BacktraceMode::kExcludeBacktrace; + boost::optional<ConnMode> includeIdleConnections; + boost::optional<SessionMode> includeIdleSessions; + boost::optional<UserMode> includeOpsFromAllUsers; + boost::optional<LocalOpsMode> showLocalOpsOnMongoS; + boost::optional<TruncationMode> truncateOps; + boost::optional<CursorMode> idleCursors; + boost::optional<BacktraceMode> backtrace; for (auto&& elem : spec.embeddedObject()) { const auto fieldName = elem.fieldNameStringData(); @@ -278,13 +279,13 @@ intrusive_ptr<DocumentSource> DocumentSourceCurrentOp::createFromBson( intrusive_ptr<DocumentSourceCurrentOp> DocumentSourceCurrentOp::create( const boost::intrusive_ptr<ExpressionContext>& pExpCtx, - ConnMode includeIdleConnections, - SessionMode includeIdleSessions, - UserMode includeOpsFromAllUsers, - LocalOpsMode showLocalOpsOnMongoS, - TruncationMode truncateOps, - CursorMode idleCursors, - BacktraceMode backtrace) { + boost::optional<ConnMode> includeIdleConnections, + boost::optional<SessionMode> includeIdleSessions, + boost::optional<UserMode> includeOpsFromAllUsers, + boost::optional<LocalOpsMode> showLocalOpsOnMongoS, + boost::optional<TruncationMode> truncateOps, + boost::optional<CursorMode> idleCursors, + boost::optional<BacktraceMode> backtrace) { return new DocumentSourceCurrentOp(pExpCtx, includeIdleConnections, includeIdleSessions, @@ -295,22 +296,38 @@ intrusive_ptr<DocumentSourceCurrentOp> DocumentSourceCurrentOp::create( backtrace); } -Value DocumentSourceCurrentOp::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceCurrentOp::serialize(const SerializationOptions& opts) const { return Value(Document{ {getSourceName(), - Document{{kIdleConnectionsFieldName, - _includeIdleConnections == ConnMode::kIncludeIdle ? Value(true) : Value()}, - {kIdleSessionsFieldName, - _includeIdleSessions == SessionMode::kExcludeIdle ? Value(false) : Value()}, - {kAllUsersFieldName, - _includeOpsFromAllUsers == UserMode::kIncludeAll ? Value(true) : Value()}, - {kLocalOpsFieldName, - _showLocalOpsOnMongoS == LocalOpsMode::kLocalMongosOps ? Value(true) : Value()}, - {kTruncateOpsFieldName, - _truncateOps == TruncationMode::kTruncateOps ? Value(true) : Value()}, - {kIdleCursorsFieldName, - _idleCursors == CursorMode::kIncludeCursors ? Value(true) : Value()}, - {kBacktraceFieldName, - _backtrace == BacktraceMode::kIncludeBacktrace ? Value(true) : Value()}}}}); + Document{ + {kIdleConnectionsFieldName, + _includeIdleConnections.has_value() + ? opts.serializeLiteral(_includeIdleConnections.value() == ConnMode::kIncludeIdle) + : Value()}, + {kIdleSessionsFieldName, + _includeIdleSessions.has_value() + ? opts.serializeLiteral(_includeIdleSessions.value() == SessionMode::kIncludeIdle) + : Value()}, + {kAllUsersFieldName, + _includeOpsFromAllUsers.has_value() + ? opts.serializeLiteral(_includeOpsFromAllUsers.value() == UserMode::kIncludeAll) + : Value()}, + {kLocalOpsFieldName, + _showLocalOpsOnMongoS.has_value() + ? opts.serializeLiteral(_showLocalOpsOnMongoS.value() == + LocalOpsMode::kLocalMongosOps) + : Value()}, + {kTruncateOpsFieldName, + _truncateOps.has_value() + ? opts.serializeLiteral(_truncateOps.value() == TruncationMode::kTruncateOps) + : Value()}, + {kIdleCursorsFieldName, + _idleCursors.has_value() + ? opts.serializeLiteral(_idleCursors.value() == CursorMode::kIncludeCursors) + : Value()}, + {kBacktraceFieldName, + _backtrace.has_value() + ? opts.serializeLiteral(_backtrace.value() == BacktraceMode::kIncludeBacktrace) + : Value()}}}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_current_op.h b/src/mongo/db/pipeline/document_source_current_op.h index 49e0be6c35d..4cc53475e4b 100644 --- a/src/mongo/db/pipeline/document_source_current_op.h +++ b/src/mongo/db/pipeline/document_source_current_op.h @@ -45,6 +45,14 @@ public: static constexpr StringData kStageName = "$currentOp"_sd; + static constexpr ConnMode kDefaultConnMode = ConnMode::kExcludeIdle; + static constexpr SessionMode kDefaultSessionMode = SessionMode::kIncludeIdle; + static constexpr UserMode kDefaultUserMode = UserMode::kExcludeOthers; + static constexpr LocalOpsMode kDefaultLocalOpsMode = LocalOpsMode::kRemoteShardOps; + static constexpr TruncationMode kDefaultTruncationMode = TruncationMode::kNoTruncation; + static constexpr CursorMode kDefaultCursorMode = CursorMode::kExcludeCursors; + static constexpr BacktraceMode kDefaultBacktraceMode = BacktraceMode::kExcludeBacktrace; + class LiteParsed final : public LiteParsedDocumentSource { public: static std::unique_ptr<LiteParsed> parse(const NamespaceString& nss, @@ -98,29 +106,28 @@ public: static boost::intrusive_ptr<DocumentSourceCurrentOp> create( const boost::intrusive_ptr<ExpressionContext>& pExpCtx, - ConnMode includeIdleConnections = ConnMode::kExcludeIdle, - SessionMode includeIdleSessions = SessionMode::kIncludeIdle, - UserMode includeOpsFromAllUsers = UserMode::kExcludeOthers, - LocalOpsMode showLocalOpsOnMongoS = LocalOpsMode::kRemoteShardOps, - TruncationMode truncateOps = TruncationMode::kNoTruncation, - CursorMode idleCursors = CursorMode::kExcludeCursors, - BacktraceMode backtrace = BacktraceMode::kExcludeBacktrace); + boost::optional<ConnMode> includeIdleConnections = boost::none, + boost::optional<SessionMode> includeIdleSessions = boost::none, + boost::optional<UserMode> includeOpsFromAllUsers = boost::none, + boost::optional<LocalOpsMode> showLocalOpsOnMongoS = boost::none, + boost::optional<TruncationMode> truncateOps = boost::none, + boost::optional<CursorMode> idleCursors = boost::none, + boost::optional<BacktraceMode> backtrace = boost::none); const char* getSourceName() const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final { - StageConstraints constraints(StreamType::kStreaming, - PositionRequirement::kFirst, - (_showLocalOpsOnMongoS == LocalOpsMode::kLocalMongosOps - ? HostTypeRequirement::kLocalOnly - : HostTypeRequirement::kAnyShard), - DiskUseRequirement::kNoDiskUse, - FacetRequirement::kNotAllowed, - TransactionRequirement::kNotAllowed, - LookupRequirement::kAllowed, - (_showLocalOpsOnMongoS == LocalOpsMode::kLocalMongosOps - ? UnionRequirement::kNotAllowed - : UnionRequirement::kAllowed)); + bool showLocalOps = + _showLocalOpsOnMongoS.value_or(kDefaultLocalOpsMode) == LocalOpsMode::kLocalMongosOps; + StageConstraints constraints( + StreamType::kStreaming, + PositionRequirement::kFirst, + (showLocalOps ? HostTypeRequirement::kLocalOnly : HostTypeRequirement::kAnyShard), + DiskUseRequirement::kNoDiskUse, + FacetRequirement::kNotAllowed, + TransactionRequirement::kNotAllowed, + LookupRequirement::kAllowed, + (showLocalOps ? UnionRequirement::kNotAllowed : UnionRequirement::kAllowed)); constraints.isIndependentOfAnyCollection = true; constraints.requiresInputDocSource = false; @@ -134,17 +141,17 @@ public: static boost::intrusive_ptr<DocumentSource> createFromBson( BSONElement spec, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; private: DocumentSourceCurrentOp(const boost::intrusive_ptr<ExpressionContext>& pExpCtx, - ConnMode includeIdleConnections, - SessionMode includeIdleSessions, - UserMode includeOpsFromAllUsers, - LocalOpsMode showLocalOpsOnMongoS, - TruncationMode truncateOps, - CursorMode idleCursors, - BacktraceMode backtrace) + boost::optional<ConnMode> includeIdleConnections, + boost::optional<SessionMode> includeIdleSessions, + boost::optional<UserMode> includeOpsFromAllUsers, + boost::optional<LocalOpsMode> showLocalOpsOnMongoS, + boost::optional<TruncationMode> truncateOps, + boost::optional<CursorMode> idleCursors, + boost::optional<BacktraceMode> backtrace) : DocumentSource(kStageName, pExpCtx), _includeIdleConnections(includeIdleConnections), _includeIdleSessions(includeIdleSessions), @@ -156,13 +163,13 @@ private: GetNextResult doGetNext() final; - ConnMode _includeIdleConnections = ConnMode::kExcludeIdle; - SessionMode _includeIdleSessions = SessionMode::kIncludeIdle; - UserMode _includeOpsFromAllUsers = UserMode::kExcludeOthers; - LocalOpsMode _showLocalOpsOnMongoS = LocalOpsMode::kRemoteShardOps; - TruncationMode _truncateOps = TruncationMode::kNoTruncation; - CursorMode _idleCursors = CursorMode::kExcludeCursors; - BacktraceMode _backtrace = BacktraceMode::kExcludeBacktrace; + boost::optional<ConnMode> _includeIdleConnections; + boost::optional<SessionMode> _includeIdleSessions; + boost::optional<UserMode> _includeOpsFromAllUsers; + boost::optional<LocalOpsMode> _showLocalOpsOnMongoS; + boost::optional<TruncationMode> _truncateOps; + boost::optional<CursorMode> _idleCursors; + boost::optional<BacktraceMode> _backtrace; std::string _shardName; diff --git a/src/mongo/db/pipeline/document_source_current_op_test.cpp b/src/mongo/db/pipeline/document_source_current_op_test.cpp index 260bfbf0829..440205960f2 100644 --- a/src/mongo/db/pipeline/document_source_current_op_test.cpp +++ b/src/mongo/db/pipeline/document_source_current_op_test.cpp @@ -154,10 +154,10 @@ TEST_F(DocumentSourceCurrentOpTest, ShouldFailToParseIfUnrecognisedParameterSpec ErrorCodes::FailedToParse); } -TEST_F(DocumentSourceCurrentOpTest, ShouldParseAndSerializeNonDefaultOptionalArguments) { +TEST_F(DocumentSourceCurrentOpTest, ShouldParseAndSerializeAllExplicitlySpecifiedArguments) { const auto specObj = fromjson( - "{$currentOp:{idleConnections:true, idleSessions:false, allUsers:true, localOps:true, " - "truncateOps:true}}"); + "{$currentOp:{idleConnections:false, idleSessions:false, allUsers:true, localOps:true, " + "truncateOps:false}}"); const auto parsed = DocumentSourceCurrentOp::createFromBson(specObj.firstElement(), getExpCtx()); @@ -165,19 +165,42 @@ TEST_F(DocumentSourceCurrentOpTest, ShouldParseAndSerializeNonDefaultOptionalArg const auto currentOp = static_cast<DocumentSourceCurrentOp*>(parsed.get()); const auto expectedOutput = Document{{"$currentOp", - Document{{"idleConnections", true}, + Document{{"idleConnections", false}, {"idleSessions", false}, {"allUsers", true}, {"localOps", true}, - {"truncateOps", true}}}}; + {"truncateOps", false}}}}; ASSERT_DOCUMENT_EQ(currentOp->serialize().getDocument(), expectedOutput); } -TEST_F(DocumentSourceCurrentOpTest, ShouldParseButNotSerializeDefaultOptionalArguments) { - const auto specObj = fromjson( - "{$currentOp:{idleConnections:false, idleSessions:true, allUsers:false, localOps:false, " - "truncateOps:false}}"); +TEST_F(DocumentSourceCurrentOpTest, + ShouldParseAndSerializeAllExplicitlySpecifiedArgumentsWithRedaction) { + auto spec = fromjson( + R"({ + $currentOp: { + idleConnections: true, + allUsers: false, + idleSessions: false, + localOps: true + } + })"); + auto docSource = DocumentSourceCurrentOp::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$currentOp": { + "idleConnections": "?bool", + "idleSessions": "?bool", + "allUsers": "?bool", + "localOps": "?bool" + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceCurrentOpTest, ShouldNotSerializeOmittedOptionalArguments) { + const auto specObj = fromjson("{$currentOp:{}}"); const auto parsed = DocumentSourceCurrentOp::createFromBson(specObj.firstElement(), getExpCtx()); @@ -189,17 +212,17 @@ TEST_F(DocumentSourceCurrentOpTest, ShouldParseButNotSerializeDefaultOptionalArg ASSERT_DOCUMENT_EQ(currentOp->serialize().getDocument(), expectedOutput); } -TEST_F(DocumentSourceCurrentOpTest, ShouldNotSerializeOmittedOptionalArguments) { +TEST_F(DocumentSourceCurrentOpTest, ShouldNotSerializeOmittedOptionalArgumentsWithRedaction) { const auto specObj = fromjson("{$currentOp:{}}"); - const auto parsed = + const auto docSource = DocumentSourceCurrentOp::createFromBson(specObj.firstElement(), getExpCtx()); - const auto currentOp = static_cast<DocumentSourceCurrentOp*>(parsed.get()); - - const auto expectedOutput = Document{{"$currentOp", Document{}}}; - - ASSERT_DOCUMENT_EQ(currentOp->serialize().getDocument(), expectedOutput); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$currentOp": {} + })", + redact(*docSource)); } TEST_F(DocumentSourceCurrentOpTest, ShouldReturnEOFImmediatelyIfNoCurrentOps) { diff --git a/src/mongo/db/pipeline/document_source_cursor.cpp b/src/mongo/db/pipeline/document_source_cursor.cpp index 55bacba9d7f..520c32f74cc 100644 --- a/src/mongo/db/pipeline/document_source_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_cursor.cpp @@ -206,9 +206,12 @@ void DocumentSourceCursor::recordPlanSummaryStats() { _exec->getPlanExplainer().getSummaryStats(&_stats.planSummaryStats); } -Value DocumentSourceCursor::serialize(boost::optional<ExplainOptions::Verbosity> verbosity) const { - // We never parse a DocumentSourceCursor, so we only serialize for explain. - if (!verbosity) +Value DocumentSourceCursor::serialize(const SerializationOptions& opts) const { + auto verbosity = opts.verbosity; + // We never parse a DocumentSourceCursor, so we only serialize for explain. Since it's never + // part of user input, there's no need to compute its query shape. + if (!verbosity || opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) return Value(); invariant(_exec); diff --git a/src/mongo/db/pipeline/document_source_cursor.h b/src/mongo/db/pipeline/document_source_cursor.h index 38a2ab183ae..29f724da0a1 100644 --- a/src/mongo/db/pipeline/document_source_cursor.h +++ b/src/mongo/db/pipeline/document_source_cursor.h @@ -72,7 +72,7 @@ public: const char* getSourceName() const override; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { StageConstraints constraints(StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_densify.cpp b/src/mongo/db/pipeline/document_source_densify.cpp index 63f3253f57a..b7899f078a2 100644 --- a/src/mongo/db/pipeline/document_source_densify.cpp +++ b/src/mongo/db/pipeline/document_source_densify.cpp @@ -645,17 +645,16 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceInternalDensify::createFromBs return results.front(); } -Value DocumentSourceInternalDensify::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalDensify::serialize(const SerializationOptions& opts) const { MutableDocument spec; - spec[kFieldFieldName] = Value(_field.fullPath()); + spec[kFieldFieldName] = Value(opts.serializeFieldPath(_field)); std::vector<Value> serializedPartitionByFields(_partitions.size()); std::transform(_partitions.begin(), _partitions.end(), serializedPartitionByFields.begin(), - [&](FieldPath field) -> Value { return Value(field.fullPath()); }); + [&](FieldPath field) -> Value { return Value(opts.serializeFieldPath(field)); }); spec[kPartitionByFieldsFieldName] = Value(serializedPartitionByFields); - spec[kRangeFieldName] = _range.serialize(); + spec[kRangeFieldName] = _range.serialize(opts); MutableDocument out; out[getSourceName()] = Value(spec.freeze()); diff --git a/src/mongo/db/pipeline/document_source_densify.h b/src/mongo/db/pipeline/document_source_densify.h index 6415c5425d8..1214c0bac53 100644 --- a/src/mongo/db/pipeline/document_source_densify.h +++ b/src/mongo/db/pipeline/document_source_densify.h @@ -241,15 +241,16 @@ public: static RangeStatement parse(RangeSpec spec); - Value serialize() const { + Value serialize(const SerializationOptions& opts) const { MutableDocument spec; - spec[kArgStep] = _step; + spec[kArgStep] = opts.serializeLiteral(_step); spec[kArgBounds] = stdx::visit( visit_helper::Overloaded{[&](Full) { return Value(kValFull); }, [&](Partition) { return Value(kValPartition); }, [&](ExplicitBounds bounds) { return Value(std::vector<Value>( - {bounds.first.toValue(), bounds.second.toValue()})); + {opts.serializeLiteral(bounds.first.toValue()), + opts.serializeLiteral(bounds.second.toValue())})); }}, _bounds); if (_unit) @@ -380,7 +381,7 @@ public: return kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DepsTracker::State getDependencies(DepsTracker* deps) const final { deps->fields.insert(_field.fullPath()); diff --git a/src/mongo/db/pipeline/document_source_densify_test.cpp b/src/mongo/db/pipeline/document_source_densify_test.cpp index 4be40700323..f83101b7736 100644 --- a/src/mongo/db/pipeline/document_source_densify_test.cpp +++ b/src/mongo/db/pipeline/document_source_densify_test.cpp @@ -49,6 +49,7 @@ using DensifyExplicitNumericTest = AggregationContextFixture; using DensifyPartitionNumericTest = AggregationContextFixture; using DensifyCloneTest = AggregationContextFixture; using DensifyStepTest = AggregationContextFixture; +using DensifyRedactionTest = AggregationContextFixture; Date_t makeDate(std::string dateStr) { auto statusDate = dateFromISOString(dateStr); @@ -1438,5 +1439,174 @@ TEST(DensifyStepTest, InternalDensifyIsOffStepForDaysWithLargeDateStep) { ASSERT_FALSE(val.isOnStepRelativeTo(base, range)); } + +TEST_F(DensifyRedactionTest, RedactionDateBounds) { + auto spec = fromjson(R"({ + $densify: { + field: "a", + range: { + step: 1, + unit: "hour", + bounds: [ + {$date: "2023-04-23T00:00:00.000Z"}, + {$date: "2023-04-23T08:00:00.000Z"} + ] + } + } + })"); + + auto docSource = + DocumentSourceInternalDensify::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalDensify": { + "field": "HASH<a>", + "partitionByFields": [], + "range": { + "step": "?number", + "bounds": [ + "?date", + "?date" + ], + "unit": "hour" + } + } + })", + redact(*docSource)); +} + +TEST_F(DensifyRedactionTest, RedactionFullBoundsWithPartitionFields) { + auto spec = fromjson(R"({ + $densify: { + field: "foo", + partitionByFields: ["a", "b", "c.d"], + range: { + bounds: "full", + step: 100 + } + } + })"); + auto docSource = + DocumentSourceInternalDensify::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalDensify": { + "field": "HASH<foo>", + "partitionByFields": [ + "HASH<a>", + "HASH<b>", + "HASH<c>.HASH<d>" + ], + "range": { + "step": "?number", + "bounds": "full" + } + } + })", + redact(*docSource)); +} + +TEST_F(DensifyRedactionTest, RedactionPartitionBounds) { + auto spec = fromjson(R"({ + $densify: { + field: "x", + partitionByFields: ["foo"], + range: { + bounds: "partition", + step: 50, + unit: "second" + } + } + })"); + auto docSource = + DocumentSourceInternalDensify::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalDensify": { + "field": "HASH<x>", + "partitionByFields": [ + "HASH<foo>" + ], + "range": { + "step": "?number", + "bounds": "partition", + "unit": "second" + } + } + })", + redact(*docSource)); +} + +void assertRangeTimeUnitSerialization(const boost::intrusive_ptr<ExpressionContext>& expCtx, + BSONObj inputStage, + BSONObj expectedStage, + const SerializationOptions& opts) { + auto parsedStage = + DocumentSourceInternalDensify::createFromBson(inputStage.firstElement(), expCtx); + std::vector<Value> serialization; + parsedStage->serializeToArray(serialization, opts); + + auto serializedStage = serialization[0].getDocument().toBson(); + ASSERT_BSONOBJ_EQ(expectedStage, serializedStage); +} + +TEST_F(DensifyRedactionTest, RangeTimeUnitSerializationRepresentative) { + assertRangeTimeUnitSerialization( + getExpCtx(), + fromjson(R"({ + $densify: { + field: "x", + partitionByFields: ["foo"], + range: { + bounds: "partition", + step: 50, + unit: "second" + } + } + })"), + fromjson(R"({ + $_internalDensify: { + field: "x", + partitionByFields: [ + "foo" + ], + range: { + step: 1, + bounds: "partition", + unit: "second" + } + } + })"), + SerializationOptions::kRepresentativeQueryShapeSerializeOptions); +} + +TEST_F(DensifyRedactionTest, RangeTimeUnitSerializationDebug) { + assertRangeTimeUnitSerialization(getExpCtx(), + fromjson( + R"({ + $densify: { + field: "x", + partitionByFields: ["foo"], + range: { + bounds: "partition", + step: 50, + unit: "second" + } + } + })"), + fromjson( + R"({ + $_internalDensify: { + field: "x", + partitionByFields: ["foo"], + range: { + step: "?number", + bounds: "partition", + unit: "second" + } + } + })"), + SerializationOptions::kDebugQueryShapeSerializeOptions); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_documents.cpp b/src/mongo/db/pipeline/document_source_documents.cpp index 934116a7c27..23f5b8c96f8 100644 --- a/src/mongo/db/pipeline/document_source_documents.cpp +++ b/src/mongo/db/pipeline/document_source_documents.cpp @@ -50,26 +50,26 @@ REGISTER_DOCUMENT_SOURCE(documents, std::list<intrusive_ptr<DocumentSource>> DocumentSourceDocuments::createFromBson( BSONElement elem, const intrusive_ptr<ExpressionContext>& expCtx) { - // genField is a temporary field to hold docs to wire $project, - // $unwind, and $replaceRoot together. - auto genField = UUID::gen().toString(); - auto projectContent = BSON(genField << elem); + // kGenFieldName is a temporary field to hold docs to wire $project, $unwind, and $replaceRoot + // together. This may show up in explain, but it will not possibly overlap with user data, since + // the user data is nested a level below, within an array or object. + auto projectContent = BSON(kGenFieldName << elem); auto queue = DocumentSourceQueue::create(expCtx, DocumentSourceDocuments::kStageName); queue->emplace_back(Document{}); /* Create the following pipeline from $documents: [...] - * => [ queue([{}]), - * project: {tempDocumentsField: [...]}, - * unwind: "$tempDocumentsField", - * replaceWith: "$tempDocumentsField" ] + * => [ {$queue: [{}] }, + * {$project: {[kGenFieldName]: [...]}}, + * {$unwind: "$" + kGenFieldName}, + * {$replaceWith: "$" + kGenFieldName} ] */ - return { - queue, - DocumentSourceProject::create(projectContent, expCtx, elem.fieldNameStringData()), - DocumentSourceUnwind::create(expCtx, genField, false, {}, true), - DocumentSourceReplaceRoot::create(expCtx, - ExpressionFieldPath::createPathFromString( - expCtx.get(), genField, expCtx->variablesParseState), - "elements within the array passed to $documents")}; + return {queue, + DocumentSourceProject::create(projectContent, expCtx, elem.fieldNameStringData()), + DocumentSourceUnwind::create(expCtx, kGenFieldName, false, {}, true), + DocumentSourceReplaceRoot::create( + expCtx, + ExpressionFieldPath::createPathFromString( + expCtx.get(), kGenFieldName, expCtx->variablesParseState), + "elements within the array passed to $documents")}; } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_documents.h b/src/mongo/db/pipeline/document_source_documents.h index d0c2b660d87..992b6c6cc4e 100644 --- a/src/mongo/db/pipeline/document_source_documents.h +++ b/src/mongo/db/pipeline/document_source_documents.h @@ -64,9 +64,10 @@ public: } }; +static const std::string kGenFieldName = "_tempDocumentsField"s; static constexpr StringData kStageName = "$documents"_sd; -static std::list<boost::intrusive_ptr<DocumentSource>> createFromBson( +std::list<boost::intrusive_ptr<DocumentSource>> createFromBson( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); }; // namespace DocumentSourceDocuments diff --git a/src/mongo/db/pipeline/document_source_documents_test.cpp b/src/mongo/db/pipeline/document_source_documents_test.cpp new file mode 100644 index 00000000000..3067df16abc --- /dev/null +++ b/src/mongo/db/pipeline/document_source_documents_test.cpp @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_documents.h" +#include "mongo/db/pipeline/document_source_unwind.h" + +namespace mongo { + +using DocumentSourceDocumentsTest = AggregationContextFixture; + +TEST_F(DocumentSourceDocumentsTest, DocumentsStageRedactsCorrectly) { + auto spec = fromjson(R"({ + $documents: [ + { x: 10 }, { x: 2 }, { x: 5 } + ] + })"); + auto docSourcesList = DocumentSourceDocuments::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_EQ(4, docSourcesList.size()); + + // We must retrieve the internally-generated field name shared across these stages in order to + // make sure they're serialized properly. + std::vector<boost::intrusive_ptr<DocumentSource>> docSourcesVec(docSourcesList.begin(), + docSourcesList.end()); + auto unwindStage = static_cast<DocumentSourceUnwind*>(docSourcesVec[2].get()); + ASSERT(unwindStage); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$queue":"?array<?object>"})", + redact(*docSourcesVec[0])); + + + ASSERT_BSONOBJ_EQ( // NOLINT + fromjson(R"({ + "$project": { + "HASH<_id>": true, + "HASH<)" + + // SERVER-87666 ensure that the generated field is consistent. + DocumentSourceDocuments::kGenFieldName + + R"(>": "?array<?object>" + } + })"), + redact(*docSourcesVec[1])); + + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + "{'$unwind': {'path' : '$HASH<" + DocumentSourceDocuments::kGenFieldName + ">' } }", + redact(*docSourcesVec[2])); + + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + "{$replaceRoot: {newRoot: '$HASH<" + DocumentSourceDocuments::kGenFieldName + ">'}}", + redact(*docSourcesVec[3])); +} + +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_exchange.cpp b/src/mongo/db/pipeline/document_source_exchange.cpp index 931b78f316e..441779021e9 100644 --- a/src/mongo/db/pipeline/document_source_exchange.cpp +++ b/src/mongo/db/pipeline/document_source_exchange.cpp @@ -91,8 +91,8 @@ const char* DocumentSourceExchange::getSourceName() const { return kStageName.rawData(); } -Value DocumentSourceExchange::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << _exchange->getSpec().toBSON())); +Value DocumentSourceExchange::serialize(const SerializationOptions& opts) const { + return Value(DOC(getSourceName() << _exchange->getSpec().toBSON(opts))); } DocumentSourceExchange::DocumentSourceExchange( diff --git a/src/mongo/db/pipeline/document_source_exchange.h b/src/mongo/db/pipeline/document_source_exchange.h index dd2c00030de..544f6fb924f 100644 --- a/src/mongo/db/pipeline/document_source_exchange.h +++ b/src/mongo/db/pipeline/document_source_exchange.h @@ -221,7 +221,7 @@ public: const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; /** * DocumentSourceExchange does not have a direct source (it is reading through the shared diff --git a/src/mongo/db/pipeline/document_source_exchange_test.cpp b/src/mongo/db/pipeline/document_source_exchange_test.cpp index ac234357b17..4ed642c49c7 100644 --- a/src/mongo/db/pipeline/document_source_exchange_test.cpp +++ b/src/mongo/db/pipeline/document_source_exchange_test.cpp @@ -747,4 +747,30 @@ TEST_F(DocumentSourceExchangeTest, RejectInvalidMissingKeys) { Exchange(parseSpec(spec), Pipeline::create({}, getExpCtx())), AssertionException, 50967); } +TEST_F(DocumentSourceExchangeTest, QueryShape) { + const size_t nDocs = 500; + + auto source = getMockSource(nDocs); + + ExchangeSpec spec; + spec.setPolicy(ExchangePolicyEnum::kRoundRobin); + spec.setConsumers(1); + spec.setBufferSize(1024); + boost::intrusive_ptr<Exchange> ex = new Exchange(spec, Pipeline::create({source}, getExpCtx())); + boost::intrusive_ptr<DocumentSourceExchange> stage = + new DocumentSourceExchange(getExpCtx(), ex, 0, nullptr); + + ASSERT_BSONOBJ_EQ_AUTO( // + R"({ + "$_internalExchange": { + "policy": "roundrobin", + "consumers": "?number", + "orderPreserving": false, + "bufferSize": "?number", + "key": "?object" + } + })", + redact(*stage)); +} + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_facet.cpp b/src/mongo/db/pipeline/document_source_facet.cpp index bc02ae2b88b..6dfe8fd0a68 100644 --- a/src/mongo/db/pipeline/document_source_facet.cpp +++ b/src/mongo/db/pipeline/document_source_facet.cpp @@ -185,11 +185,12 @@ DocumentSource::GetNextResult DocumentSourceFacet::doGetNext() { return resultDoc.freeze(); } -Value DocumentSourceFacet::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceFacet::serialize(const SerializationOptions& opts) const { MutableDocument serialized; for (auto&& facet : _facets) { - serialized[facet.name] = Value(explain ? facet.pipeline->writeExplainOps(*explain) - : facet.pipeline->serialize()); + serialized[opts.serializeFieldPathFromString(facet.name)] = + Value(opts.verbosity ? facet.pipeline->writeExplainOps(opts) + : facet.pipeline->serialize(opts)); } return Value(Document{{"$facet", serialized.freezeToValue()}}); } @@ -336,20 +337,25 @@ intrusive_ptr<DocumentSource> DocumentSourceFacet::createFromBson( }); }); - // Validate that none of the facet pipelines have any conflicting HostTypeRequirements. This - // verifies both that all stages within each pipeline are consistent, and that the pipelines - // are consistent with one another. - if (!needsShard && pipeline->needsShard()) { - needsShard.emplace(facetName); - } - if (!needsMongoS && pipeline->needsMongosMerger()) { - needsMongoS.emplace(facetName); + // These checks potentially require that we check the catalog to determine where our data + // lives. In circumstances where we aren't actually running the query, we don't need to do + // this (and it can erroneously error - SERVER-83912). + if (expCtx->mongoProcessInterface->isExpectedToExecuteQueries()) { + // Validate that none of the facet pipelines have any conflicting HostTypeRequirements. + // This verifies both that all stages within each pipeline are consistent, and that the + // pipelines are consistent with one another. + if (!needsShard && pipeline->needsShard()) { + needsShard.emplace(facetName); + } + if (!needsMongoS && pipeline->needsMongosMerger()) { + needsMongoS.emplace(facetName); + } + uassert(ErrorCodes::IllegalOperation, + str::stream() << "$facet pipeline '" << *needsMongoS + << "' must run on mongoS, but '" << *needsShard + << "' requires a shard", + !(needsShard && needsMongoS)); } - uassert(ErrorCodes::IllegalOperation, - str::stream() << "$facet pipeline '" << *needsMongoS - << "' must run on mongoS, but '" << *needsShard - << "' requires a shard", - !(needsShard && needsMongoS)); facetPipelines.emplace_back(facetName, std::move(pipeline)); } diff --git a/src/mongo/db/pipeline/document_source_facet.h b/src/mongo/db/pipeline/document_source_facet.h index 193b9e12dd1..e3efb9c37cd 100644 --- a/src/mongo/db/pipeline/document_source_facet.h +++ b/src/mongo/db/pipeline/document_source_facet.h @@ -160,7 +160,7 @@ private: size_t bufferSizeBytes, size_t maxOutputDocBytes); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<TeeBuffer> _teeBuffer; std::vector<FacetPipeline> _facets; diff --git a/src/mongo/db/pipeline/document_source_facet_test.cpp b/src/mongo/db/pipeline/document_source_facet_test.cpp index 44e3d9668e9..78367b24b02 100644 --- a/src/mongo/db/pipeline/document_source_facet_test.cpp +++ b/src/mongo/db/pipeline/document_source_facet_test.cpp @@ -186,9 +186,19 @@ TEST_F(DocumentSourceFacetTest, ShouldAcceptLegalSpecification) { ASSERT_TRUE(facetStage.get()); } +/* + * Override the stub interface to allow full execution in these tests. + */ +class ExecutableStubMongoProcessInterface : public StubMongoProcessInterface { + bool isExpectedToExecuteQueries() override { + return true; + } +}; + TEST_F(DocumentSourceFacetTest, ShouldRejectConflictingHostTypeRequirementsWithinSinglePipeline) { auto ctx = getExpCtx(); ctx->inMongos = true; + ctx->mongoProcessInterface = std::make_unique<ExecutableStubMongoProcessInterface>(); auto spec = fromjson( "{$facet: {badPipe: [{$_internalSplitPipeline: {mergeType: 'anyShard'}}, " @@ -202,6 +212,7 @@ TEST_F(DocumentSourceFacetTest, ShouldRejectConflictingHostTypeRequirementsWithi TEST_F(DocumentSourceFacetTest, ShouldRejectConflictingHostTypeRequirementsAcrossPipelines) { auto ctx = getExpCtx(); ctx->inMongos = true; + ctx->mongoProcessInterface = std::make_unique<ExecutableStubMongoProcessInterface>(); auto spec = fromjson( "{$facet: {shardPipe: [{$_internalSplitPipeline: {mergeType: 'anyShard'}}], mongosPipe: " @@ -904,5 +915,165 @@ TEST_F(DocumentSourceFacetTest, ShouldSurfaceStrictestRequirementsOfEachConstrai ASSERT_FALSE( facetStage->constraints(Pipeline::SplitState::kUnsplit).isAllowedInLookupPipeline()); } + +TEST_F(DocumentSourceFacetTest, RedactsCorrectly) { + auto spec = fromjson(R"({ + $facet: { + a: [ + { $unwind: "$foo" }, + { $sortByCount: "$foo" } + ], + b: [ + { + $match: { + bar: { $exists: 1 } + } + }, + { + $bucket: { + groupBy: "$bar.foo", + boundaries: [0, 50, 100, 200], + output: { + z: { $sum : 1 } + } + } + } + ], + c: [ + { + $bucketAuto: { + groupBy: "$bar.baz", + buckets: 4 + } + } + ] + } + })"); + auto docSource = DocumentSourceFacet::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$facet": { + "HASH<a>": [ + { + "$unwind": { + "path": "$HASH<foo>" + } + }, + { + "$group": { + "_id": "$HASH<foo>", + "HASH<count>": { + "$sum": "?number" + } + } + }, + { + "$sort": { + "HASH<count>": -1 + } + } + ], + "HASH<b>": [ + { + "$match": { + "HASH<bar>": { + "$exists": "?bool" + } + } + }, + { + "$group": { + "_id": { + "$switch": { + "branches": [ + { + "case": { + "$and": [ + { + "$gte": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + }, + { + "$lt": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + } + ] + }, + "then": "?number" + }, + { + "case": { + "$and": [ + { + "$gte": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + }, + { + "$lt": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + } + ] + }, + "then": "?number" + }, + { + "case": { + "$and": [ + { + "$gte": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + }, + { + "$lt": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + } + ] + }, + "then": "?number" + } + ] + } + }, + "HASH<z>": { + "$sum": "?number" + } + } + }, + { + "$sort": { + "HASH<_id>": 1 + } + } + ], + "HASH<c>": [ + { + "$bucketAuto": { + "groupBy": "$HASH<bar>.HASH<baz>", + "buckets": "?number", + "output": { + "HASH<count>": { + "$sum": "?number" + } + } + } + } + ] + } + })", + redact(*docSource)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp b/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp index 13229ba6d3e..d98aed38f9b 100644 --- a/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp @@ -177,12 +177,12 @@ StageConstraints DocumentSourceFindAndModifyImageLookup::constraints( ChangeStreamRequirement::kDenylist); } -Value DocumentSourceFindAndModifyImageLookup::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceFindAndModifyImageLookup::serialize(const SerializationOptions& opts) const { return Value( Document{{kStageName, Value(Document{{kIncludeCommitTransactionTimestampFieldName, - _includeCommitTransactionTimestamp ? Value(true) : Value()}})}}); + _includeCommitTransactionTimestamp ? opts.serializeLiteral(true) + : Value()}})}}); } DepsTracker::State DocumentSourceFindAndModifyImageLookup::getDependencies( diff --git a/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.h b/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.h index 9fac2c429c4..9e554afcc2e 100644 --- a/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.h +++ b/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.h @@ -60,7 +60,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/pipeline/document_source_geo_near.cpp b/src/mongo/db/pipeline/document_source_geo_near.cpp index 1ee840df794..dc4aa5f8be6 100644 --- a/src/mongo/db/pipeline/document_source_geo_near.cpp +++ b/src/mongo/db/pipeline/document_source_geo_near.cpp @@ -57,40 +57,45 @@ REGISTER_DOCUMENT_SOURCE(geoNear, DocumentSourceGeoNear::createFromBson, AllowedWithApiStrict::kAlways); -Value DocumentSourceGeoNear::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceGeoNear::serialize(const SerializationOptions& opts) const { MutableDocument result; if (keyFieldPath) { - result.setField(kKeyFieldName, Value(keyFieldPath->fullPath())); + result.setField(kKeyFieldName, Value(opts.serializeFieldPath(*keyFieldPath))); } auto nearValue = [&]() -> Value { if (auto constGeometry = dynamic_cast<ExpressionConstant*>(_nearGeometry.get()); constGeometry) { - return constGeometry->getValue(); + return opts.serializeLiteral(constGeometry->getValue()); } else { - return _nearGeometry->serialize(static_cast<bool>(explain)); + return _nearGeometry->serialize(opts); } }(); result.setField("near", nearValue); - result.setField("distanceField", Value(distanceField->fullPath())); + result.setField("distanceField", Value(opts.serializeFieldPath(*distanceField))); if (maxDistance) { - result.setField("maxDistance", Value(*maxDistance)); + result.setField("maxDistance", opts.serializeLiteral(*maxDistance)); } if (minDistance) { - result.setField("minDistance", Value(*minDistance)); + result.setField("minDistance", opts.serializeLiteral(*minDistance)); } - result.setField("query", Value(query)); - result.setField("spherical", Value(spherical)); + if (opts.transformIdentifiers || opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + auto matchExpr = uassertStatusOK(MatchExpressionParser::parse(query, pExpCtx)); + result.setField("query", Value(matchExpr->serialize(opts))); + } else { + result.setField("query", Value(query)); + } + result.setField("spherical", opts.serializeLiteral(spherical)); if (distanceMultiplier) { - result.setField("distanceMultiplier", Value(*distanceMultiplier)); + result.setField("distanceMultiplier", opts.serializeLiteral(*distanceMultiplier)); } if (includeLocs) - result.setField("includeLocs", Value(includeLocs->fullPath())); + result.setField("includeLocs", Value(opts.serializeFieldPath(*includeLocs))); return Value(DOC(getSourceName() << result.freeze())); } diff --git a/src/mongo/db/pipeline/document_source_geo_near.h b/src/mongo/db/pipeline/document_source_geo_near.h index 2b012285560..e9fd04e5f2d 100644 --- a/src/mongo/db/pipeline/document_source_geo_near.h +++ b/src/mongo/db/pipeline/document_source_geo_near.h @@ -79,7 +79,7 @@ public: MONGO_UNREACHABLE; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<DocumentSource> optimize() final; diff --git a/src/mongo/db/pipeline/document_source_geo_near_test.cpp b/src/mongo/db/pipeline/document_source_geo_near_test.cpp index cc3a510b84e..3ef5920c99e 100644 --- a/src/mongo/db/pipeline/document_source_geo_near_test.cpp +++ b/src/mongo/db/pipeline/document_source_geo_near_test.cpp @@ -111,5 +111,98 @@ TEST_F(DocumentSourceGeoNearTest, CanParseAndSerializeKeyField) { {"spherical", false}}}}}}; ASSERT_VALUE_EQ(expectedSerialization, serialized[0]); } + +TEST_F(DocumentSourceGeoNearTest, RedactionWithGeoJSONPoint) { + auto spec = fromjson(R"({ + $geoNear: { + distanceField: "a", + maxDistance: 2, + minDistance: 1, + near: { + type: "Point", + coordinates: [ -23.484, 28.3913 ] + }, + query: { foo : "bar" }, + spherical: true + } + })"); + auto docSource = DocumentSourceGeoNear::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$geoNear": { + "near": "?object", + "distanceField": "HASH<a>", + "maxDistance": "?number", + "minDistance": "?number", + "query": { + "HASH<foo>": { + "$eq": "?string" + } + }, + "spherical": "?bool" + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGeoNearTest, RedactionWithGeoJSONLineString) { + auto spec = fromjson(R"({ + $geoNear: { + distanceField: "a", + near: { + type: "LineString", + coordinates: [[0,0], [-1,-1]] + }, + minDistance: 0.5 + } + })"); + auto docSource = DocumentSourceGeoNear::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$geoNear": { + "near": "?object", + "distanceField": "HASH<a>", + "minDistance": "?number", + "query": {}, + "spherical": "?bool" + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGeoNearTest, RedactionWithLegacyCoordinates) { + auto spec = fromjson(R"({ + $geoNear: { + distanceField: "foo", + distanceMultiplier: 3.14, + includeLocs: "bar.baz", + near: [10, 10], + key: "z", + query: { + a : { $gt: 10 } + }, + spherical: false + } + })"); + auto docSource = DocumentSourceGeoNear::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$geoNear": { + "key": "HASH<z>", + "near": "?array<?number>", + "distanceField": "HASH<foo>", + "query": { + "HASH<a>": { + "$gt": "?number" + } + }, + "spherical": "?bool", + "distanceMultiplier": "?number", + "includeLocs": "HASH<bar>.HASH<baz>" + } + })", + redact(*docSource)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_graph_lookup.cpp b/src/mongo/db/pipeline/document_source_graph_lookup.cpp index 044875f737f..62a35123173 100644 --- a/src/mongo/db/pipeline/document_source_graph_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_graph_lookup.cpp @@ -544,46 +544,56 @@ void DocumentSourceGraphLookUp::checkMemoryUsage() { _cache.evictDownTo(_maxMemoryUsageBytes - _frontierUsageBytes - _visitedUsageBytes); } -void DocumentSourceGraphLookUp::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { +void DocumentSourceGraphLookUp::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { auto fromValue = (pExpCtx->ns.db() == _from.db()) - ? Value(_from.coll()) - : Value(Document{{"db", _from.db()}, {"coll", _from.coll()}}); + ? Value(opts.serializeIdentifier(_from.coll())) + : Value(Document{{"db", opts.serializeIdentifier(_from.db())}, + {"coll", opts.serializeIdentifier(_from.coll())}}); // Serialize default options. - MutableDocument spec(DOC("from" << fromValue << "as" << _as.fullPath() << "connectToField" - << _connectToField.fullPath() << "connectFromField" - << _connectFromField.fullPath() << "startWith" - << _startWith->serialize(false))); + MutableDocument spec(DOC("from" << fromValue << "as" << opts.serializeFieldPath(_as) + << "connectToField" << opts.serializeFieldPath(_connectToField) + << "connectFromField" + << opts.serializeFieldPath(_connectFromField) << "startWith" + << _startWith->serialize(opts))); // depthField is optional; serialize it if it was specified. if (_depthField) { - spec["depthField"] = Value(_depthField->fullPath()); + spec["depthField"] = Value(opts.serializeFieldPath(*_depthField)); } if (_maxDepth) { - spec["maxDepth"] = Value(*_maxDepth); + spec["maxDepth"] = Value(opts.serializeLiteral(*_maxDepth)); } if (_additionalFilter) { - spec["restrictSearchWithMatch"] = Value(*_additionalFilter); + if (opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + auto matchExpr = + uassertStatusOK(MatchExpressionParser::parse(*_additionalFilter, pExpCtx)); + spec["restrictSearchWithMatch"] = Value(matchExpr->serialize(opts)); + } else { + spec["restrictSearchWithMatch"] = Value(*_additionalFilter); + } } // If we are explaining, include an absorbed $unwind inside the $graphLookup specification. - if (_unwind && explain) { + if (_unwind && opts.verbosity) { const boost::optional<FieldPath> indexPath = (*_unwind)->indexPath(); spec["unwinding"] = Value(DOC("preserveNullAndEmptyArrays" - << (*_unwind)->preserveNullAndEmptyArrays() << "includeArrayIndex" - << (indexPath ? Value((*indexPath).fullPath()) : Value()))); + << opts.serializeLiteral((*_unwind)->preserveNullAndEmptyArrays()) + << "includeArrayIndex" + << (indexPath ? Value(opts.serializeFieldPath(*indexPath)) : Value()))); } array.push_back(Value(DOC(getSourceName() << spec.freeze()))); // If we are not explaining, the output of this method must be parseable, so serialize our // $unwind into a separate stage. - if (_unwind && !explain) { - (*_unwind)->serializeToArray(array); + if (_unwind && !opts.verbosity) { + (*_unwind)->serializeToArray(array, opts); } } diff --git a/src/mongo/db/pipeline/document_source_graph_lookup.h b/src/mongo/db/pipeline/document_source_graph_lookup.h index a2b6ef6a952..52134b61efe 100644 --- a/src/mongo/db/pipeline/document_source_graph_lookup.h +++ b/src/mongo/db/pipeline/document_source_graph_lookup.h @@ -97,7 +97,7 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Returns the 'as' path, and possibly the fields modified by an absorbed $unwind. @@ -188,9 +188,10 @@ private: boost::optional<long long> maxDepth, boost::optional<boost::intrusive_ptr<DocumentSourceUnwind>> unwindSrc); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { // Should not be called; use serializeToArray instead. - MONGO_UNREACHABLE; + MONGO_UNREACHABLE_TASSERT(7484306); } /** diff --git a/src/mongo/db/pipeline/document_source_graph_lookup_test.cpp b/src/mongo/db/pipeline/document_source_graph_lookup_test.cpp index 26345aa16ab..c5dd404a04e 100644 --- a/src/mongo/db/pipeline/document_source_graph_lookup_test.cpp +++ b/src/mongo/db/pipeline/document_source_graph_lookup_test.cpp @@ -713,5 +713,126 @@ TEST_F(DocumentSourceGraphLookUpTest, ShouldNotExpandArraysWithinArraysAtEndOfCo ASSERT(graphLookupStage->getNext().isEOF()); } +TEST_F(DocumentSourceGraphLookUpTest, RedactionStartWithSingleField) { + NamespaceString graphLookupNs(getExpCtx()->ns.db(), "coll"); + getExpCtx()->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {graphLookupNs.coll().toString(), {graphLookupNs, std::vector<BSONObj>()}}}); + + auto spec = fromjson(R"({ + "$graphLookup": { + "from": "coll", + "startWith": "$a.b", + "connectFromField": "c.d", + "connectToField": "e.f", + "as": "x", + "depthField": "y", + "maxDepth": 5, + "restrictSearchWithMatch": { + "foo": "abc", + "bar.baz": { "$gt": 5 } + } + } + })"); + auto docSource = DocumentSourceGraphLookUp::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$graphLookup": { + "from": "HASH<coll>", + "as": "HASH<x>", + "connectToField": "HASH<e>.HASH<f>", + "connectFromField": "HASH<c>.HASH<d>", + "startWith": "$HASH<a>.HASH<b>", + "depthField": "HASH<y>", + "maxDepth": "?number", + "restrictSearchWithMatch": { + "$and": [ + { + "HASH<foo>": { + "$eq": "?string" + } + }, + { + "HASH<bar>.HASH<baz>": { + "$gt": "?number" + } + } + ] + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGraphLookUpTest, RedactionStartWithArrayOfFields) { + NamespaceString graphLookupNs(getExpCtx()->ns.db(), "coll"); + getExpCtx()->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {graphLookupNs.coll().toString(), {graphLookupNs, std::vector<BSONObj>()}}}); + + auto spec = fromjson(R"({ + $graphLookup: { + from: "coll", + startWith: ["$a.b", "$bar.baz"], + connectFromField: "x", + connectToField: "y", + as: "z" + } + })"); + auto docSource = DocumentSourceGraphLookUp::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$graphLookup": { + "from": "HASH<coll>", + "as": "HASH<z>", + "connectToField": "HASH<y>", + "connectFromField": "HASH<x>", + "startWith": ["$HASH<a>.HASH<b>", "$HASH<bar>.HASH<baz>"] + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGraphLookUpTest, RedactionWithAbsorbedUnwind) { + auto expCtx = getExpCtx(); + + NamespaceString graphLookupNs(getExpCtx()->ns.db(), "coll"); + expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {graphLookupNs.coll().toString(), {graphLookupNs, std::vector<BSONObj>()}}}); + + auto unwindStage = DocumentSourceUnwind::create(expCtx, "results", false, boost::none); + auto graphLookupStage = DocumentSourceGraphLookUp::create( + getExpCtx(), + graphLookupNs, + "results", + "from", + "to", + ExpressionFieldPath::deprecatedCreate(expCtx.get(), "startPoint"), + boost::none, + boost::none, + boost::none, + unwindStage); + + auto serialized = redactToArray(*graphLookupStage); + ASSERT_EQ(2, serialized.size()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$graphLookup": { + "from": "HASH<coll>", + "as": "HASH<results>", + "connectToField": "HASH<to>", + "connectFromField": "HASH<from>", + "startWith": "$HASH<startPoint>" + } + })", + serialized[0].getDocument().toBson()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$unwind": { + path: "$HASH<results>" + } + })", + serialized[1].getDocument().toBson()); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_group_base.cpp b/src/mongo/db/pipeline/document_source_group_base.cpp index 0268cd57127..2c4e8c2dd3b 100644 --- a/src/mongo/db/pipeline/document_source_group_base.cpp +++ b/src/mongo/db/pipeline/document_source_group_base.cpp @@ -69,19 +69,20 @@ using std::pair; using std::shared_ptr; using std::vector; -Value DocumentSourceGroupBase::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceGroupBase::serialize(const SerializationOptions& opts) const { MutableDocument insides; // Add the _id. if (_idFieldNames.empty()) { invariant(_idExpressions.size() == 1); - insides["_id"] = _idExpressions[0]->serialize(static_cast<bool>(explain)); + insides["_id"] = _idExpressions[0]->serialize(opts); } else { // Decomposed document case. invariant(_idExpressions.size() == _idFieldNames.size()); MutableDocument md; for (size_t i = 0; i < _idExpressions.size(); i++) { - md[_idFieldNames[i]] = _idExpressions[i]->serialize(static_cast<bool>(explain)); + md[opts.serializeFieldPathFromString(_idFieldNames[i])] = + _idExpressions[i]->serialize(opts); } insides["_id"] = md.freezeToValue(); } @@ -89,34 +90,34 @@ Value DocumentSourceGroupBase::serialize(boost::optional<ExplainOptions::Verbosi // Add the remaining fields. for (auto&& accumulatedField : _accumulatedFields) { intrusive_ptr<AccumulatorState> accum = accumulatedField.makeAccumulator(); - insides[accumulatedField.fieldName] = - Value(accum->serialize(accumulatedField.expr.initializer, - accumulatedField.expr.argument, - static_cast<bool>(explain))); + insides[opts.serializeFieldPathFromString(accumulatedField.fieldName)] = + Value(accum->serialize( + accumulatedField.expr.initializer, accumulatedField.expr.argument, opts)); } if (_doingMerge) { - insides["$doingMerge"] = Value(true); + insides["$doingMerge"] = opts.serializeLiteral(true); } - serializeAdditionalFields(insides, explain); + serializeAdditionalFields(insides, opts); MutableDocument out; out[getSourceName()] = insides.freezeToValue(); - if (explain && *explain >= ExplainOptions::Verbosity::kExecStats) { + if (opts.verbosity && *opts.verbosity >= ExplainOptions::Verbosity::kExecStats) { MutableDocument md; for (size_t i = 0; i < _accumulatedFields.size(); i++) { - md[_accumulatedFields[i].fieldName] = Value(static_cast<long long>( - _memoryTracker[_accumulatedFields[i].fieldName].maxMemoryBytes())); + md[opts.serializeFieldPathFromString(_accumulatedFields[i].fieldName)] = + opts.serializeLiteral(static_cast<long long>( + _memoryTracker[_accumulatedFields[i].fieldName].maxMemoryBytes())); } out["maxAccumulatorMemoryUsageBytes"] = Value(md.freezeToValue()); out["totalOutputDataSizeBytes"] = - Value(static_cast<long long>(_stats.totalOutputDataSizeBytes)); - out["usedDisk"] = Value(_stats.spills > 0); - out["spills"] = Value(static_cast<long long>(_stats.spills)); + opts.serializeLiteral(static_cast<long long>(_stats.totalOutputDataSizeBytes)); + out["usedDisk"] = opts.serializeLiteral(_stats.spills > 0); + out["spills"] = opts.serializeLiteral(static_cast<long long>(_stats.spills)); } return out.freezeToValue(); diff --git a/src/mongo/db/pipeline/document_source_group_base.h b/src/mongo/db/pipeline/document_source_group_base.h index 3c20f1bd420..c7a296c2583 100644 --- a/src/mongo/db/pipeline/document_source_group_base.h +++ b/src/mongo/db/pipeline/document_source_group_base.h @@ -54,7 +54,7 @@ public: using Accumulators = std::vector<boost::intrusive_ptr<AccumulatorState>>; using GroupsMap = ValueUnorderedMap<Accumulators>; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<DocumentSource> optimize() final; DepsTracker::State getDependencies(DepsTracker* deps) const final; GetModPathsReturn getModifiedPaths() const final; @@ -200,7 +200,7 @@ protected: } virtual void serializeAdditionalFields( - MutableDocument& out, boost::optional<ExplainOptions::Verbosity> explain) const {}; + MutableDocument& out, const SerializationOptions& opts = SerializationOptions{}) const {}; // If the expression for the '_id' field represents a non-empty object, we track its fields' // names in '_idFieldNames'. diff --git a/src/mongo/db/pipeline/document_source_group_test.cpp b/src/mongo/db/pipeline/document_source_group_test.cpp index 9a621cffdcd..f3bb0863adf 100644 --- a/src/mongo/db/pipeline/document_source_group_test.cpp +++ b/src/mongo/db/pipeline/document_source_group_test.cpp @@ -246,6 +246,103 @@ TEST_F(DocumentSourceGroupTest, ShouldNotReportDottedGroupKeyAsARename) { ASSERT_EQ(modifiedPathsRet.renames.size(), 0UL); } +TEST_F(DocumentSourceGroupTest, GroupRedactsCorrectWithIdNull) { + auto spec = fromjson(R"({ + $group: { + _id: null, + foo: { $count: {} } + } + })"); + auto docSource = DocumentSourceGroup::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$group":{"_id":"?null","HASH<foo>":{"$sum":"?number"}}})", + redact(*docSource)); +} + +TEST_F(DocumentSourceGroupTest, GroupRedactsCorrectWithIdSingleField) { + auto spec = fromjson(R"({ + $group: { + _id: '$foo' + } + })"); + auto docSource = DocumentSourceGroup::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$group": { + "_id": "$HASH<foo>" + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGroupTest, GroupRedactsCorrectWithIdDocument) { + auto spec = fromjson(R"({ + $group: { + _id: { + x: '$x', + y: '$z' + }, + foo: { + $sum: { + $multiply: ['$a.b', '$c', '$d'] + } + }, + bar: { + $first: '$baz' + } + } + })"); + auto docSource = DocumentSourceGroup::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$group": { + "_id": { + "HASH<x>": "$HASH<x>", + "HASH<y>": "$HASH<z>" + }, + "HASH<foo>": { + "$sum": { + "$multiply": ["$HASH<a>.HASH<b>", "$HASH<c>", "$HASH<d>"] + } + }, + "HASH<bar>": { + "$first": "$HASH<baz>" + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGroupTest, StreamingGroupRedactsCorrectly) { + auto spec = fromjson(R"({ + $_internalStreamingGroup: { + _id: { + a: "$a", + b: "$b" + }, + a: { + $first: '$b' + }, + $monotonicIdFields: [ "a", "b" ] + } + })"); + auto docSource = DocumentSourceStreamingGroup::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalStreamingGroup": { + "_id": { + "HASH<a>": "$HASH<a>", + "HASH<b>": "$HASH<b>" + }, + "HASH<a>": { + "$first": "$HASH<b>" + }, + "$monotonicIdFields": [ "HASH<a>", "HASH<b>" ] + } + })", + redact(*docSource)); +} + BSONObj toBson(const intrusive_ptr<DocumentSource>& source) { vector<Value> arr; source->serializeToArray(arr); diff --git a/src/mongo/db/pipeline/document_source_index_stats.cpp b/src/mongo/db/pipeline/document_source_index_stats.cpp index 139fdde3bca..bd4ee6bf760 100644 --- a/src/mongo/db/pipeline/document_source_index_stats.cpp +++ b/src/mongo/db/pipeline/document_source_index_stats.cpp @@ -78,8 +78,7 @@ intrusive_ptr<DocumentSource> DocumentSourceIndexStats::createFromBson( return new DocumentSourceIndexStats(pExpCtx); } -Value DocumentSourceIndexStats::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceIndexStats::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_index_stats.h b/src/mongo/db/pipeline/document_source_index_stats.h index bd5b7577365..f56e5e22059 100644 --- a/src/mongo/db/pipeline/document_source_index_stats.h +++ b/src/mongo/db/pipeline/document_source_index_stats.h @@ -75,7 +75,7 @@ public: // virtuals from DocumentSource const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { StageConstraints constraints(StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.cpp b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.cpp index 6c9693e0443..e24bf112bcf 100644 --- a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.cpp +++ b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.cpp @@ -118,17 +118,18 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalAllCollectionStats::do } void DocumentSourceInternalAllCollectionStats::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { + std::vector<Value>& array, const SerializationOptions& opts) const { + auto explain = opts.verbosity; if (explain) { BSONObjBuilder bob; - _internalAllCollectionStatsSpec.serialize(&bob); + _internalAllCollectionStatsSpec.serialize(&bob, opts); if (_absorbedMatch) { bob.append("match", _absorbedMatch->getQuery()); } auto doc = Document{{getSourceName(), bob.obj()}}; array.push_back(Value(doc)); } else { - array.push_back(serialize(explain)); + array.push_back(serialize(opts)); if (_absorbedMatch) { _absorbedMatch->serializeToArray(array); } @@ -156,8 +157,7 @@ const char* DocumentSourceInternalAllCollectionStats::getSourceName() const { return kStageNameInternal.rawData(); } -Value DocumentSourceInternalAllCollectionStats::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{getSourceName(), _internalAllCollectionStatsSpec.toBSON()}}); +Value DocumentSourceInternalAllCollectionStats::serialize(const SerializationOptions& opts) const { + return Value(Document{{getSourceName(), _internalAllCollectionStatsSpec.toBSON(opts)}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.h b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.h index 38324fb51ad..1648939ddd1 100644 --- a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.h +++ b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.h @@ -81,7 +81,7 @@ public: const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { StageConstraints constraints(StreamType::kStreaming, @@ -110,7 +110,7 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; private: GetNextResult doGetNext() final; diff --git a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.idl b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.idl index 61fc5d02f03..c473a5e5ba3 100644 --- a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.idl +++ b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.idl @@ -40,8 +40,10 @@ structs: DocumentSourceInternalAllCollectionStatsSpec: description: Specification for an $_internalAllCollectionStats stage. strict: true + query_shape_component: true fields: stats: description: Specification for a $collStats stage. type: DocumentSourceCollStatsSpec optional: true + query_shape: literal diff --git a/src/mongo/db/pipeline/document_source_internal_all_collection_stats_test.cpp b/src/mongo/db/pipeline/document_source_internal_all_collection_stats_test.cpp new file mode 100644 index 00000000000..e303febfacb --- /dev/null +++ b/src/mongo/db/pipeline/document_source_internal_all_collection_stats_test.cpp @@ -0,0 +1,143 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/document_source_internal_all_collection_stats.h" +#include "mongo/db/pipeline/document_source_internal_all_collection_stats_gen.h" +#include <boost/cstdint.hpp> +#include <cstdint> + +#include <boost/move/utility_core.hpp> +#include <boost/smart_ptr/intrusive_ptr.hpp> + +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_coll_stats.h" +#include "mongo/db/pipeline/document_source_coll_stats_gen.h" +#include "mongo/db/pipeline/storage_stats_spec_gen.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/intrusive_counter.h" + +namespace mongo { +namespace { +using DocumentSourceInternalAllCollectionStatsTest = AggregationContextFixture; + +auto representativeShape(const DocumentSourceInternalAllCollectionStats& allCollStatsStage) { + SerializationOptions opts{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + return allCollStatsStage.serialize(opts).getDocument().toBson(); +} + +TEST_F(DocumentSourceInternalAllCollectionStatsTest, QueryShape) { + auto innerSpec = DocumentSourceCollStatsSpec(); + auto spec = DocumentSourceInternalAllCollectionStatsSpec(); + spec.setStats(innerSpec); + + auto stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalAllCollectionStats":{"stats":{}}})", + redact(*stage)); + + innerSpec.setCount(BSONObj()); + innerSpec.setQueryExecStats(BSONObj()); + spec.setStats(innerSpec); + stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalAllCollectionStats":{"stats":{"count":{},"queryExecStats":{}}}})", + redact(*stage)); + + auto latencyStats = LatencyStatsSpec(); + latencyStats.setHistograms(true); + innerSpec.setLatencyStats(latencyStats); + spec.setStats(innerSpec); + stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalAllCollectionStats": { + "stats": { + "latencyStats": { + "histograms": true + }, + "count": {}, + "queryExecStats": {} + } + } + })", + redact(*stage)); + + auto storageStats = StorageStatsSpec(); + storageStats.setScale(2); + storageStats.setVerbose(true); + innerSpec.setStorageStats(storageStats); + spec.setStats(innerSpec); + stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalAllCollectionStats": { + "stats": { + "latencyStats": { + "histograms": true + }, + "storageStats": { + "scale": "?number", + "verbose": true, + "waitForLock": true, + "numericOnly": false + }, + "count": {}, + "queryExecStats": {} + } + } + })", + redact(*stage)); + + storageStats.setWaitForLock(false); + storageStats.setNumericOnly(false); + innerSpec.setStorageStats(storageStats); + spec.setStats(innerSpec); + stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalAllCollectionStats": { + "stats": { + "latencyStats": { + "histograms": true + }, + "storageStats": { + "scale": "?number", + "verbose": true, + "waitForLock": false, + "numericOnly": false + }, + "count": {}, + "queryExecStats": {} + } + } + })", + redact(*stage)); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.cpp b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.cpp index 51b0dee8ac1..2d499e0d88c 100644 --- a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.cpp +++ b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.cpp @@ -95,9 +95,9 @@ DocumentSource::GetNextResult DocumentSourceInternalApplyOplogUpdate::doGetNext( return Document(doc.getObject()); } -Value DocumentSourceInternalApplyOplogUpdate::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{kStageName, Document{{kOplogUpdateFieldName, _oplogUpdate}}}}); +Value DocumentSourceInternalApplyOplogUpdate::serialize(const SerializationOptions& opts) const { + return Value(Document{ + {kStageName, Document{{kOplogUpdateFieldName, opts.serializeLiteral(_oplogUpdate)}}}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.h b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.h index ada146b79ea..89860c90efd 100644 --- a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.h +++ b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.h @@ -81,8 +81,7 @@ public: } private: - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; GetNextResult doGetNext() override; diff --git a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update_test.cpp b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update_test.cpp index fc95d45e6fa..47123e55af2 100644 --- a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update_test.cpp @@ -217,6 +217,7 @@ TEST_F(DocumentSourceInternalApplyOplogUpdateTest, ShouldErrorOnInvalidDiffs) { ASSERT_THROWS_CODE(stage->getNext(), DBException, 4770507); } +#if 0 // TODO(SERVER-95699): Disabling test which creates fieldnames with embedded NULs { auto spec = BSON("$_internalApplyOplogUpdate" << BSON("oplogUpdate" << BSON("$v" << 2 << "diff" @@ -238,6 +239,27 @@ TEST_F(DocumentSourceInternalApplyOplogUpdateTest, ShouldErrorOnInvalidDiffs) { stage->setSource(mock.get()); ASSERT_THROWS_CODE(stage->getNext(), DBException, 4770505); } +#endif +} + +TEST_F(DocumentSourceInternalApplyOplogUpdateTest, RedactsCorrectly) { + auto spec = fromjson(R"({ + $_internalApplyOplogUpdate: { + oplogUpdate: { + $v: 2, + diff: { sa: [] } + } + } + })"); + auto docSource = + DocumentSourceInternalApplyOplogUpdate::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalApplyOplogUpdate": { + "oplogUpdate":"?object" + } + })", + redact(*docSource)); } } // namespace diff --git a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.cpp b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.cpp index f643bcb32f3..b44d2bb52cd 100644 --- a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.cpp +++ b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.cpp @@ -139,15 +139,16 @@ DocumentSource::GetNextResult DocumentSourceInternalGeoNearDistance::doGetNext() return next; } -Value DocumentSourceInternalGeoNearDistance::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalGeoNearDistance::serialize(const SerializationOptions& opts) const { MutableDocument out; - out.setField(DocumentSourceInternalGeoNearDistance::kNearFieldName, Value(_coords)); - out.setField(DocumentSourceInternalGeoNearDistance::kKeyFieldName, Value(_key)); + out.setField(DocumentSourceInternalGeoNearDistance::kNearFieldName, + opts.serializeLiteral(_coords)); + out.setField(DocumentSourceInternalGeoNearDistance::kKeyFieldName, + Value(opts.serializeFieldPathFromString(_key))); out.setField(DocumentSourceInternalGeoNearDistance::kDistanceFieldFieldName, - Value(_distanceField.fullPath())); + Value(opts.serializeFieldPath(_distanceField))); out.setField(DocumentSourceInternalGeoNearDistance::kDistanceMultiplierFieldName, - Value(_distanceMultiplier)); + opts.serializeLiteral(_distanceMultiplier)); return Value(DOC(getSourceName() << out.freeze())); } diff --git a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h index b1a24787224..5dc81ea7401 100644 --- a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h +++ b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h @@ -84,8 +84,7 @@ public: } private: - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; GetNextResult doGetNext() override; diff --git a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance_test.cpp b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance_test.cpp index 687fe4f9e7a..67f7300fd2b 100644 --- a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance_test.cpp @@ -32,6 +32,7 @@ #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/exec/document_value/document_value_test_util.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" #include "mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h" #include "mongo/db/pipeline/document_source_mock.h" @@ -145,5 +146,31 @@ TEST_F(DocumentSourceInternalGeoNearDistanceTest, DistanceBetweenTwoMixedPointsS ASSERT_APPROX_EQUAL(doc["dist"].coerceToDouble(), meterToLatDegree, 300); } +TEST_F(DocumentSourceInternalGeoNearDistanceTest, RedactsCorrectly) { + BSONObj computeGeoSpec = fromjson(R"( + { $_internalComputeGeoNearDistance: { + near: { + type: "Point", + coordinates: [0, 1] + }, + key: "loc", + distanceMultiplier: 1, + distanceField: "dist" + }})"); + auto geoDist = DocumentSourceInternalGeoNearDistance::createFromBson( + computeGeoSpec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalComputeGeoNearDistance": { + "near": "?object", + "key": "HASH<loc>", + "distanceField": "HASH<dist>", + "distanceMultiplier": "?number" + } + })", + redact(*geoDist, true)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp index be9784f5346..9d52a0f7124 100644 --- a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp +++ b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp @@ -33,6 +33,7 @@ #include "mongo/bson/bsonobj.h" #include "mongo/db/exec/document_value/document.h" +#include "mongo/db/index/index_descriptor.h" #include "mongo/db/list_indexes_gen.h" #include "mongo/db/pipeline/expression_context.h" #include "mongo/db/pipeline/lite_parsed_document_source.h" @@ -146,11 +147,13 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceInternalConvertBucketIndexSta } Value DocumentSourceInternalConvertBucketIndexStats::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { + const SerializationOptions& opts) const { MutableDocument out; - out.addField(timeseries::kTimeFieldName, Value{_timeseriesOptions.timeField}); + out.addField(timeseries::kTimeFieldName, + Value{opts.serializeFieldPathFromString(_timeseriesOptions.timeField)}); if (_timeseriesOptions.metaField) { - out.addField(timeseries::kMetaFieldName, Value{*_timeseriesOptions.metaField}); + out.addField(timeseries::kMetaFieldName, + Value{opts.serializeFieldPathFromString(*_timeseriesOptions.metaField)}); } return Value(DOC(getSourceName() << out.freeze())); } diff --git a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h index 498518ee951..5e7b4c09959 100644 --- a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h +++ b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h @@ -77,7 +77,7 @@ public: ChangeStreamRequirement::kDenylist}; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::optional<DistributedPlanLogic> distributedPlanLogic() final { return boost::none; diff --git a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats_test.cpp b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats_test.cpp new file mode 100644 index 00000000000..d0b1bddbe0b --- /dev/null +++ b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats_test.cpp @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/document_value/document_value_test_util.h" +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/unittest/bson_test_util.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { +using InternalConvertBucketIndexStatsTest = AggregationContextFixture; + +TEST_F(InternalConvertBucketIndexStatsTest, QueryShapeAndRedaction) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + auto stage = std::make_unique<DocumentSourceInternalConvertBucketIndexStats>( + expCtx, TimeseriesConversionOptions{"timefield"}); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalConvertBucketIndexStats":{"timeField":"HASH<timefield>"}})", + redact(*stage)); + + std::string metaField = "metafield"; + stage = std::make_unique<DocumentSourceInternalConvertBucketIndexStats>( + expCtx, TimeseriesConversionOptions{"timefield", metaField}); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalConvertBucketIndexStats": { + "timeField": "HASH<timefield>", + "metaField": "HASH<metafield>" + } + })", + redact(*stage)); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.cpp b/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.cpp index af9d4c297b1..6cd368fde1a 100644 --- a/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.cpp +++ b/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.cpp @@ -60,8 +60,7 @@ DocumentSource::GetNextResult DocumentSourceInternalInhibitOptimization::doGetNe return pSource->getNext(); } -Value DocumentSourceInternalInhibitOptimization::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalInhibitOptimization::serialize(const SerializationOptions& opts) const { return Value(Document{{getSourceName(), Value{Document{}}}}); } diff --git a/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.h b/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.h index 79e39f865d9..80d74219af0 100644 --- a/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.h +++ b/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.h @@ -71,7 +71,7 @@ public: private: GetNextResult doGetNext() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_shard_filter.cpp b/src/mongo/db/pipeline/document_source_internal_shard_filter.cpp index a8b112ae834..f63125ff9b6 100644 --- a/src/mongo/db/pipeline/document_source_internal_shard_filter.cpp +++ b/src/mongo/db/pipeline/document_source_internal_shard_filter.cpp @@ -95,8 +95,7 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalShardFilter::doOptimiz return ret; } -Value DocumentSourceInternalShardFilter::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalShardFilter::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } diff --git a/src/mongo/db/pipeline/document_source_internal_shard_filter.h b/src/mongo/db/pipeline/document_source_internal_shard_filter.h index e223af734a9..a8b88869321 100644 --- a/src/mongo/db/pipeline/document_source_internal_shard_filter.h +++ b/src/mongo/db/pipeline/document_source_internal_shard_filter.h @@ -64,8 +64,8 @@ public: ChangeStreamRequirement::kDenylist); } - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::optional<DistributedPlanLogic> distributedPlanLogic() override { return boost::none; diff --git a/src/mongo/db/pipeline/document_source_internal_shardserver_info.cpp b/src/mongo/db/pipeline/document_source_internal_shardserver_info.cpp new file mode 100644 index 00000000000..6e3fe14c578 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_internal_shardserver_info.cpp @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/pipeline/document_source_internal_shardserver_info.h" +#include "mongo/db/pipeline/pipeline.h" + +namespace mongo { + +REGISTER_DOCUMENT_SOURCE(_internalShardServerInfo, + DocumentSourceInternalShardServerInfo::LiteParsed::parse, + DocumentSourceInternalShardServerInfo::createFromBson, + AllowedWithApiStrict::kNeverInVersion1); + +boost::intrusive_ptr<DocumentSource> DocumentSourceInternalShardServerInfo::createFromBson( + BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx) { + uassert(ErrorCodes::TypeMismatch, + str::stream() << "$_internalShardServerInfo must take an empty object but found: " + << elem, + elem.type() == BSONType::Object && elem.Obj().isEmpty()); + + return new DocumentSourceInternalShardServerInfo(expCtx); +} + +DocumentSource::GetNextResult DocumentSourceInternalShardServerInfo::doGetNext() { + if (!_didEmit) { + auto shardName = pExpCtx->mongoProcessInterface->getShardName(pExpCtx->opCtx); + auto hostAndPort = pExpCtx->mongoProcessInterface->getHostAndPort(pExpCtx->opCtx); + _didEmit = true; + return DocumentSource::GetNextResult(DOC("shard" << shardName << "host" << hostAndPort)); + } + + return DocumentSource::GetNextResult::makeEOF(); +} + +Value DocumentSourceInternalShardServerInfo::serialize(const SerializationOptions& opts) const { + return Value(Document{{getSourceName(), Value{Document{{}}}}}); +} + +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_shardserver_info.h b/src/mongo/db/pipeline/document_source_internal_shardserver_info.h new file mode 100644 index 00000000000..5a1cbaa8f63 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_internal_shardserver_info.h @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/pipeline/document_source.h" + +namespace mongo { + +/** + * An internal stage available for testing. Gets the host and shard name for every shard server in + * the cluster. + */ +class DocumentSourceInternalShardServerInfo final : public DocumentSource { +public: + class LiteParsed : public LiteParsedDocumentSource { + public: + static std::unique_ptr<LiteParsed> parse(const NamespaceString& nss, + const BSONElement& spec) { + return std::make_unique<LiteParsed>(spec.fieldName()); + } + + LiteParsed(std::string parseTimeName) + : LiteParsedDocumentSource(std::move(parseTimeName)) {} + + stdx::unordered_set<NamespaceString> getInvolvedNamespaces() const final { + return stdx::unordered_set<NamespaceString>(); + } + + PrivilegeVector requiredPrivileges(bool isMongos, + bool bypassDocumentValidation) const final { + return {}; + } + + bool allowedToPassthroughFromMongos() const final { + return false; + } + }; + + static constexpr StringData kStageName = "$_internalShardServerInfo"_sd; + + static boost::intrusive_ptr<DocumentSource> createFromBson( + BSONElement, const boost::intrusive_ptr<ExpressionContext>&); + + static boost::intrusive_ptr<DocumentSource> create( + const boost::intrusive_ptr<ExpressionContext>& expCtx) { + return new DocumentSourceInternalShardServerInfo(expCtx); + } + + const char* getSourceName() const final { + return kStageName.rawData(); + } + + boost::optional<DistributedPlanLogic> distributedPlanLogic() final { + return boost::none; + } + + StageConstraints constraints(Pipeline::SplitState pipeState) const final { + StageConstraints constraints{StreamType::kStreaming, + PositionRequirement::kFirst, + HostTypeRequirement::kAllShardServers, + DiskUseRequirement::kNoDiskUse, + FacetRequirement::kNotAllowed, + TransactionRequirement::kNotAllowed, + LookupRequirement::kNotAllowed, + UnionRequirement::kAllowed}; + constraints.isIndependentOfAnyCollection = true; + constraints.requiresInputDocSource = false; + return constraints; + } + +private: + DocumentSourceInternalShardServerInfo(const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSource(kStageName, expCtx) {} + + GetNextResult doGetNext() final; + + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + + bool _didEmit = false; +}; + +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_shred_documents.cpp b/src/mongo/db/pipeline/document_source_internal_shred_documents.cpp index 8b51bb2a1ca..3bb636c5004 100644 --- a/src/mongo/db/pipeline/document_source_internal_shred_documents.cpp +++ b/src/mongo/db/pipeline/document_source_internal_shred_documents.cpp @@ -55,8 +55,7 @@ DocumentSource::GetNextResult DocumentSourceInternalShredDocuments::doGetNext() return next; } -Value DocumentSourceInternalShredDocuments::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalShredDocuments::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } diff --git a/src/mongo/db/pipeline/document_source_internal_shred_documents.h b/src/mongo/db/pipeline/document_source_internal_shred_documents.h index 22f1585f557..ebb095d404c 100644 --- a/src/mongo/db/pipeline/document_source_internal_shred_documents.h +++ b/src/mongo/db/pipeline/document_source_internal_shred_documents.h @@ -66,8 +66,7 @@ public: return constraints; } - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::optional<DistributedPlanLogic> distributedPlanLogic() override { return boost::none; diff --git a/src/mongo/db/pipeline/document_source_internal_split_pipeline.cpp b/src/mongo/db/pipeline/document_source_internal_split_pipeline.cpp index 8f5f9927ada..73ad3e7ffd2 100644 --- a/src/mongo/db/pipeline/document_source_internal_split_pipeline.cpp +++ b/src/mongo/db/pipeline/document_source_internal_split_pipeline.cpp @@ -86,8 +86,7 @@ DocumentSource::GetNextResult DocumentSourceInternalSplitPipeline::doGetNext() { return pSource->getNext(); } -Value DocumentSourceInternalSplitPipeline::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalSplitPipeline::serialize(const SerializationOptions& opts) const { std::string mergeTypeString; switch (_mergeType) { diff --git a/src/mongo/db/pipeline/document_source_internal_split_pipeline.h b/src/mongo/db/pipeline/document_source_internal_split_pipeline.h index 967440775d9..ff752d1e22d 100644 --- a/src/mongo/db/pipeline/document_source_internal_split_pipeline.h +++ b/src/mongo/db/pipeline/document_source_internal_split_pipeline.h @@ -85,8 +85,7 @@ private: : DocumentSource(kStageName, expCtx), _mergeType(mergeType) {} GetNextResult doGetNext() final; - - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; HostTypeRequirement _mergeType = HostTypeRequirement::kNone; }; diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp index f352b734e01..d38f5992eaf 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp @@ -232,6 +232,7 @@ boost::intrusive_ptr<DocumentSourceGroup> createBucketGroupForReorder( void optimizePrefix(Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { auto prefix = Pipeline::SourceContainer(container->begin(), itr); Pipeline::optimizeContainer(&prefix); + Pipeline::optimizeEachStage(&prefix); container->erase(container->begin(), itr); container->splice(itr, prefix); } @@ -474,15 +475,17 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceInternalUnpackBucket::createF expCtx, BucketUnpacker{std::move(bucketSpec)}, 3600, assumeClean); } -void DocumentSourceInternalUnpackBucket::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { +void DocumentSourceInternalUnpackBucket::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { + auto explain = opts.verbosity; + MutableDocument out; auto behavior = _bucketUnpacker.behavior() == BucketSpec::Behavior::kInclude ? kInclude : kExclude; const auto& spec = _bucketUnpacker.bucketSpec(); std::vector<Value> fields; for (auto&& field : spec.fieldSet()) { - fields.emplace_back(field); + fields.emplace_back(opts.serializeFieldPathFromString(field)); } if (((_bucketUnpacker.includeMetaField() && _bucketUnpacker.behavior() == BucketSpec::Behavior::kInclude) || @@ -491,23 +494,26 @@ void DocumentSourceInternalUnpackBucket::serializeToArray( std::find(spec.computedMetaProjFields().cbegin(), spec.computedMetaProjFields().cend(), *spec.metaField()) == spec.computedMetaProjFields().cend()) - fields.emplace_back(*spec.metaField()); + fields.emplace_back(opts.serializeFieldPathFromString(*spec.metaField())); out.addField(behavior, Value{std::move(fields)}); - out.addField(timeseries::kTimeFieldName, Value{spec.timeField()}); + out.addField(timeseries::kTimeFieldName, + Value{opts.serializeFieldPathFromString(spec.timeField())}); if (spec.metaField()) { - out.addField(timeseries::kMetaFieldName, Value{*spec.metaField()}); + out.addField(timeseries::kMetaFieldName, + Value{opts.serializeFieldPathFromString(*spec.metaField())}); } - out.addField(kBucketMaxSpanSeconds, Value{_bucketMaxSpanSeconds}); + out.addField(kBucketMaxSpanSeconds, opts.serializeLiteral(Value{_bucketMaxSpanSeconds})); if (_assumeNoMixedSchemaData) - out.addField(kAssumeNoMixedSchemaData, Value(_assumeNoMixedSchemaData)); + out.addField(kAssumeNoMixedSchemaData, + opts.serializeLiteral(Value(_assumeNoMixedSchemaData))); if (spec.usesExtendedRange()) { // Include this flag so that 'explain' is more helpful. // But this is not so useful for communicating from one process to another, // because mongos and/or the primary shard don't know whether any other shard // has extended-range data. - out.addField(kUsesExtendedRange, Value{true}); + out.addField(kUsesExtendedRange, opts.serializeLiteral(Value{true})); } if (!spec.computedMetaProjFields().empty()) @@ -516,34 +522,40 @@ void DocumentSourceInternalUnpackBucket::serializeToArray( std::transform(spec.computedMetaProjFields().cbegin(), spec.computedMetaProjFields().cend(), std::back_inserter(compFields), - [](auto&& projString) { return Value{projString}; }); + [opts](auto&& projString) { + return Value{ + opts.serializeFieldPathFromString(projString)}; + }); return compFields; }()}); if (_bucketUnpacker.includeMinTimeAsMetadata()) { - out.addField(kIncludeMinTimeAsMetadata, Value{_bucketUnpacker.includeMinTimeAsMetadata()}); + out.addField(kIncludeMinTimeAsMetadata, + opts.serializeLiteral(Value{_bucketUnpacker.includeMinTimeAsMetadata()})); } if (_bucketUnpacker.includeMaxTimeAsMetadata()) { - out.addField(kIncludeMaxTimeAsMetadata, Value{_bucketUnpacker.includeMaxTimeAsMetadata()}); + out.addField(kIncludeMaxTimeAsMetadata, + opts.serializeLiteral(Value{_bucketUnpacker.includeMaxTimeAsMetadata()})); } if (_wholeBucketFilter) { - out.addField(kWholeBucketFilter, Value{_wholeBucketFilter->serialize()}); + out.addField(kWholeBucketFilter, Value{_wholeBucketFilter->serialize(opts)}); } if (_eventFilter) { - out.addField(kEventFilter, Value{_eventFilter->serialize()}); + out.addField(kEventFilter, Value{_eventFilter->serialize(opts)}); } if (!explain) { array.push_back(Value(DOC(getSourceName() << out.freeze()))); if (_sampleSize) { auto sampleSrc = DocumentSourceSample::create(pExpCtx, *_sampleSize); - sampleSrc->serializeToArray(array); + sampleSrc->serializeToArray(array, opts); } } else { if (_sampleSize) { - out.addField("sample", Value{static_cast<long long>(*_sampleSize)}); - out.addField("bucketMaxCount", Value{_bucketMaxCount}); + out.addField("sample", + opts.serializeLiteral(Value{static_cast<long long>(*_sampleSize)})); + out.addField("bucketMaxCount", opts.serializeLiteral(Value{_bucketMaxCount})); } array.push_back(Value(DOC(getSourceName() << out.freeze()))); } @@ -1205,6 +1217,34 @@ bool findSequentialDocumentCache(Pipeline::SourceContainer::iterator start, return start != end; } +Pipeline::SourceContainer::iterator DocumentSourceInternalUnpackBucket::optimizeAtRestOfPipeline( + Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { + if (itr == container->end()) { + return itr; + } + + invariant(*itr == this); + Pipeline::SourceContainer::iterator unpackBucket = itr; + + itr = std::next(itr); + + try { + while (itr != container->end()) { + if (itr == unpackBucket) { + itr = std::next(itr); + if (itr == container->end()) + break; + } + itr = (*itr).get()->optimizeAt(itr, container); + } + } catch (DBException& ex) { + ex.addContext("Failed to optimize pipeline"); + throw; + } + + return itr; +} + DepsTracker DocumentSourceInternalUnpackBucket::getRestPipelineDependencies( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container, @@ -1297,31 +1337,31 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalUnpackBucket::doOptimi } } - // Optimize the pipeline after this stage to merge $match stages and push them forward, and to - // take advantage of $expr rewrite optimizations. + // OptimizeAt the pipeline after this stage to merge $match stages and push them forward. if (!_optimizedEndOfPipeline) { _optimizedEndOfPipeline = true; if (std::next(itr) == container->end()) { return container->end(); } - if (auto nextStage = dynamic_cast<DocumentSourceGeoNear*>(std::next(itr)->get())) { - // If the end of the pipeline starts with a $geoNear stage, make sure it gets optimized - // in a context where it knows there are other stages before it. It will split itself - // up into separate $match and $sort stages. But it doesn't split itself up when it's - // the first stage, because it expects to use a special DocumentSouceGeoNearCursor plan. - nextStage->optimizeAt(std::next(itr), container); - } + auto cacheFound = findSequentialDocumentCache(itr, container->end()); if (cacheFound) { - // optimizeAt() is responsible for reordering stages, and optimize() is responsible for - // simplifying individual stages. $sequentialCache's optimizeAt() places the stage where - // it can cache as big a prefix of the pipeline as possible. To do so correctly, it - // needs to look at dependencies: a stage that depends on a let-variable cannot be - // cached. But optimize() can inline variables. Therefore, we want to avoid calling - // optimize() before $sequentialCache has a chance to run optimizeAt(). - return Pipeline::optimizeAtEndOfPipeline(itr, container); + // We want to call optimizeAt() on the rest of the pipeline first, and exit this + // function since any calls to optimize() will interfere with the + // sequentialDocumentCache's ability to properly place itself or abandon. + return DocumentSourceInternalUnpackBucket::optimizeAtRestOfPipeline(itr, container); } else { + if (auto nextStage = dynamic_cast<DocumentSourceGeoNear*>(std::next(itr)->get())) { + // If the end of the pipeline starts with a $geoNear stage, make sure it gets + // optimized in a context where it knows there are other stages before it. It will + // split itself up into separate $match and $sort stages. But it doesn't split + // itself up when it's the first stage, because it expects to use a special + // DocumentSouceGeoNearCursor plan. + nextStage->optimizeAt(std::next(itr), container); + } + // We want to optimize the rest of the pipeline to ensure the stages are in their + // optimal position and expressions have been optimized to allow for certain rewrites. Pipeline::optimizeEndOfPipeline(itr, container); } @@ -1418,9 +1458,8 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalUnpackBucket::doOptimi // Create a loose bucket predicate and push it before the unpacking stage. if (predicates.loosePredicate) { - BSONObjBuilder bob; - predicates.loosePredicate->serialize(&bob); - container->insert(itr, DocumentSourceMatch::create(bob.obj(), pExpCtx)); + container->insert( + itr, DocumentSourceMatch::create(predicates.loosePredicate->serialize(), pExpCtx)); // Give other stages a chance to optimize with the new $match. return std::prev(itr) == container->begin() ? std::prev(itr) diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h index 4d20a22f107..8d28e5e75cd 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h @@ -77,13 +77,14 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Use 'serializeToArray' above. */ - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { - MONGO_UNREACHABLE; + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { + MONGO_UNREACHABLE_TASSERT(7484305); } bool includeMetaField() const { @@ -270,6 +271,17 @@ private: bool haveComputedMetaField() const; + /** + * Applies optimizeAt() to all stages in the given pipeline after the stage that 'itr' points + * to, which is the bucket unpack stage. + * + * Due to the manipulation of 'itr' through the optimizations, it may be possible that + * preceeding stages will be optimized. However, optimization of the bucket unpack stage will be + * skipped. + */ + Pipeline::SourceContainer::iterator optimizeAtRestOfPipeline( + Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container); + // If buckets contained a mixed type schema along some path, we have to push down special // predicates in order to ensure correctness. bool _assumeNoMixedSchemaData = false; diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp index 39721256948..d51e8a5a90e 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp @@ -55,7 +55,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.max.a': {$_internalExprGt: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -77,7 +77,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.max.a': {$_internalExprGte: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -99,7 +99,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.min.a': {$_internalExprLt: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -121,7 +121,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.min.a': {$_internalExprLte: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -143,7 +143,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {$and:[{'control.min.a': {$_internalExprLte: 1}}," "{'control.max.a': {$_internalExprGte: 1}}]}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," @@ -190,7 +190,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, " ]}}" " ]}" "]}"); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), expected); + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), expected); ASSERT_FALSE(predicate.tightPredicate); } @@ -214,7 +214,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.max.a': {$_internalExprGt: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -241,7 +241,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.max.a': {$_internalExprGte: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -316,7 +316,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {$and:[{'control.min.a': {$_internalExprLte: 1}}," "{'control.max.a': {$_internalExprGte: 1}}]}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," @@ -339,7 +339,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$and: [ {$or: [ {'control.max.b': {$_internalExprGt: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.b\" ]}," "{$type: [ \"$control.max.b\" ]} ]}} ]}," @@ -383,7 +383,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [" " {'control.max.b': {$_internalExprGt: 1}}," " {$expr: {$ne: [" @@ -410,7 +410,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$and: [ {$or: [ {'control.max.b': {$_internalExprGte: 2}}," "{$expr: {$ne: [ {$type: [ \"$control.min.b\" ]}," "{$type: [ \"$control.max.b\" ]} ]}} ]}," @@ -439,7 +439,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, ->createPredicatesOnBucketLevelField(original->getMatchExpression()); ASSERT(predicate.loosePredicate); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [" " {$or: [" " {'control.max.b': {$_internalExprGt: 1}}," @@ -516,7 +516,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [" " {$or: [" " {'control.max.b': {$_internalExprGte: 2}}," @@ -557,7 +557,8 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, ASSERT_EQ(pipeline->getSources().size(), 2U); // To get the optimized $match from the pipeline, we have to serialize with explain. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps( + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); ASSERT_EQ(stages.size(), 2U); ASSERT_BSONOBJ_EQ(stages[0].getDocument().toBson(), @@ -677,8 +678,8 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, ->createPredicatesOnBucketLevelField(original->getMatchExpression()); // Meta predicates are mapped to the meta field, not the control min/max fields. - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), fromjson("{meta: {$gt: 5}}")); - ASSERT_BSONOBJ_EQ(predicate.tightPredicate->serialize(true), fromjson("{meta: {$gt: 5}}")); + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{meta: {$gt: 5}}")); + ASSERT_BSONOBJ_EQ(predicate.tightPredicate->serialize(), fromjson("{meta: {$gt: 5}}")); } TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, @@ -697,10 +698,8 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, ->createPredicatesOnBucketLevelField(original->getMatchExpression()); // Meta predicates are mapped to the meta field, not the control min/max fields. - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), - fromjson("{'meta.foo': {$gt: 5}}")); - ASSERT_BSONOBJ_EQ(predicate.tightPredicate->serialize(true), - fromjson("{'meta.foo': {$gt: 5}}")); + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{'meta.foo': {$gt: 5}}")); + ASSERT_BSONOBJ_EQ(predicate.tightPredicate->serialize(), fromjson("{'meta.foo': {$gt: 5}}")); } TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, @@ -718,7 +717,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$and: [" " {$or: [" " {'control.max.a': {$_internalExprGt: 1}}," @@ -770,10 +769,10 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 3); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.min.time" << BSON("$_internalExprLt" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.max.time" << BSON("$_internalExprLt" << datePlusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[2].get()); @@ -827,10 +826,10 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 3); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.min.time" << BSON("$_internalExprLte" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.max.time" << BSON("$_internalExprLte" << datePlusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[2].get()); @@ -876,15 +875,15 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 6); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.min.time" << BSON("$_internalExprLte" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.min.time" << BSON("$_internalExprGte" << dateMinusBucketSpan))); - ASSERT_BSONOBJ_EQ((*children)[2]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[2]->serialize(), BSON("control.max.time" << BSON("$_internalExprGte" << date))); ASSERT_BSONOBJ_EQ( - (*children)[3]->serialize(true), + (*children)[3]->serialize(), BSON("control.max.time" << BSON("$_internalExprLte" << datePlusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[4].get()); @@ -938,10 +937,10 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 3); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.max.time" << BSON("$_internalExprGt" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.min.time" << BSON("$_internalExprGt" << dateMinusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[2].get()); @@ -987,10 +986,10 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 3); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.max.time" << BSON("$_internalExprGte" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.min.time" << BSON("$_internalExprGte" << dateMinusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[2].get()); @@ -1138,7 +1137,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$_internalBucketGeoWithin: { withinRegion: { $geometry: { type : " "\"Polygon\" ,coordinates: [ [ [ 0, 0 ], [ 3, 6 ], [ 6, 1 ], [ 0, 0 " "] ] ]}},field: \"loc\"}}")); diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/optimize_pipeline_test.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/optimize_pipeline_test.cpp index 0f9a60dbf2a..5b87a64a258 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/optimize_pipeline_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/optimize_pipeline_test.cpp @@ -40,7 +40,8 @@ namespace mongo { namespace { using OptimizePipeline = AggregationContextFixture; - +const auto kExplain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; TEST_F(OptimizePipeline, MixedMatchPushedDown) { auto unpack = fromjson( "{$_internalUnpackBucket: { exclude: [], timeField: 'time', metaField: 'myMeta', " @@ -53,7 +54,7 @@ TEST_F(OptimizePipeline, MixedMatchPushedDown) { pipeline->optimizePipeline(); // To get the optimized $match from the pipeline, we have to serialize with explain. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(2u, stages.size()); // We should push down the $match on the metaField and the predicates on the control field. @@ -104,7 +105,7 @@ TEST_F(OptimizePipeline, MixedMatchOr) { pipeline->optimizePipeline(); - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(2u, stages.size()); auto expected = fromjson( "{$match: {$and: [" @@ -170,7 +171,7 @@ TEST_F(OptimizePipeline, MultipleMatchesPushedDown) { // We should push down both the $match on the metaField and the predicates on the control field. // The created $match stages should be added before $_internalUnpackBucket and merged. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(2u, stages.size()); ASSERT_BSONOBJ_EQ(fromjson("{$match: {$and: [ {meta: {$gte: 0}}," "{meta: {$lte: 5}}," @@ -199,7 +200,7 @@ TEST_F(OptimizePipeline, MultipleMatchesPushedDownWithSort) { // We should push down both the $match on the metaField and the predicates on the control field. // The created $match stages should be added before $_internalUnpackBucket and merged. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(3u, stages.size()); ASSERT_BSONOBJ_EQ(fromjson("{$match: {$and: [ { meta: { $gte: 0 } }," "{meta: { $lte: 5 } }," @@ -343,7 +344,7 @@ TEST_F(OptimizePipeline, MixedMatchThenProjectPushedDown) { pipeline->optimizePipeline(); // We can push down part of the $match and use dependency analysis on the end of the pipeline. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(3u, stages.size()); ASSERT_BSONOBJ_EQ(fromjson("{$match: {$and: [{meta: {$eq: 'abc'}}," "{$or: [ {'control.min.a': { $_internalExprLte: 4 } }," @@ -392,7 +393,7 @@ TEST_F(OptimizePipeline, ProjectThenMixedMatchPushedDown) { pipeline->optimizePipeline(); // We should push down part of the $match and do dependency analysis on the rest. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(3u, stages.size()); ASSERT_BSONOBJ_EQ(fromjson("{$match: {$and: [{meta: {$eq: \"abc\"}}," "{$or: [ {'control.min.a': {$_internalExprLte: 4}}," @@ -423,7 +424,7 @@ TEST_F(OptimizePipeline, ProjectWithRenameThenMixedMatchPushedDown) { pipeline->optimizePipeline(); // We should push down part of the $match and do dependency analysis on the end of the pipeline. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(3u, stages.size()); ASSERT_BSONOBJ_EQ( fromjson("{$match: {$and: [{$or: [ {'control.max.y': {$_internalExprGte: \"abc\"}}," @@ -1021,5 +1022,52 @@ TEST_F(OptimizePipeline, StreamingGroupIsNotEnabledWhenTimeFieldIsModified) { ASSERT_EQ(serialized.size(), 4U); ASSERT_BSONOBJ_EQ(groupSpecObj, serialized.back()); } + +TEST_F(OptimizePipeline, ComputedMetaProjFieldsAreNotInInclusionProjection) { + auto pipeline = Pipeline::parse( + makeVector( + fromjson( + "{$_internalUnpackBucket: { exclude: [], timeField: 'time', metaField: " + "'myMeta', bucketMaxSpanSeconds: 3600, computedMetaProjFields: ['time', 'y']}}"), + fromjson("{$project: {time: 1, x: 1}}")), + getExpCtx()); + ASSERT_EQ(2u, pipeline->getSources().size()); + + pipeline->optimizePipeline(); + + // The fields in 'computedMetaProjFields' that are not in the project should be removed. + auto serialized = pipeline->serializeToBson(); + ASSERT_EQ(1u, serialized.size()); + ASSERT_BSONOBJ_EQ( + fromjson("{$_internalUnpackBucket: { include: ['_id', 'time', 'x'], timeField: 'time', " + "metaField: " + "'myMeta', bucketMaxSpanSeconds: 3600, computedMetaProjFields: ['time']}}"), + serialized[0]); +} + +TEST_F(OptimizePipeline, ComputedMetaProjectFieldsAfterInclusionGetsAddedToIncludes) { + + auto pipeline = Pipeline::parse( + makeVector(fromjson("{$_internalUnpackBucket: { exclude: [], timeField: 'time', metaField: " + "'myMeta', bucketMaxSpanSeconds: 3600, computedMetaProjFields: []}}"), + fromjson("{$project: {myMeta: 1}}"), + fromjson("{$addFields: {newMeta: {$toUpper : '$myMeta'}}}")), + getExpCtx()); + ASSERT_EQ(3u, pipeline->getSources().size()); + + pipeline->optimizePipeline(); + + auto serialized = pipeline->serializeToBson(); + ASSERT_EQ(2u, serialized.size()); + + ASSERT_BSONOBJ_EQ(fromjson("{$addFields: {newMeta: {$toUpper: ['$meta']}}}"), serialized[0]); + + // 'newMeta' field gets added to 'computedMetaProjFields' and to 'include'. + auto expectedSpecObj = fromjson( + "{$_internalUnpackBucket: { include: ['_id','newMeta', 'myMeta'], timeField: 'time', " + "metaField: 'myMeta', " + "bucketMaxSpanSeconds: 3600, computedMetaProjFields: ['newMeta']}}"); + ASSERT_BSONOBJ_EQ(expectedSpecObj, serialized[1]); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/unpack_bucket_exec_test.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/unpack_bucket_exec_test.cpp index 06217344dad..3a02d4d0331 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/unpack_bucket_exec_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/unpack_bucket_exec_test.cpp @@ -56,37 +56,38 @@ TEST_F(InternalUnpackBucketExecTest, UnpackBasicIncludeAllMeasurementFields) { // This source will produce two buckets. auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {'m1': 9, 'm2': 9, 'm3': 9}, data: {_id: {'0':3, '1':4}, " - "time: {'0':3, '1':4}, " + "time: {'0':Date(3), '1':Date(4)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); // The first result exists and is as expected. auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -105,36 +106,38 @@ TEST_F(InternalUnpackBucketExecTest, UnpackExcludeASingleField) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}, " + "{'0':Date(3), '1':Date(4)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); // The first result exists and is as expected. auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -152,10 +155,10 @@ TEST_F(InternalUnpackBucketExecTest, UnpackEmptyInclude) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}, " + "{'0':Date(3), '1':Date(4)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); @@ -189,37 +192,38 @@ TEST_F(InternalUnpackBucketExecTest, UnpackEmptyExclude) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}, " + "{'0':Date(3), '1':Date(4)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -242,7 +246,7 @@ TEST_F(InternalUnpackBucketExecTest, UnpackNeitherIncludeNorExcludeDefaultsToEmp meta: {'m1': 999, 'm2': 9999}, data: { _id: {'0':1, '1':2}, - time: {'0':1, '1':2}, + time: {'0':Date(1), '1':Date(2)}, a:{'0':1, '1':2}, b:{'1':1} } @@ -252,7 +256,7 @@ TEST_F(InternalUnpackBucketExecTest, UnpackNeitherIncludeNorExcludeDefaultsToEmp meta: {m1: 9, m2: 9, m3: 9}, data: { _id: {'0':3, '1':4}, - time: {'0':3, '1':4}, + time: {'0':Date(3), '1':Date(4)}, a:{'0':1, '1':2}, b:{'1':1} } @@ -262,27 +266,28 @@ TEST_F(InternalUnpackBucketExecTest, UnpackNeitherIncludeNorExcludeDefaultsToEmp auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -300,20 +305,22 @@ TEST_F(InternalUnpackBucketExecTest, SparseColumnsWhereOneColumnIsExhaustedBefor auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, b: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -334,37 +341,38 @@ TEST_F(InternalUnpackBucketExecTest, UnpackBasicIncludeWithDollarPrefix) { // This source will produce two buckets. auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "$a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}, " + "{'0':Date(3), '1':Date(4)}, " "$a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); // The first result exists and is as expected. auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, $a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, $a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, $a: 2, b: 1}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, $a: 2, b: 1}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, $a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, $a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, $a: 2, b: 1}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, $a: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -382,32 +390,34 @@ TEST_F(InternalUnpackBucketExecTest, UnpackMetadataOnly) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}}}", + "time: {'0':Date(1), '1':Date(2)}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}}}"}, + "{'0':Date(3), '1':Date(4)}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}"))); + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -425,42 +435,45 @@ TEST_F(InternalUnpackBucketExecTest, UnpackWithStrangeTimestampOrdering) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", "{control: {'version': 1}, meta: {'m1': 9, 'm2': 9, 'm3': 9}, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}"))); + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 999, m2: 9999}, _id: 3}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 999, m2: 9999}, _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 5, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 5}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(5), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 6, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 6}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(6), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -474,39 +487,39 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesMissingMetadataWhenMet << DocumentSourceInternalUnpackBucket::kBucketMaxSpanSeconds << 3600)); auto unpack = DocumentSourceInternalUnpackBucket::createFromBsonInternal(spec.firstElement(), expCtx); - auto source = - DocumentSourceMock::createForTest({"{control: {'version': 1}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", - "{control: {'version': 1}, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, - expCtx); + auto source = DocumentSourceMock::createForTest( + {"{control: {'version': 1}, data: {_id: {'1':1, " + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", + "{control: {'version': 1}, data: {_id: {'1':4, " + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, + expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 1, _id: 1}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(1), _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 2, _id: 2}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(2), _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 3, _id: 3}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(3), _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 4, _id: 4}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(4), _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 5, _id: 5}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(5), _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 6, _id: 6}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(6), _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -524,37 +537,37 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesExcludedMetadataWhenBu DocumentSourceInternalUnpackBucket::createFromBsonInternal(spec.firstElement(), expCtx); auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", "{control: {'version': 1}, meta: {'m1': 9, 'm2': 9, 'm3': 9}, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 1, _id: 1}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(1), _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 2, _id: 2}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(2), _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 3, _id: 3}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(3), _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 4, _id: 4}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(4), _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 5, _id: 5}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(5), _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 6, _id: 6}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(6), _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -572,7 +585,7 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerThrowsOnUndefinedMetadata) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: undefined, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}"}, + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}"}, expCtx); unpack->setSource(source.get()); ASSERT_THROWS_CODE(unpack->getNext(), AssertionException, 5369600); @@ -589,9 +602,9 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerThrowsWhenMetadataIsPresentUn auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", "{control: {'version': 1}, meta: null, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, expCtx); unpack->setSource(source.get()); @@ -610,39 +623,42 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesNullMetadata) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", "{control: {'version': 1}, meta: null, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}"))); + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 999, m2: 9999}, _id: 3}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 999, m2: 9999}, _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 4, myMeta: null, _id: 4}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), + Document(fromjson("{time: Date(4), myMeta: null, _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 5, myMeta: null, _id: 5}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), + Document(fromjson("{time: Date(5), myMeta: null, _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 6, myMeta: null, _id: 6}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), + Document(fromjson("{time: Date(6), myMeta: null, _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -668,7 +684,7 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesMissingMetadata) { }, data: { _id: {'1':1, '0':2, '2': 3}, - time: {'1':1, '0': 2, '2': 3} + time: {'1':Date(1), '0': Date(2), '2': Date(3)} } })", R"( @@ -676,7 +692,7 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesMissingMetadata) { control: {'version': 1}, data: { _id: {'1':4, '0':5, '2':6}, - time: {'1':4, '0': 5, '2': 6} + time: {'1':Date(4), '0': Date(5), '2': Date(6)} } })"}, expCtx); @@ -685,30 +701,30 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesMissingMetadata) { auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}"))); + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 999, m2: 9999}, _id: 3}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 999, m2: 9999}, _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 4, _id: 4}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(4), _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 5, _id: 5}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(5), _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 6, _id: 6}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(6), _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -890,7 +906,13 @@ TEST_F(InternalUnpackBucketExecTest, ParserRoundtripsComputedMetaProjFieldsInclu auto array = std::vector<Value>{}; DocumentSourceInternalUnpackBucket::createFromBsonInternal(bson.firstElement(), getExpCtx()) ->serializeToArray(array); - ASSERT_BSONOBJ_EQ(array[0].getDocument().toBson(), bson); + + // Since fields in 'computedMetaProjFields' are not in 'include' they should be removed. + auto expectedBson = fromjson( + "{$_internalUnpackBucket: {include: [], timeField: 'time', metaField: 'meta', " + "bucketMaxSpanSeconds: 3600}}"); + + ASSERT_BSONOBJ_EQ(array[0].getDocument().toBson(), expectedBson); } TEST_F(InternalUnpackBucketExecTest, ParserRoundtripsComputedMetaProjFieldsIncludeWithCompute) { @@ -925,5 +947,18 @@ TEST_F(InternalUnpackBucketExecTest, ParserRoundtripsComputedMetaProjFieldOverri unpackBucket->serializeToArray(array); ASSERT_BSONOBJ_EQ(array[0].getDocument().toBson(), bson); } + +TEST_F(InternalUnpackBucketExecTest, RedactsCorrectly) { + auto bson = fromjson( + "{$_internalUnpackBucket: {include: ['a', 'b', 'c'], timeField: 'time', metaField: 'meta', " + "bucketMaxSpanSeconds: 3600, computedMetaProjFields: ['a', 'b', 'c']}}"); + auto docSource = DocumentSourceInternalUnpackBucket::createFromBsonInternal(bson.firstElement(), + getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + "{$_internalUnpackBucket: {include: [\"HASH<a>\", \"HASH<b>\", \"HASH<c>\"], timeField: " + "\"HASH<time>\", metaField: \"HASH<meta>\", bucketMaxSpanSeconds: \"?number\", " + "computedMetaProjFields: [\"HASH<a>\", \"HASH<b>\", \"HASH<c>\"]}}", + redact(*docSource)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_limit.cpp b/src/mongo/db/pipeline/document_source_limit.cpp index f50e795897d..2347e185a55 100644 --- a/src/mongo/db/pipeline/document_source_limit.cpp +++ b/src/mongo/db/pipeline/document_source_limit.cpp @@ -88,8 +88,8 @@ DocumentSource::GetNextResult DocumentSourceLimit::doGetNext() { return nextInput; } -Value DocumentSourceLimit::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{getSourceName(), _limit}}); +Value DocumentSourceLimit::serialize(const SerializationOptions& opts) const { + return Value(Document{{getSourceName(), opts.serializeLiteral(_limit)}}); } intrusive_ptr<DocumentSourceLimit> DocumentSourceLimit::create( diff --git a/src/mongo/db/pipeline/document_source_limit.h b/src/mongo/db/pipeline/document_source_limit.h index 6fe5370b0c2..82de8094431 100644 --- a/src/mongo/db/pipeline/document_source_limit.h +++ b/src/mongo/db/pipeline/document_source_limit.h @@ -69,7 +69,7 @@ public: */ Pipeline::SourceContainer::iterator doOptimizeAt(Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DepsTracker::State getDependencies(DepsTracker* deps) const final { return DepsTracker::State::SEE_NEXT; // This doesn't affect needed fields diff --git a/src/mongo/db/pipeline/document_source_limit_test.cpp b/src/mongo/db/pipeline/document_source_limit_test.cpp index 899dbe4de42..5465cf1bdf2 100644 --- a/src/mongo/db/pipeline/document_source_limit_test.cpp +++ b/src/mongo/db/pipeline/document_source_limit_test.cpp @@ -161,5 +161,12 @@ TEST_F(DocumentSourceLimitTest, ShouldPropagatePauses) { ASSERT_TRUE(limit->getNext().isEOF()); } +TEST_F(DocumentSourceLimitTest, RedactsCorrectly) { + auto limit = DocumentSourceLimit::create(getExpCtx(), 2); + ASSERT_VALUE_EQ_AUTO( // NOLINT + "{ $limit: \"?number\" }", + redact(*limit)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_list_cached_and_active_users.h b/src/mongo/db/pipeline/document_source_list_cached_and_active_users.h index f1f1e25e55e..ce8a4f9883a 100644 --- a/src/mongo/db/pipeline/document_source_list_cached_and_active_users.h +++ b/src/mongo/db/pipeline/document_source_list_cached_and_active_users.h @@ -87,7 +87,8 @@ public: return kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { return Value(Document{{getSourceName(), Document{}}}); } diff --git a/src/mongo/db/pipeline/document_source_list_catalog.cpp b/src/mongo/db/pipeline/document_source_list_catalog.cpp index 7e7920bd99e..1675f931d52 100644 --- a/src/mongo/db/pipeline/document_source_list_catalog.cpp +++ b/src/mongo/db/pipeline/document_source_list_catalog.cpp @@ -133,8 +133,7 @@ intrusive_ptr<DocumentSource> DocumentSourceListCatalog::createFromBson( return new DocumentSourceListCatalog(pExpCtx); } -Value DocumentSourceListCatalog::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceListCatalog::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_list_catalog.h b/src/mongo/db/pipeline/document_source_list_catalog.h index c3bcb52fe77..89398ccb519 100644 --- a/src/mongo/db/pipeline/document_source_list_catalog.h +++ b/src/mongo/db/pipeline/document_source_list_catalog.h @@ -75,7 +75,7 @@ public: // virtuals from DocumentSource const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { StageConstraints constraints(StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_list_local_sessions.h b/src/mongo/db/pipeline/document_source_list_local_sessions.h index 5306fc1fb7e..22d7657ea3d 100644 --- a/src/mongo/db/pipeline/document_source_list_local_sessions.h +++ b/src/mongo/db/pipeline/document_source_list_local_sessions.h @@ -101,8 +101,9 @@ public: return DocumentSourceListLocalSessions::kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { - return Value(Document{{getSourceName(), _spec.toBSON()}}); + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { + return Value(Document{{getSourceName(), _spec.toBSON(opts)}}); } StageConstraints constraints(Pipeline::SplitState pipeState) const final { diff --git a/src/mongo/db/pipeline/document_source_list_sessions.cpp b/src/mongo/db/pipeline/document_source_list_sessions.cpp index 3c19beb4d5f..53744d3aa12 100644 --- a/src/mongo/db/pipeline/document_source_list_sessions.cpp +++ b/src/mongo/db/pipeline/document_source_list_sessions.cpp @@ -74,13 +74,12 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceListSessions::createFromBson( return new DocumentSourceListSessions(query, pExpCtx, spec.getAllUsers(), spec.getUsers()); } -Value DocumentSourceListSessions::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceListSessions::serialize(const SerializationOptions& opts) const { ListSessionsSpec spec; spec.setAllUsers(_allUsers); spec.setUsers(_users); spec.setPredicate(_predicate); - return Value(Document{{getSourceName(), spec.toBSON()}}); + return Value(Document{{getSourceName(), spec.toBSON(opts)}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_list_sessions.h b/src/mongo/db/pipeline/document_source_list_sessions.h index 9c3e4d10d68..ec47f6c1f89 100644 --- a/src/mongo/db/pipeline/document_source_list_sessions.h +++ b/src/mongo/db/pipeline/document_source_list_sessions.h @@ -98,7 +98,7 @@ public: return DocumentSourceListSessions::kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { return {StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_list_sessions.idl b/src/mongo/db/pipeline/document_source_list_sessions.idl index 0db282831dc..c293e85b589 100644 --- a/src/mongo/db/pipeline/document_source_list_sessions.idl +++ b/src/mongo/db/pipeline/document_source_list_sessions.idl @@ -39,21 +39,31 @@ structs: ListSessionsUser: description: "A struct representing a $listSessions/$listLocalSessions User" strict: true + query_shape_component: true fields: - user: string - db: string + user: + type: string + query_shape: anonymize + db: + type: string + query_shape: anonymize ListSessionsSpec: description: "$listSessions and $listLocalSessions pipeline spec" strict: true + query_shape_component: true fields: allUsers: type: bool default: false + # This boolean parameterizes the stage rather than representing user input, so do not abstract the literal. + query_shape: parameter users: type: array<ListSessionsUser> optional: true + query_shape: literal $_internalPredicate: cpp_name: predicate type: object optional: true + query_shape: literal # This is a MatchExpression predicate and could be shape-ified rather than completely abstracted. diff --git a/src/mongo/db/pipeline/document_source_lookup.cpp b/src/mongo/db/pipeline/document_source_lookup.cpp index 5e3d067f21c..0b623cb899d 100644 --- a/src/mongo/db/pipeline/document_source_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_lookup.cpp @@ -40,6 +40,7 @@ #include "mongo/db/pipeline/document_path_support.h" #include "mongo/db/pipeline/document_source_documents.h" #include "mongo/db/pipeline/document_source_merge_gen.h" +#include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/document_source_sort.h" #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/expression_context.h" @@ -143,11 +144,9 @@ bool checkModifiedPathsSortReorder(const SortPattern& sortPattern, } // namespace -DocumentSourceLookUp::DocumentSourceLookUp( - NamespaceString fromNs, - std::string as, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, - const boost::intrusive_ptr<ExpressionContext>& expCtx) +DocumentSourceLookUp::DocumentSourceLookUp(NamespaceString fromNs, + std::string as, + const boost::intrusive_ptr<ExpressionContext>& expCtx) : DocumentSource(kStageName, expCtx), _fromNs(std::move(fromNs)), _as(std::move(as)), @@ -159,35 +158,34 @@ DocumentSourceLookUp::DocumentSourceLookUp( _fromExpCtx = expCtx->copyForSubPipeline(resolvedNamespace.ns, resolvedNamespace.uuid); _fromExpCtx->inLookup = true; - if (fromCollator) { - _fromExpCtx->setCollator(std::move(fromCollator.get())); - _hasExplicitCollation = true; - } } -DocumentSourceLookUp::DocumentSourceLookUp( - NamespaceString fromNs, - std::string as, - std::string localField, - std::string foreignField, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, - const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSourceLookUp(fromNs, as, std::move(fromCollator), expCtx) { +DocumentSourceLookUp::DocumentSourceLookUp(NamespaceString fromNs, + std::string as, + std::string localField, + std::string foreignField, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSourceLookUp(fromNs, as, expCtx) { _localField = std::move(localField); _foreignField = std::move(foreignField); // We append an additional BSONObj to '_resolvedPipeline' as a placeholder for the $match stage // we'll eventually construct from the input document. _resolvedPipeline.reserve(_resolvedPipeline.size() + 1); - _resolvedPipeline.push_back(BSON("$match" << BSONObj())); - _fieldMatchPipelineIdx = _resolvedPipeline.size() - 1; + // Initialize the introspection pipeline before we insert the $match. This is okay because we do + // not use the introspection pipeline during/after query execution, which is when the $match is + // necessary. initializeResolvedIntrospectionPipeline(); + + _resolvedPipeline.push_back(BSON("$match" << BSONObj())); + _fieldMatchPipelineIdx = _resolvedPipeline.size() - 1; } std::vector<BSONObj> extractSourceStage(const std::vector<BSONObj>& pipeline) { if (!pipeline.empty() && (pipeline[0].hasField(DocumentSourceDocuments::kStageName) || + pipeline[0].hasField(DocumentSourceQueue::kStageName) || pipeline[0].hasField("$search"_sd))) { return {pipeline[0]}; } @@ -199,10 +197,9 @@ DocumentSourceLookUp::DocumentSourceLookUp( std::string as, std::vector<BSONObj> pipeline, BSONObj letVariables, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, boost::optional<std::pair<std::string, std::string>> localForeignFields, const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSourceLookUp(fromNs, as, std::move(fromCollator), expCtx) { + : DocumentSourceLookUp(fromNs, as, expCtx) { // '_resolvedPipeline' will first be initialized by the constructor delegated to within this // constructor's initializer list. It will be populated with view pipeline prefix if 'fromNs' // represents a view. We will then append stages to ensure any view prefix is not overwritten. @@ -214,8 +211,9 @@ DocumentSourceLookUp::DocumentSourceLookUp( // the local/foreignField $match. It must next after $documents if present. auto sourceStages = extractSourceStage(pipeline); _resolvedPipeline.insert(_resolvedPipeline.end(), sourceStages.begin(), sourceStages.end()); - _resolvedPipeline.push_back(BSON("$match" << BSONObj())); - _fieldMatchPipelineIdx = _resolvedPipeline.size() - 1; + // Save the correct position of the $match, but wait to insert it until we have finished + // constructing the pipeline and created the introspection pipeline below. + _fieldMatchPipelineIdx = _resolvedPipeline.size(); // Add the user pipeline to '_resolvedPipeline' after any potential view prefix and $match _resolvedPipeline.insert( _resolvedPipeline.end(), pipeline.begin() + sourceStages.size(), pipeline.end()); @@ -240,7 +238,20 @@ DocumentSourceLookUp::DocumentSourceLookUp( _variablesParseState.defineVariable(varName)); } + // Initialize the introspection pipeline before we insert the $match (if applicable). This is + // okay because we only use the introspection pipeline for reference while doing query analysis + // and analyzing involved dependencies/variables/collections/constraints. We do not use the + // introspection pipeline during/after query execution, which is when the $match is necessary. + // It wouldn't hurt anything to include the $match in this pipeline, but we also use the + // introspection pipeline in serialization, so it would be a bit odd to include an extra empty + // $match. initializeResolvedIntrospectionPipeline(); + + // Finally, insert the $match placeholder if we need it. + if (_fieldMatchPipelineIdx) { + _resolvedPipeline.insert(_resolvedPipeline.begin() + *_fieldMatchPipelineIdx, + BSON("$match" << BSONObj())); + } } DocumentSourceLookUp::DocumentSourceLookUp(const DocumentSourceLookUp& original, @@ -256,10 +267,9 @@ DocumentSourceLookUp::DocumentSourceLookUp(const DocumentSourceLookUp& original, _variables(original._variables), _variablesParseState(original._variablesParseState.copyWith(_variables.useIdGenerator())), _fromExpCtx(original._fromExpCtx->copyWith(_resolvedNs, original._fromExpCtx->uuid)), - _hasExplicitCollation(original._hasExplicitCollation), _resolvedPipeline(original._resolvedPipeline), _userPipeline(original._userPipeline), - _resolvedIntrospectionPipeline(original._resolvedIntrospectionPipeline->clone()), + _resolvedIntrospectionPipeline(original._resolvedIntrospectionPipeline->clone(_fromExpCtx)), _letVariables(original._letVariables) { if (!_localField && !_foreignField) { _cache.emplace(internalDocumentSourceCursorBatchSizeBytes.load()); @@ -321,10 +331,8 @@ std::unique_ptr<DocumentSourceLookUp::LiteParsed> DocumentSourceLookUp::LitePars liteParsedPipeline = LiteParsedPipeline(fromNss, pipeline); } - bool hasInternalCollation = static_cast<bool>(specObj["_internalCollation"]); - return std::make_unique<DocumentSourceLookUp::LiteParsed>( - spec.fieldName(), std::move(fromNss), std::move(liteParsedPipeline), hasInternalCollation); + spec.fieldName(), std::move(fromNss), std::move(liteParsedPipeline)); } PrivilegeVector DocumentSourceLookUp::LiteParsed::requiredPrivileges( @@ -380,8 +388,6 @@ void DocumentSourceLookUp::determineSbeCompatibility() { // than indexes into arrays, which is compatible with SBE.) && !FieldRef(_localField->fullPath()).hasNumericPathComponents() && !FieldRef(_foreignField->fullPath()).hasNumericPathComponents() - // Setting a collator on an individual $lookup stage with _internalCollation isn't supported - && !_hasExplicitCollation // We currently don't lower $lookup against views ('_fromNs' does not correspond to a // view). && pExpCtx->getResolvedNamespace(_fromNs).pipeline.empty(); @@ -558,6 +564,7 @@ std::unique_ptr<Pipeline, PipelineDeleter> DocumentSourceLookUp::buildPipeline( // If we don't have a cache, build and return the pipeline immediately. if (!_cache || _cache->isAbandoned()) { MakePipelineOptions pipelineOpts; + pipelineOpts.alreadyOptimized = false; pipelineOpts.optimize = true; pipelineOpts.attachCursorSource = true; pipelineOpts.validator = lookupPipeValidator; @@ -592,6 +599,7 @@ std::unique_ptr<Pipeline, PipelineDeleter> DocumentSourceLookUp::buildPipeline( // Construct the basic pipeline without a cache stage. Avoid optimizing here since we need to // add the cache first, as detailed below. MakePipelineOptions pipelineOpts; + pipelineOpts.alreadyOptimized = false; pipelineOpts.optimize = false; pipelineOpts.attachCursorSource = false; pipelineOpts.validator = lookupPipeValidator; @@ -647,21 +655,55 @@ std::unique_ptr<Pipeline, PipelineDeleter> DocumentSourceLookUp::buildPipeline( return pipeline; } +/** + * Method that looks for a DocumentSourceSequentialDocumentCache stage and calls optimizeAt() on + * it if it has yet to be optimized. + */ +void findAndOptimizeSequentialDocumentCache(Pipeline& pipeline) { + auto& container = pipeline.getSources(); + auto itr = (&container)->begin(); + while (itr != (&container)->end()) { + if (dynamic_cast<DocumentSourceSequentialDocumentCache*>(itr->get())) { + auto sequentialCache = dynamic_cast<DocumentSourceSequentialDocumentCache*>(itr->get()); + if (!sequentialCache->hasOptimizedPos()) { + sequentialCache->optimizeAt(itr, &container); + } + } + itr = std::next(itr); + } +} + void DocumentSourceLookUp::addCacheStageAndOptimize(Pipeline& pipeline) { - // Add the cache stage at the end and optimize. During the optimization process, the cache will - // either move itself to the correct position in the pipeline, or will abandon itself if no - // suitable cache position exists. Do it only if pipeline optimization is enabled, otherwise - // Pipeline::optimizePipeline() will exit early and correct placement of the cache will not - // occur. + // Adds the cache to the end of the pipeline and calls optimizeContainer which will ensure the + // stages of the pipeline are in the correct and optimal order, before the cache runs + // doOptimizeAt. During the optimization process, the cache will either move itself to the + // correct position in the pipeline, or abandon itself if no suitable cache position exists. + // Once the cache is finished optimizing, the entire pipeline is optimized. + // + // When pipeline optimization is disabled, 'Pipeline::optimizePipeline()' exits early and so the + // cache would not be placed correctly. So we only add the cache when pipeline optimization is + // enabled. if (auto fp = globalFailPointRegistry().find("disablePipelineOptimization"); fp && fp->shouldFail()) { _cache->abandon(); } else { + // The cache needs to see the full pipeline in its correct order in order to properly place + // itself, therefore we are adding it to the end of the pipeline, and calling + // optimizeContainer on the pipeline to ensure the rest of the pipeline is in its correct + // order before optimizing the cache. pipeline.addFinalSource( DocumentSourceSequentialDocumentCache::create(_fromExpCtx, _cache.get_ptr())); - } - pipeline.optimizePipeline(); + auto& container = pipeline.getSources(); + + Pipeline::optimizeContainer(&container); + + // We want to ensure the cache has been optimized prior to any calls to optimize(). + findAndOptimizeSequentialDocumentCache(pipeline); + + // Optimize the pipeline, with the cache in its correct position if it exists. + Pipeline::optimizeEachStage(&container); + } } DocumentSource::GetModPathsReturn DocumentSourceLookUp::getModifiedPaths() const { @@ -766,12 +808,14 @@ Pipeline::SourceContainer::iterator DocumentSourceLookUp::doOptimizeAt( bool isMatchOnlyOnAs = true; auto computeWhetherMatchOnAs = [&isMatchOnlyOnAs, &outputPath](MatchExpression* expression, std::string path) -> void { - // If 'expression' is the child of a $elemMatch, we cannot internalize the $match. For - // example, {b: {$elemMatch: {$gt: 1, $lt: 4}}}, where "b" is our "_as" field. This is - // because there's no way to modify the expression to be a match just on 'b'--we cannot - // change the path to an empty string, or remove the node entirely. - if (expression->matchType() == MatchExpression::ELEM_MATCH_VALUE || - expression->matchType() == MatchExpression::ELEM_MATCH_OBJECT) { + // There are certain situations where this rewrite would not be correct. For example, + // if 'expression' is the child of a value $elemMatch, we cannot internalize the $match. + // Consider {b: {$elemMatch: {$gt: 1, $lt: 4}}}, where "b" is our "_as" field. This rewrite + // is not supported because there's no way to modify the expression to be a match just on + // 'b'--we cannot change the path to an empty string, or remove the node entirely. + // For other internal nodes with paths, we don't support the rewrite to keep the + // descendMatchOnPath implementation simple. + if (MatchExpression::isInternalNodeWithPath(expression->matchType())) { isMatchOnlyOnAs = false; } if (expression->numChildren() == 0) { @@ -1023,44 +1067,65 @@ void DocumentSourceLookUp::appendSpecificExecStats(MutableDocument& doc) const { doc["indexesUsed"] = Value{std::move(indexesUsedVec)}; } -void DocumentSourceLookUp::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { - +void DocumentSourceLookUp::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { // Support alternative $lookup from config.cache.chunks* namespaces. auto fromValue = (pExpCtx->ns.db() == _fromNs.db()) - ? Value(_fromNs.coll()) - : Value(Document{{"db", _fromNs.db()}, {"coll", _fromNs.coll()}}); + ? Value(opts.serializeIdentifier(_fromNs.coll())) + : Value(Document{{"db", opts.serializeIdentifier(_fromNs.db())}, + {"coll", opts.serializeIdentifier(_fromNs.coll())}}); - MutableDocument output( - Document{{getSourceName(), Document{{"from", fromValue}, {"as", _as.fullPath()}}}}); + MutableDocument output(Document{ + {getSourceName(), Document{{"from", fromValue}, {"as", opts.serializeFieldPath(_as)}}}}); if (hasLocalFieldForeignFieldJoin()) { - output[getSourceName()]["localField"] = Value(_localField->fullPath()); - output[getSourceName()]["foreignField"] = Value(_foreignField->fullPath()); + output[getSourceName()]["localField"] = Value(opts.serializeFieldPath(_localField.value())); + output[getSourceName()]["foreignField"] = + Value(opts.serializeFieldPath(_foreignField.value())); } // Add a pipeline field if only-pipeline syntax was used (to ensure the output is valid $lookup // syntax) or if a $match was absorbed. - auto pipeline = _userPipeline.get_value_or(std::vector<BSONObj>()); + auto serializedPipeline = [&]() -> std::vector<BSONObj> { + if (!_userPipeline) { + return std::vector<BSONObj>{}; + } + if (opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + return Pipeline::parse(*_userPipeline, _fromExpCtx, lookupPipeValidator) + ->serializeToBson(opts); + } + if (opts.serializeForQueryAnalysis) { + // If we are in query analysis, encrypted fields will have been marked in the + // introspection pipeline, so we need to serialize that here. + return _resolvedIntrospectionPipeline->serializeToBson(opts); + } + return *_userPipeline; + }(); if (_additionalFilter) { - pipeline.emplace_back(BSON("$match" << *_additionalFilter)); + auto serializedFilter = [&]() -> BSONObj { + if (opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + auto filter = + uassertStatusOK(MatchExpressionParser::parse(*_additionalFilter, pExpCtx)); + return filter->serialize(opts); + } + return *_additionalFilter; + }(); + serializedPipeline.emplace_back(BSON("$match" << serializedFilter)); } - if (!hasLocalFieldForeignFieldJoin() || pipeline.size() > 0) { + if (!hasLocalFieldForeignFieldJoin() || serializedPipeline.size() > 0) { MutableDocument exprList; for (auto letVar : _letVariables) { - exprList.addField(letVar.name, - letVar.expression->serialize(static_cast<bool>(explain))); + exprList.addField(opts.serializeFieldPathFromString(letVar.name), + letVar.expression->serialize(opts)); } output[getSourceName()]["let"] = Value(exprList.freeze()); - output[getSourceName()]["pipeline"] = Value(pipeline); - } - - if (_hasExplicitCollation) { - output[getSourceName()]["_internalCollation"] = Value(_fromExpCtx->getCollatorBSON()); + output[getSourceName()]["pipeline"] = Value(serializedPipeline); } - if (explain) { + if (opts.verbosity) { if (_unwindSrc) { const boost::optional<FieldPath> indexPath = _unwindSrc->indexPath(); output[getSourceName()]["unwinding"] = @@ -1068,8 +1133,11 @@ void DocumentSourceLookUp::serializeToArray( << _unwindSrc->preserveNullAndEmptyArrays() << "includeArrayIndex" << (indexPath ? Value(indexPath->fullPath()) : Value()))); } - - if (explain.get() >= ExplainOptions::Verbosity::kExecStats) { + // Conflict is .get() vs .value(), changed in 6.1 during + // https://jira.mongodb.org/browse/SERVER-68246. Going to stick with 7.0's version of + // '.value()', I think there is no meaningful difference here since line 1080 check that the + // optional is set. + if (opts.verbosity.value() >= ExplainOptions::Verbosity::kExecStats) { appendSpecificExecStats(output); } @@ -1203,7 +1271,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( std::vector<BSONObj> pipeline; bool hasPipeline = false; bool hasLet = false; - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator; for (auto&& argument : elem.Obj()) { const auto argName = argument.fieldNameStringData(); @@ -1229,16 +1296,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( continue; } - if (argName == "_internalCollation"_sd) { - const auto& collationSpec = argument.Obj(); - if (!collationSpec.isEmpty()) { - fromCollator.emplace(uassertStatusOK( - CollatorFactoryInterface::get(pExpCtx->opCtx->getServiceContext()) - ->makeFromBSON(collationSpec))); - } - continue; - } - uassert(ErrorCodes::FailedToParse, str::stream() << "$lookup argument '" << argName << "' must be a string, found " << argument << ": " << argument.type(), @@ -1270,7 +1327,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( std::move(as), std::move(pipeline), std::move(letVariables), - std::move(fromCollator), boost::none, pExpCtx); } else { @@ -1285,7 +1341,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( std::move(as), std::move(pipeline), std::move(letVariables), - std::move(fromCollator), std::pair(std::move(localField), std::move(foreignField)), pExpCtx); } @@ -1303,7 +1358,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( std::move(as), std::move(localField), std::move(foreignField), - std::move(fromCollator), pExpCtx); } lookupStage->determineSbeCompatibility(); diff --git a/src/mongo/db/pipeline/document_source_lookup.h b/src/mongo/db/pipeline/document_source_lookup.h index 7985601c02e..61298c298f5 100644 --- a/src/mongo/db/pipeline/document_source_lookup.h +++ b/src/mongo/db/pipeline/document_source_lookup.h @@ -71,11 +71,9 @@ public: LiteParsed(std::string parseTimeName, NamespaceString foreignNss, - boost::optional<LiteParsedPipeline> pipeline, - bool hasInternalCollation) + boost::optional<LiteParsedPipeline> pipeline) : LiteParsedDocumentSourceNestedPipelines( - std::move(parseTimeName), std::move(foreignNss), std::move(pipeline)), - _hasInternalCollation(hasInternalCollation) {} + std::move(parseTimeName), std::move(foreignNss), std::move(pipeline)) {} /** * Lookup from a sharded collection may not be allowed. @@ -91,16 +89,6 @@ public: return (involvedNss.find(nss) == involvedNss.end()); } - void assertPermittedInAPIVersion(const APIParameters& apiParameters) const final { - if (apiParameters.getAPIVersion() && *apiParameters.getAPIVersion() == "1" && - apiParameters.getAPIStrict().value_or(false)) { - uassert( - ErrorCodes::APIStrictError, - "The _internalCollation argument to $lookup is not supported in API Version 1", - !_hasInternalCollation); - } - } - void getForeignExecutionNamespaces( stdx::unordered_set<NamespaceString>& nssSet) const final { // We do not recurse on, nor insert '_foreignNss' in the event that this $lookup has @@ -112,10 +100,7 @@ public: } PrivilegeVector requiredPrivileges(bool isMongos, - bool bypassDocumentValidation) const override final; - - private: - bool _hasInternalCollation = false; + bool bypassDocumentValidation) const final; }; /** @@ -127,7 +112,7 @@ public: const char* getSourceName() const final; void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Returns the 'as' path, and possibly fields modified by an absorbed $unwind. @@ -276,7 +261,6 @@ private: */ DocumentSourceLookUp(NamespaceString fromNs, std::string as, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, const boost::intrusive_ptr<ExpressionContext>& expCtx); /** * Constructor used for a $lookup stage specified using the {from: ..., localField: ..., @@ -286,7 +270,6 @@ private: std::string as, std::string localField, std::string foreignField, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, const boost::intrusive_ptr<ExpressionContext>& expCtx); /** @@ -298,15 +281,15 @@ private: std::string as, std::vector<BSONObj> pipeline, BSONObj letVariables, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, boost::optional<std::pair<std::string, std::string>> localForeignFields, const boost::intrusive_ptr<ExpressionContext>& expCtx); /** * Should not be called; use serializeToArray instead. */ - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { - MONGO_UNREACHABLE; + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { + MONGO_UNREACHABLE_TASSERT(7484304); } GetNextResult unwindResult(); @@ -400,12 +383,6 @@ private: // namespace. boost::intrusive_ptr<ExpressionContext> _fromExpCtx; - // When a `_internalCollation` has been specified on a $lookup stage, we will set that collation - // on `_fromExpCtx`. An explicit simple collation however is represented in the same way as the - // default binary collation. We need to differentiate between the two to avoid serializing the - // collation when not set explicitly. - bool _hasExplicitCollation = false; - // Can this $lookup be pushed down into SBE? bool _sbeCompatible = false; diff --git a/src/mongo/db/pipeline/document_source_lookup_test.cpp b/src/mongo/db/pipeline/document_source_lookup_test.cpp index a565fb60c26..30f764808db 100644 --- a/src/mongo/db/pipeline/document_source_lookup_test.cpp +++ b/src/mongo/db/pipeline/document_source_lookup_test.cpp @@ -61,7 +61,8 @@ using std::vector; using DocumentSourceLookUpTest = AggregationContextFixture; const long long kDefaultMaxCacheSize = internalDocumentSourceLookupCacheSizeBytes.load(); -const auto kExplain = ExplainOptions::Verbosity::kQueryPlanner; +const auto kExplain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; // For tests which need to run in a replica set context. class ReplDocumentSourceLookUpTest : public DocumentSourceLookUpTest { @@ -608,49 +609,46 @@ TEST_F(DocumentSourceLookUpTest, LookupWithLetReParseSerializedStageWithFromDBAn ASSERT_VALUE_EQ(newSerialization[0], serialization[0]); } - -// Tests that $lookup with 'collation' can be round tripped. -TEST_F(DocumentSourceLookUpTest, LookupReParseSerializedStageWithCollation) { +// Tests that $lookup with '$documents' can be round tripped. +TEST_F(DocumentSourceLookUpTest, LookupReParseSerializedStageWithDocumentsPipelineStage) { auto expCtx = getExpCtx(); - NamespaceString fromNs("test", "coll"); + NamespaceString fromNs = NamespaceString("unittest", "$cmd.aggregate"); expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ {fromNs.coll().toString(), {fromNs, std::vector<BSONObj>()}}}); - - auto originalBSON = BSON( - "$lookup" << BSON("from" - << "coll" - << "let" - << BSON("local_x" - << "$x") - << "pipeline" << BSON_ARRAY(BSON("$match" << BSON("x" << 1))) << "as" - << "as" - << "_internalCollation" - << BSON("locale" - << "en_US" - << "caseLevel" << false << "caseFirst" - << "off" - << "strength" << 1 << "numericOrdering" << false << "alternate" - << "non-ignorable" - << "maxVariable" - << "punct" - << "normalization" << false << "backwards" << false << "version" - << "57.1"))); + auto originalBSON = + BSON("$lookup" << BSON("localField" + << "y" + << "foreignField" + << "x" + << "pipeline" + << BSON_ARRAY(BSON("$documents" + << BSON_ARRAY(BSON("x" << 5) << BSON("y" << 15)))) + << "as" + << "as")); auto lookupStage = DocumentSourceLookUp::createFromBson(originalBSON.firstElement(), expCtx); // // Serialize the $lookup stage and confirm contents. // vector<Value> serialization; - static const UnorderedFieldsBSONObjComparator kComparator; - lookupStage->serializeToArray(serialization); - auto serializedBSON = serialization[0].getDocument().toBson(); - std::cout << "serializedBSON: " << serializedBSON << std::endl; - ASSERT_EQ(kComparator.compare(serializedBSON, originalBSON), 0); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + lookupStage->serializeToArray(serialization, opts); + auto serializedDoc = serialization[0].getDocument(); + ASSERT_EQ(serializedDoc["$lookup"].getType(), BSONType::Object); - auto roundTripped = DocumentSourceLookUp::createFromBson(serializedBSON.firstElement(), expCtx); + // Ensure the $documents desugared to $queue properly. + auto serializedStage = serializedDoc["$lookup"].getDocument(); + ASSERT_EQ(serializedStage["pipeline"].getType(), BSONType::Array); + ASSERT_EQ(serializedStage["pipeline"].getArrayLength(), 4UL); + + ASSERT_EQ(serializedStage["pipeline"][0].getType(), BSONType::Object); + ASSERT_EQ(serializedStage["pipeline"][0]["$queue"].getType(), BSONType::Array); + + auto roundTripped = + DocumentSourceLookUp::createFromBson(serializedDoc.toBson().firstElement(), expCtx); vector<Value> newSerialization; - roundTripped->serializeToArray(newSerialization); + roundTripped->serializeToArray(newSerialization, opts); ASSERT_EQ(newSerialization.size(), 1UL); ASSERT_VALUE_EQ(newSerialization[0], serialization[0]); @@ -1106,13 +1104,10 @@ TEST_F(DocumentSourceLookUpTest, ExprEmbeddedInMatchExpressionShouldBeOptimized) auto& matchSource = dynamic_cast<const DocumentSourceMatch&>(*secondSource); // Ensure that the '$$var' in the embedded expression got optimized to ExpressionConstant. - BSONObjBuilder builder; - matchSource.getMatchExpression()->serialize(&builder); - auto serializedMatch = builder.obj(); auto expectedMatch = fromjson("{$and: [{_id: {$_internalExprEq: 5}}, {$expr: {$eq: ['$_id', {$const: 5}]}}]}"); - ASSERT_VALUE_EQ(Value(serializedMatch), Value(expectedMatch)); + ASSERT_VALUE_EQ(Value(matchSource.getMatchExpression()->serialize()), Value(expectedMatch)); } TEST_F(DocumentSourceLookUpTest, @@ -1426,5 +1421,57 @@ TEST_F(DocumentSourceLookUpTest, ShouldNotCacheIfCorrelatedStageIsAbsorbedIntoPl ASSERT_VALUE_EQ(Value(subPipeline->writeExplainOps(kExplain)), Value(BSONArray(expectedPipe))); } +TEST_F(DocumentSourceLookUpTest, RedactsCorrectlyWithPipeline) { + auto expCtx = getExpCtx(); + NamespaceString fromNs("test", "coll"); + expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {fromNs.coll().toString(), {fromNs, std::vector<BSONObj>()}}}); + + BSONArrayBuilder pipeline; + pipeline << BSON("$match" << BSON("a" + << "myStr")); + pipeline << BSON("$project" << BSON("_id" << 0 << "a" << 1)); + auto docSource = DocumentSourceLookUp::createFromBson( + BSON("$lookup" << BSON("from" << fromNs.coll() << "localField" + << "foo" + << "foreignField" + << "bar" + << "let" + << BSON("var1" + << "$x") + << "pipeline" << pipeline.arr() << "as" + << "out")) + .firstElement(), + expCtx); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$lookup": { + "from": "HASH<coll>", + "as": "HASH<out>", + "localField": "HASH<foo>", + "foreignField": "HASH<bar>", + "let": { + "HASH<var1>": "$HASH<x>" + }, + "pipeline": [ + { + "$match": { + "HASH<a>": { + "$eq": "?string" + } + } + }, + { + "$project": { + "HASH<a>": true, + "HASH<_id>": false + } + } + ] + } + })", + redact(*docSource)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_match.cpp b/src/mongo/db/pipeline/document_source_match.cpp index 418f1821b4b..3690b603ab7 100644 --- a/src/mongo/db/pipeline/document_source_match.cpp +++ b/src/mongo/db/pipeline/document_source_match.cpp @@ -47,6 +47,7 @@ #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/lite_parsed_document_source.h" #include "mongo/db/pipeline/semantic_analysis.h" +#include "mongo/logv2/redaction.h" #include "mongo/util/ctype.h" #include "mongo/util/str.h" @@ -67,11 +68,10 @@ const char* DocumentSourceMatch::getSourceName() const { return kStageName.rawData(); } -Value DocumentSourceMatch::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { - BSONObjBuilder builder; - _expression->serialize(&builder); - return Value(DOC(getSourceName() << Document(builder.obj()))); +Value DocumentSourceMatch::serialize(const SerializationOptions& opts) const { + if (opts.verbosity || opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + return Value(DOC(getSourceName() << Document(_expression->serialize(opts)))); } return Value(DOC(getSourceName() << Document(getQuery()))); } @@ -449,15 +449,11 @@ DocumentSourceMatch::splitSourceByFunc(const OrderedPathSet& fields, // the corresponding BSONObj may not exist. Therefore, we take each of these expressions, // serialize them, and then re-parse them, constructing new BSON that is owned by the // DocumentSourceMatch. - BSONObjBuilder firstBob; - newExpr.first->serialize(&firstBob); - auto firstMatch = DocumentSourceMatch::create(firstBob.obj(), pExpCtx); + auto firstMatch = DocumentSourceMatch::create(newExpr.first->serialize(), pExpCtx); intrusive_ptr<DocumentSourceMatch> secondMatch; if (newExpr.second) { - BSONObjBuilder secondBob; - newExpr.second->serialize(&secondBob); - secondMatch = DocumentSourceMatch::create(secondBob.obj(), pExpCtx); + secondMatch = DocumentSourceMatch::create(newExpr.second->serialize(), pExpCtx); } return {std::move(firstMatch), std::move(secondMatch)}; @@ -469,8 +465,9 @@ boost::intrusive_ptr<DocumentSourceMatch> DocumentSourceMatch::descendMatchOnPat const intrusive_ptr<ExpressionContext>& expCtx) { expression::mapOver(matchExpr, [&descendOn](MatchExpression* node, std::string path) -> void { // Cannot call this method on a $match including a $elemMatch. - invariant(node->matchType() != MatchExpression::ELEM_MATCH_OBJECT && - node->matchType() != MatchExpression::ELEM_MATCH_VALUE); + tassert(9224700, + "The given match expression has a node that represents a partial path.", + !MatchExpression::isInternalNodeWithPath(node->matchType())); // Only leaf and array match expressions have a path. if (node->getCategory() != MatchExpression::MatchCategory::kLeaf && node->getCategory() != MatchExpression::MatchCategory::kArrayMatching) { @@ -478,7 +475,10 @@ boost::intrusive_ptr<DocumentSourceMatch> DocumentSourceMatch::descendMatchOnPat } auto leafPath = node->path(); - invariant(expression::isPathPrefixOf(descendOn, leafPath)); + tassert(9224701, + str::stream() << "Expected '" << redact(descendOn) << "' to be a prefix of '" + << redact(leafPath) << "', but it is not.", + expression::isPathPrefixOf(descendOn, leafPath)); auto newPath = leafPath.substr(descendOn.size() + 1); if (node->getCategory() == MatchExpression::MatchCategory::kLeaf) { @@ -490,9 +490,7 @@ boost::intrusive_ptr<DocumentSourceMatch> DocumentSourceMatch::descendMatchOnPat } }); - BSONObjBuilder query; - matchExpr->serialize(&query); - return new DocumentSourceMatch(query.obj(), expCtx); + return new DocumentSourceMatch(matchExpr->serialize(), expCtx); } std::pair<boost::intrusive_ptr<DocumentSourceMatch>, boost::intrusive_ptr<DocumentSourceMatch>> @@ -579,4 +577,22 @@ void DocumentSourceMatch::rebuild(BSONObj filter) { getDependencies(&_dependencies); } +Value DocumentSourceInternalChangeStreamMatch::serialize(const SerializationOptions& opts) const { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || opts.transformIdentifiers) { + // Stages made internally by 'DocumentSourceChangeStream' should not be serialized for + // query stats. For query stats we will serialize only the user specified $changeStream + // stage. + return Value(); + } + return doSerialize(opts); +} + +intrusive_ptr<DocumentSourceInternalChangeStreamMatch> +DocumentSourceInternalChangeStreamMatch::create(BSONObj filter, + const intrusive_ptr<ExpressionContext>& expCtx) { + intrusive_ptr<DocumentSourceInternalChangeStreamMatch> internalMatch( + new DocumentSourceInternalChangeStreamMatch(filter, expCtx)); + return internalMatch; +} + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_match.h b/src/mongo/db/pipeline/document_source_match.h index 0f55c2c9dba..53c336a322d 100644 --- a/src/mongo/db/pipeline/document_source_match.h +++ b/src/mongo/db/pipeline/document_source_match.h @@ -126,8 +126,7 @@ public: ChangeStreamRequirement::kAllowlist}; } - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const override; /** * Attempts to combine with any subsequent $match stages, joining the query objects with a @@ -232,4 +231,42 @@ private: DepsTracker _dependencies; }; +/** + * A DocumentSource class for all internal change stream stages that are also match stages. This + * currently handles parsing for query stats. + */ +class DocumentSourceInternalChangeStreamMatch : public DocumentSourceMatch { +public: + DocumentSourceInternalChangeStreamMatch(std::unique_ptr<MatchExpression> expr, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSourceMatch(std::move(expr), expCtx) {} + + static boost::intrusive_ptr<DocumentSourceInternalChangeStreamMatch> create( + BSONObj filter, const boost::intrusive_ptr<ExpressionContext>& expCtx); + + /** + * Must override the serialize method, since internal change stream stages are serialized + * differently than match stages. This function mirrors + * DocumentSourceInternalChangeStreamStage::serialize and was added because this class cannot + * inherit from both DocumentSourceInternalChangeStreamStage and DocumentSourceMatch. + */ + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + + virtual Value doSerialize(const SerializationOptions& opts) const { + return DocumentSourceMatch::serialize(opts); + }; + +protected: + DocumentSourceInternalChangeStreamMatch(const BSONObj& query, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSourceMatch(query, expCtx) {} + + DocumentSourceInternalChangeStreamMatch( + const DocumentSourceInternalChangeStreamMatch& other, + const boost::intrusive_ptr<ExpressionContext>& newExpCtx) + : DocumentSourceMatch( + other.serialize().getDocument().toBson().firstElement().embeddedObject(), + newExpCtx ? newExpCtx : other.pExpCtx) {} +}; + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_match_test.cpp b/src/mongo/db/pipeline/document_source_match_test.cpp index f5b53a393e8..4017720e667 100644 --- a/src/mongo/db/pipeline/document_source_match_test.cpp +++ b/src/mongo/db/pipeline/document_source_match_test.cpp @@ -54,7 +54,6 @@ using std::string; // This provides access to getExpCtx(), but we'll use a different name for this test suite. using DocumentSourceMatchTest = AggregationContextFixture; -constexpr auto kExplain = ExplainOptions::Verbosity::kQueryPlanner; TEST_F(DocumentSourceMatchTest, RedactSafePortion) { auto expCtx = getExpCtx(); @@ -589,7 +588,7 @@ TEST_F(DocumentSourceMatchTest, RepeatedJoinWithShouldNotNestAnds) { DEATH_TEST_REGEX_F(DocumentSourceMatchTest, ShouldFailToDescendExpressionOnPathThatIsNotACommonPrefix, - "Invariant failure.*expression::isPathPrefixOf") { + "Tripwire assertion.*Expected 'a' to be a prefix of 'b.c', but it is not.") { const auto expCtx = getExpCtx(); const auto matchSpec = BSON("a.b" << 1 << "b.c" << 1); const auto matchExpression = @@ -597,25 +596,23 @@ DEATH_TEST_REGEX_F(DocumentSourceMatchTest, DocumentSourceMatch::descendMatchOnPath(matchExpression.get(), "a", expCtx); } -DEATH_TEST_REGEX_F(DocumentSourceMatchTest, - ShouldFailToDescendExpressionOnPathThatContainsElemMatchWithObject, - R"#(Invariant failure.*node->matchType\(\))#") { +DEATH_TEST_REGEX_F( + DocumentSourceMatchTest, + ShouldFailToDescendExpressionOnPathThatContainsElemMatchWithObject, + "Tripwire assertion.*The given match expression has a node that represents a partial path.") { const auto expCtx = getExpCtx(); const auto matchSpec = BSON("a" << BSON("$elemMatch" << BSON("a.b" << 1))); const auto matchExpression = unittest::assertGet(MatchExpressionParser::parse(matchSpec, expCtx)); - BSONObjBuilder out; - matchExpression->serialize(&out); DocumentSourceMatch::descendMatchOnPath(matchExpression.get(), "a", expCtx); } -// Due to the order of traversal of the MatchExpression tree, this test may actually trigger the -// invariant failure that the path being descended is not a prefix of the path of the -// MatchExpression node corresponding to the '$gt' expression, which will report an empty path. -DEATH_TEST_F(DocumentSourceMatchTest, - ShouldFailToDescendExpressionOnPathThatContainsElemMatchWithValue, - "Invariant failure") { +DEATH_TEST_REGEX_F(DocumentSourceMatchTest, + ShouldFailToDescendExpressionOnPathThatContainsElemMatchWithValue, + "Tripwire assertion.") { const auto expCtx = getExpCtx(); + // We will either hit the assertion that $elemMatch is not allowed to be descended on or the + // assertion that the path of the '$gt' expression (empty path) is not prefixed by 'a' const auto matchSpec = BSON("a" << BSON("$elemMatch" << BSON("$gt" << 0))); const auto matchExpression = unittest::assertGet(MatchExpressionParser::parse(matchSpec, expCtx)); @@ -698,9 +695,71 @@ TEST_F(DocumentSourceMatchTest, ShouldShowOptimizationsInExplainOutputWhenOptimi auto expectedMatch = fromjson("{$match: {a:{$eq: 1}}}"); - ASSERT_VALUE_EQ( - Value((static_cast<DocumentSourceMatch*>(optimizedMatch.get()))->serialize(kExplain)), - Value(expectedMatch)); + ASSERT_VALUE_EQ(Value((static_cast<DocumentSourceMatch*>(optimizedMatch.get())) + ->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})), + Value(expectedMatch)); +} + +TEST_F(DocumentSourceMatchTest, RedactionWithAnd) { + auto spec = fromjson(R"({ + $match: { + $and: [ + { + "a.c": "abc" + }, + { + "b": { + $gt: 10 + } + } + ] + }})"); + auto docSource = DocumentSourceMatch::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$match": { + "$and": [ + { + "HASH<a>.HASH<c>": { + "$eq": "?string" + } + }, + { + "HASH<b>": { + "$gt": "?number" + } + } + ] + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceMatchTest, RedactionWithExprPipeline) { + auto spec = fromjson(R"({ + $match: { + $expr: { + $eq: [ + '$foo', + '$bar' + ] + } + } + })"); + auto docSource = DocumentSourceMatch::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$match": { + "$expr": { + "$eq": [ + "$HASH<foo>", + "$HASH<bar>" + ] + } + } + })", + redact(*docSource)); } } // namespace diff --git a/src/mongo/db/pipeline/document_source_merge.cpp b/src/mongo/db/pipeline/document_source_merge.cpp index e2746296f01..c46cfa8c641 100644 --- a/src/mongo/db/pipeline/document_source_merge.cpp +++ b/src/mongo/db/pipeline/document_source_merge.cpp @@ -605,7 +605,7 @@ boost::optional<DocumentSource::DistributedPlanLogic> DocumentSourceMerge::distr return DocumentSourceWriter::distributedPlanLogic(); } -Value DocumentSourceMerge::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceMerge::serialize(const SerializationOptions& opts) const { DocumentSourceMergeSpec spec; spec.setTargetNss(_outputNs); spec.setLet([&]() -> boost::optional<BSONObj> { @@ -615,11 +615,27 @@ Value DocumentSourceMerge::serialize(boost::optional<ExplainOptions::Verbosity> BSONObjBuilder bob; for (auto&& [name, expr] : *_letVariables) { - bob << name << expr->serialize(static_cast<bool>(explain)); + bob << opts.serializeFieldPathFromString(name) << expr->serialize(opts); } return bob.obj(); }()); - spec.setWhenMatched(MergeWhenMatchedPolicy{_descriptor.mode.first, _pipeline}); + spec.setWhenMatched(MergeWhenMatchedPolicy{ + _descriptor.mode.first, [&]() -> boost::optional<std::vector<BSONObj>> { + if (!_pipeline.has_value()) { + return boost::none; + } + auto expCtxWithLetVariables = pExpCtx->copyWith(getOutputNs()); + if (spec.getLet()) { + BSONObjBuilder cleanLetSpecBuilder; + for (auto&& [name, expr] : *_letVariables) { + cleanLetSpecBuilder.append(name, BSONObj{}); + } + expCtxWithLetVariables->variables.seedVariablesWithLetParameters( + expCtxWithLetVariables.get(), cleanLetSpecBuilder.obj()); + } + return Pipeline::parse(_pipeline.value(), expCtxWithLetVariables) + ->serializeToBson(opts); + }()}); spec.setWhenNotMatched(_descriptor.mode.second); spec.setOn([&]() { std::vector<std::string> mergeOnFields; @@ -629,7 +645,7 @@ Value DocumentSourceMerge::serialize(boost::optional<ExplainOptions::Verbosity> return mergeOnFields; }()); spec.setTargetCollectionVersion(_targetCollectionVersion); - return Value(Document{{getSourceName(), spec.toBSON()}}); + return Value(Document{{getSourceName(), spec.toBSON(opts)}}); } std::pair<DocumentSourceMerge::BatchObject, int> DocumentSourceMerge::makeBatchObject( diff --git a/src/mongo/db/pipeline/document_source_merge.h b/src/mongo/db/pipeline/document_source_merge.h index 9388add24ba..fe5bec997e2 100644 --- a/src/mongo/db/pipeline/document_source_merge.h +++ b/src/mongo/db/pipeline/document_source_merge.h @@ -135,7 +135,7 @@ public: boost::optional<DistributedPlanLogic> distributedPlanLogic() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Creates a new $merge stage from the given arguments. diff --git a/src/mongo/db/pipeline/document_source_merge.idl b/src/mongo/db/pipeline/document_source_merge.idl index 7948517e773..6aa54ec327d 100644 --- a/src/mongo/db/pipeline/document_source_merge.idl +++ b/src/mongo/db/pipeline/document_source_merge.idl @@ -81,16 +81,19 @@ structs: DocumentSourceMergeSpec: description: A document used to specify the $merge stage of an aggregation pipeline. strict: true + query_shape_component: true fields: into: cpp_name: targetNss type: MergeTargetNss description: Target namespace to merge documents from an aggregation pipeline into. + query_shape: custom on: type: MergeOnFields optional: true description: A single field or array of fields that uniquely identify a document. + query_shape: custom let: type: object @@ -98,18 +101,21 @@ structs: description: Specifies variables to use in the update pipeline defined in MergeWhenMatchedPolicy when the 'whenMatched' mode is a custom pipeline. + query_shape: custom whenMatched: type: MergeWhenMatchedPolicy optional: true description: The merge mode for the merge operation when source and target elements match. + query_shape: parameter whenNotMatched: type: MergeWhenNotMatchedMode optional: true description: The merge mode for the merge operation when source and target elements do not match. + query_shape: parameter targetCollectionVersion: type: ChunkVersionArrayWronglyEncodedAsBSONObjFormat @@ -119,3 +125,4 @@ structs: in which case the shard key may have changed, or had its shard key refined. This also can be used to detect if the collection has gone from unsharded to sharded, and thus now has a shard key. + query_shape: parameter diff --git a/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp b/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp index 61b2aef0c97..5f349309ac7 100644 --- a/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp +++ b/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp @@ -53,6 +53,7 @@ #include "mongo/s/client/shard_registry.h" #include "mongo/s/sharding_router_test_fixture.h" #include "mongo/stdx/thread.h" +#include "mongo/unittest/bson_test_util.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -380,5 +381,42 @@ TEST_F(DocumentSourceMergeCursorsTest, ShouldEnforceSortSpecifiedViaARMParams) { future.default_timed_get(); } + +using DocumentSourceMergeCursorsShapeTest = AggregationContextFixture; +TEST_F(DocumentSourceMergeCursorsShapeTest, QueryShape) { + auto expCtx = getExpCtx(); + AsyncResultsMergerParams armParams; + armParams.setNss(kTestNss); + std::vector<RemoteCursor> cursors; + cursors.emplace_back( + makeRemoteCursor(kTestShardIds[0], kTestShardHosts[0], CursorResponse(expCtx->ns, 1, {}))); + cursors.emplace_back( + makeRemoteCursor(kTestShardIds[1], kTestShardHosts[1], CursorResponse(expCtx->ns, 2, {}))); + armParams.setRemotes(std::move(cursors)); + auto stage = DocumentSourceMergeCursors::create(expCtx, std::move(armParams)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$mergeCursors": { + "compareWholeSortKey": "?bool", + "remotes": [ + { + "shardId": "HASH<FakeShard1>", + "hostAndPort": "HASH<FakeShard1Host:12345>", + "cursorResponse": "?object" + }, + { + "shardId": "HASH<FakeShard2>", + "hostAndPort": "HASH<FakeShard2Host:12345>", + "cursorResponse": "?object" + } + ], + "nss": "HASH<test.mergeCursors>", + "allowPartialResults": false, + "recordRemoteOpWaitTime": false + } + })", + redact(*stage)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_merge_spec.cpp b/src/mongo/db/pipeline/document_source_merge_spec.cpp index 41926a7cfd9..84184804dd2 100644 --- a/src/mongo/db/pipeline/document_source_merge_spec.cpp +++ b/src/mongo/db/pipeline/document_source_merge_spec.cpp @@ -37,6 +37,7 @@ #include "mongo/db/pipeline/aggregation_request_helper.h" #include "mongo/db/pipeline/document_source_merge.h" #include "mongo/db/pipeline/document_source_merge_gen.h" +#include "mongo/db/query/query_shape/serialization_options.h" namespace mongo { using namespace fmt::literals; @@ -66,8 +67,11 @@ NamespaceString mergeTargetNssParseFromBSON(const BSONElement& elem) { void mergeTargetNssSerializeToBSON(const NamespaceString& targetNss, StringData fieldName, - BSONObjBuilder* bob) { - bob->append(fieldName, BSON("db" << targetNss.db() << "coll" << targetNss.coll())); + BSONObjBuilder* bob, + const SerializationOptions& opts) { + bob->append(fieldName, + BSON("db" << opts.serializeIdentifier(targetNss.db()) << "coll" + << opts.serializeIdentifier(targetNss.coll()))); } std::vector<std::string> mergeOnFieldsParseFromBSON(const BSONElement& elem) { @@ -104,11 +108,12 @@ std::vector<std::string> mergeOnFieldsParseFromBSON(const BSONElement& elem) { void mergeOnFieldsSerializeToBSON(const std::vector<std::string>& fields, StringData fieldName, - BSONObjBuilder* bob) { + BSONObjBuilder* bob, + const SerializationOptions& opts) { if (fields.size() == 1) { - bob->append(fieldName, fields.front()); + bob->append(fieldName, opts.serializeFieldPathFromString(fields.front())); } else { - bob->append(fieldName, fields); + bob->append(fieldName, opts.serializeFieldPathFromString(fields)); } } diff --git a/src/mongo/db/pipeline/document_source_merge_spec.h b/src/mongo/db/pipeline/document_source_merge_spec.h index 35c353bb78a..2ccd4f8548a 100644 --- a/src/mongo/db/pipeline/document_source_merge_spec.h +++ b/src/mongo/db/pipeline/document_source_merge_spec.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/db/query/query_shape/serialization_options.h" #include <boost/optional.hpp> #include <string> #include <vector> @@ -54,7 +55,8 @@ struct MergeWhenMatchedPolicy { */ void mergeTargetNssSerializeToBSON(const NamespaceString& targetNss, StringData fieldName, - BSONObjBuilder* bob); + BSONObjBuilder* bob, + const SerializationOptions& opts = {}); NamespaceString mergeTargetNssParseFromBSON(const BSONElement& elem); /** @@ -63,7 +65,8 @@ NamespaceString mergeTargetNssParseFromBSON(const BSONElement& elem); */ void mergeOnFieldsSerializeToBSON(const std::vector<std::string>& fields, StringData fieldName, - BSONObjBuilder* bob); + BSONObjBuilder* bob, + const SerializationOptions& opts = {}); std::vector<std::string> mergeOnFieldsParseFromBSON(const BSONElement& elem); /** diff --git a/src/mongo/db/pipeline/document_source_merge_test.cpp b/src/mongo/db/pipeline/document_source_merge_test.cpp index e7d87707900..1987d09da07 100644 --- a/src/mongo/db/pipeline/document_source_merge_test.cpp +++ b/src/mongo/db/pipeline/document_source_merge_test.cpp @@ -29,8 +29,10 @@ #include "mongo/platform/basic.h" +#include "mongo/unittest/bson_test_util.h" #include <boost/intrusive_ptr.hpp> +#include "mongo/bson/unordered_fields_bsonobj_comparator.h" #include "mongo/db/exec/document_value/document.h" #include "mongo/db/exec/document_value/document_value_test_util.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" @@ -792,12 +794,12 @@ TEST_F(DocumentSourceMergeTest, SerializeDefaultLetVariable) { // Test the behaviour of 'let' serialization for each whenNotMatched mode. TEST_F(DocumentSourceMergeTest, SerializeLetVariables) { - auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" - << "$$v1" - << "y" - << "$$v2" - << "z" - << "$$v3"))); + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("_id" << true << "x" + << "$$v1" + << "y" + << "$$v2" + << "z" + << "$$v3"))); const auto createAndSerializeMergeStage = [this, &pipeline](StringData whenNotMatched) { auto spec = BSON("$merge" << BSON("into" @@ -843,8 +845,8 @@ TEST_F(DocumentSourceMergeTest, SerializeLetVariables) { TEST_F(DocumentSourceMergeTest, SerializeLetArrayVariable) { for (auto&& whenNotMatched : {"insert", "fail", "discard"}) { - auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" - << "$$v1"))); + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("_id" << true << "x" + << "$$v1"))); auto spec = BSON( "$merge" << BSON("into" << "target_collection" @@ -872,8 +874,9 @@ TEST_F(DocumentSourceMergeTest, SerializeLetArrayVariable) { // SERVER-41272, this test should be updated to accordingly. TEST_F(DocumentSourceMergeTest, SerializeNullLetVariablesAsDefault) { for (auto&& whenNotMatched : {"insert", "fail", "discard"}) { - auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" - << "1"))); + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("_id" << true << "x" + << BSON("$const" + << "1")))); auto spec = BSON("$merge" << BSON("into" << "target_collection" << "let" << BSONNULL << "whenMatched" << pipeline @@ -890,8 +893,9 @@ TEST_F(DocumentSourceMergeTest, SerializeNullLetVariablesAsDefault) { TEST_F(DocumentSourceMergeTest, SerializeEmptyLetVariables) { for (auto&& whenNotMatched : {"insert", "fail", "discard"}) { - auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" - << "1"))); + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("_id" << true << "x" + << BSON("$const" + << "1")))); auto spec = BSON("$merge" << BSON("into" << "target_collection" << "let" << BSONObj() << "whenMatched" << pipeline @@ -907,6 +911,41 @@ TEST_F(DocumentSourceMergeTest, SerializeEmptyLetVariables) { } } +TEST_F(DocumentSourceMergeTest, SerializeEmptyLetVariableMentionNew) { + auto pipeline = BSON_ARRAY(fromjson("{$project: {_id: true, x: '$$new'}}")); + auto spec = + BSON("$merge" << BSON("into" + << "target_collection" + << "let" << BSONObj() << "whenMatched" << pipeline << "whenNotMatched" + << "insert")); + auto mergeStage = createMergeStage(spec); + ASSERT(mergeStage); + auto serialized = mergeStage->serialize().getDocument(); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$merge": { + "into": { + "db": "unittests", + "coll": "target_collection" + }, + "on": "_id", + "let": { + "new": "$$ROOT" + }, + "whenMatched": [ + { + "$project": { + "_id": true, + "x": "$$new" + } + } + ], + "whenNotMatched": "insert" + } + })", + serialized.toBson()); +} + TEST_F(DocumentSourceMergeTest, OnlyObjectCanBeUsedAsLetVariables) { for (auto&& whenNotMatched : {"insert", "fail", "discard"}) { auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" @@ -950,5 +989,49 @@ TEST_F(DocumentSourceMergeTest, FailsToParseIfOnFieldHaveDuplicates) { ASSERT_THROWS_CODE(createMergeStage(spec), AssertionException, 31465); } +TEST_F(DocumentSourceMergeTest, QueryShape) { + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" + << "1"))); + auto let = BSON("new" + << "$$ROOT" + << "year" + << "2020"); + auto spec = + BSON("$merge" << BSON("into" + << "target_collection" + << "let" << let << "whenMatched" << pipeline << "whenNotMatched" + << "insert")); + auto mergeStage = createMergeStage(spec); + ASSERT(mergeStage); + auto serialized = mergeStage->serialize().getDocument(); + + auto expectedBson = fromjson(R"({ + "$merge": { + "into": { + "db": "HASH<unittests>", + "coll": "HASH<target_collection>" + }, + "on": "HASH<_id>", + "let": { + "HASH<year>": "?string", + "HASH<new>": "$$ROOT" + }, + "whenMatched": [ + { + "$project": { + "HASH<_id>": true, + "HASH<x>": "?string" + } + } + ], + "whenNotMatched": "insert" + } + })"); + auto result = redact(*mergeStage); + UnorderedFieldsBSONObjComparator comparator; + ASSERT_EQ(0, comparator.compare(redact(*mergeStage), expectedBson)) + << "Expected [" << expectedBson << "] but found [" << result << "]"; +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_mock.h b/src/mongo/db/pipeline/document_source_mock.h index 39b3e17a265..67f3aad8649 100644 --- a/src/mongo/db/pipeline/document_source_mock.h +++ b/src/mongo/db/pipeline/document_source_mock.h @@ -76,7 +76,7 @@ public: DocumentSourceMock(std::deque<GetNextResult>, const boost::intrusive_ptr<ExpressionContext>&); Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override { + const SerializationOptions& opts = SerializationOptions{}) const final override { // Unlike the queue, it's okay to serialize this stage for testing purposes. return Value(Document{{getSourceName(), Document()}}); } diff --git a/src/mongo/db/pipeline/document_source_operation_metrics.cpp b/src/mongo/db/pipeline/document_source_operation_metrics.cpp index a5e523465e4..7a9dd328f9f 100644 --- a/src/mongo/db/pipeline/document_source_operation_metrics.cpp +++ b/src/mongo/db/pipeline/document_source_operation_metrics.cpp @@ -113,8 +113,7 @@ intrusive_ptr<DocumentSource> DocumentSourceOperationMetrics::createFromBson( return new DocumentSourceOperationMetrics(pExpCtx, clearMetrics); } -Value DocumentSourceOperationMetrics::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceOperationMetrics::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_operation_metrics.h b/src/mongo/db/pipeline/document_source_operation_metrics.h index d4e25d47bc5..95ac5585b42 100644 --- a/src/mongo/db/pipeline/document_source_operation_metrics.h +++ b/src/mongo/db/pipeline/document_source_operation_metrics.h @@ -89,7 +89,7 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; static boost::intrusive_ptr<DocumentSource> createFromBson( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); diff --git a/src/mongo/db/pipeline/document_source_out.cpp b/src/mongo/db/pipeline/document_source_out.cpp index 9b826fb3c89..a6188dfd382 100644 --- a/src/mongo/db/pipeline/document_source_out.cpp +++ b/src/mongo/db/pipeline/document_source_out.cpp @@ -210,8 +210,11 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceOut::createFromBson( return create(targetNS, expCtx); } -Value DocumentSourceOut::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(kStageName << DOC("db" << _outputNs.db() << "coll" << _outputNs.coll()))); +Value DocumentSourceOut::serialize(const SerializationOptions& opts) const { + MutableDocument spec; + spec["coll"] = Value(opts.serializeIdentifier(_outputNs.coll())); + spec["db"] = Value(opts.serializeIdentifier(_outputNs.db())); + return Value(Document{{kStageName, spec.freezeToValue()}}); } void DocumentSourceOut::waitWhileFailPointEnabled() { diff --git a/src/mongo/db/pipeline/document_source_out.h b/src/mongo/db/pipeline/document_source_out.h index 0be5153cdf0..4b2e043dd22 100644 --- a/src/mongo/db/pipeline/document_source_out.h +++ b/src/mongo/db/pipeline/document_source_out.h @@ -91,8 +91,7 @@ public: UnionRequirement::kNotAllowed}; } - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Creates a new $out stage from the given arguments. diff --git a/src/mongo/db/pipeline/document_source_out_test.cpp b/src/mongo/db/pipeline/document_source_out_test.cpp index aef9be321f0..1338addb727 100644 --- a/src/mongo/db/pipeline/document_source_out_test.cpp +++ b/src/mongo/db/pipeline/document_source_out_test.cpp @@ -114,5 +114,24 @@ TEST_F(DocumentSourceOutTest, SerializeToString) { ASSERT_EQ(reSerialized["$out"]["coll"].getStringData(), "some_collection"); } +TEST_F(DocumentSourceOutTest, Redaction) { + auto spec = fromjson(R"({ + $out: { + db: "foo", + coll: "bar" + } + })"); + auto docSource = DocumentSourceOut::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + $out: { + coll: "HASH<bar>", + db: "HASH<foo>" + } + })", + redact(*docSource)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_plan_cache_stats.cpp b/src/mongo/db/pipeline/document_source_plan_cache_stats.cpp index 51f7bf3d650..f49a3cba01f 100644 --- a/src/mongo/db/pipeline/document_source_plan_cache_stats.cpp +++ b/src/mongo/db/pipeline/document_source_plan_cache_stats.cpp @@ -57,9 +57,13 @@ DocumentSourcePlanCacheStats::DocumentSourcePlanCacheStats( const boost::intrusive_ptr<ExpressionContext>& expCtx) : DocumentSource(kStageName, expCtx) {} -void DocumentSourcePlanCacheStats::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +void DocumentSourcePlanCacheStats::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { + if (opts.verbosity) { + tassert(7513100, + "$planCacheStats is not equipped to serialize in explain mode with redaction on", + !opts.transformIdentifiers && + opts.literalPolicy == LiteralSerializationPolicy::kUnchanged); array.push_back(Value{ Document{{kStageName, Document{{"match"_sd, @@ -67,7 +71,7 @@ void DocumentSourcePlanCacheStats::serializeToArray( } else { array.push_back(Value{Document{{kStageName, Document{}}}}); if (_absorbedMatch) { - _absorbedMatch->serializeToArray(array); + _absorbedMatch->serializeToArray(array, opts); } } } diff --git a/src/mongo/db/pipeline/document_source_plan_cache_stats.h b/src/mongo/db/pipeline/document_source_plan_cache_stats.h index d72b0a22ac8..e1db4cf732c 100644 --- a/src/mongo/db/pipeline/document_source_plan_cache_stats.h +++ b/src/mongo/db/pipeline/document_source_plan_cache_stats.h @@ -117,7 +117,7 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + const SerializationOptions& opts = SerializationOptions{}) const final override; private: DocumentSourcePlanCacheStats(const boost::intrusive_ptr<ExpressionContext>& expCtx); @@ -125,8 +125,8 @@ private: GetNextResult doGetNext() final; Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override { - MONGO_UNREACHABLE; // Should call serializeToArray instead. + const SerializationOptions& opts = SerializationOptions{}) const final override { + MONGO_UNREACHABLE_TASSERT(7484303); // Should call serializeToArray instead. } // If running through mongos in a sharded cluster, stores the shard name so that it can be diff --git a/src/mongo/db/pipeline/document_source_plan_cache_stats_test.cpp b/src/mongo/db/pipeline/document_source_plan_cache_stats_test.cpp index 4f89e47acdf..262806e0780 100644 --- a/src/mongo/db/pipeline/document_source_plan_cache_stats_test.cpp +++ b/src/mongo/db/pipeline/document_source_plan_cache_stats_test.cpp @@ -41,6 +41,8 @@ namespace mongo { using DocumentSourcePlanCacheStatsTest = AggregationContextFixture; +static const SerializationOptions kExplain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; /** * A MongoProcessInterface used for testing which returns artificial plan cache stats. */ @@ -106,7 +108,9 @@ TEST_F(DocumentSourcePlanCacheStatsTest, CanParseAndSerializeAsExplainSuccessful const auto specObj = fromjson("{$planCacheStats: {}}"); auto stage = DocumentSourcePlanCacheStats::createFromBson(specObj.firstElement(), getExpCtx()); std::vector<Value> serialized; - stage->serializeToArray(serialized, ExplainOptions::Verbosity::kQueryPlanner); + stage->serializeToArray( + serialized, + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); ASSERT_EQ(1u, serialized.size()); ASSERT_BSONOBJ_EQ(specObj, serialized[0].getDocument().toBson()); } @@ -139,12 +143,31 @@ TEST_F(DocumentSourcePlanCacheStatsTest, SerializesSuccessfullyAfterAbsorbingMat pipeline->optimizePipeline(); ASSERT_EQ(1u, pipeline->getSources().size()); - auto serialized = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto serialized = pipeline->writeExplainOps(kExplain); ASSERT_EQ(1u, serialized.size()); ASSERT_BSONOBJ_EQ(fromjson("{$planCacheStats: {match: {foo: 'bar'}}}"), serialized[0].getDocument().toBson()); } +TEST_F(DocumentSourcePlanCacheStatsTest, RedactsSuccessfullyAfterAbsorbingMatch) { + const auto specObj = fromjson("{$planCacheStats: {}}"); + auto planCacheStats = + DocumentSourcePlanCacheStats::createFromBson(specObj.firstElement(), getExpCtx()); + auto match = DocumentSourceMatch::create(fromjson("{foo: 'bar'}"), getExpCtx()); + auto pipeline = Pipeline::create({planCacheStats, match}, getExpCtx()); + ASSERT_EQ(2u, pipeline->getSources().size()); + + pipeline->optimizePipeline(); + ASSERT_EQ(1u, pipeline->getSources().size()); + auto serialized = redactToArray(*pipeline->getSources().front()); + ASSERT_EQ(2u, serialized.size()); + + ASSERT_BSONOBJ_EQ(specObj, serialized[0].getDocument().toBson()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$match":{"HASH<foo>":{"$eq":"?string"}}})", + serialized[1].getDocument().toBson().getOwned()); +} + TEST_F(DocumentSourcePlanCacheStatsTest, ReturnsImmediateEOFWithEmptyPlanCache) { getExpCtx()->mongoProcessInterface = std::make_shared<PlanCacheStatsMongoProcessInterface>(std::vector<BSONObj>{}); diff --git a/src/mongo/db/pipeline/document_source_project_test.cpp b/src/mongo/db/pipeline/document_source_project_test.cpp index ced99ce0200..bd25639ee3e 100644 --- a/src/mongo/db/pipeline/document_source_project_test.cpp +++ b/src/mongo/db/pipeline/document_source_project_test.cpp @@ -338,6 +338,46 @@ TEST_F(ProjectStageTest, CannotAddNestedDocumentExceedingDepthLimit) { ErrorCodes::Overflow); } +/** + * A default redaction strategy that generates easy to check results for testing purposes. + */ +std::string transformIdentifiersForTest(StringData s) { + return str::stream() << "HASH<" << s << ">"; +} + +TEST_F(ProjectStageTest, ShapifyAndRedact) { + auto inclusionProject = DocumentSourceProject::create( + fromjson("{a: true, x: '$b', y: {$and: ['$c','$d']}, z: {$meta: 'textScore'}}"), + getExpCtx(), + "$project"_sd); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$project": { + "HASH<_id>": true, + "HASH<a>": true, + "HASH<x>": "$HASH<b>", + "HASH<y>": { + "$and": [ "$HASH<c>", "$HASH<d>" ] + }, + "HASH<z>": { "$meta": "textScore" } + } + })", + redact(*inclusionProject)); + + auto exclusionProject = DocumentSourceProject::create( + fromjson("{a: false, 'b.c': false}"), getExpCtx(), "$project"_sd); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$project": { + "HASH<a>": false, + "HASH<b>": { + "HASH<c>": false }, + "HASH<_id>": true } + })", + redact(*exclusionProject)); +} + TEST_F(UnsetTest, AcceptsValidUnsetSpecWithArray) { auto spec = BSON("$unset" << BSON_ARRAY("a" << "b" diff --git a/src/mongo/db/pipeline/document_source_query_stats.cpp b/src/mongo/db/pipeline/document_source_query_stats.cpp new file mode 100644 index 00000000000..69170760ace --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats.cpp @@ -0,0 +1,301 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/document_source_query_stats.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQueryStats + +#include "mongo/idl/idl_parser.h" +#include "mongo/logv2/log.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/debug_util.h" + +namespace mongo { +namespace { +Counter64 queryStatsHmacApplicationErrors; +ServerStatusMetricField<Counter64> displayWriteErrorsMetric("queryStats.numHmacApplicationErrors", + &queryStatsHmacApplicationErrors); +} // namespace + +REGISTER_DOCUMENT_SOURCE(queryStats, + DocumentSourceQueryStats::LiteParsed::parse, + DocumentSourceQueryStats::createFromBson, + AllowedWithApiStrict::kNeverInVersion1); + +namespace { + +/** + * Parse the spec object calling the `ctor` with the TransformAlgorithm enum algorithm and + * std::string hmacKey arguments. + */ +template <typename Ctor> +auto parseSpec(const BSONElement& spec, const Ctor& ctor) { + uassert(ErrorCodes::FailedToParse, + str::stream() << DocumentSourceQueryStats::kStageName + << " value must be an object. Found: " << typeName(spec.type()), + spec.type() == BSONType::Object); + BSONObj obj = spec.embeddedObject(); + TransformAlgorithmEnum algorithm = TransformAlgorithmEnum::kNone; + std::string hmacKey; + auto parsed = DocumentSourceQueryStatsSpec::parse(DocumentSourceQueryStats::kStageName, obj); + boost::optional<TransformIdentifiersSpec> transformIdentifiers = + parsed.getTransformIdentifiers(); + + if (transformIdentifiers) { + algorithm = transformIdentifiers->getAlgorithm(); + boost::optional<ConstDataRange> hmacKeyContainer = transformIdentifiers->getHmacKey(); + uassert(ErrorCodes::FailedToParse, + str::stream() << "The 'hmacKey' parameter of the $queryStats stage must be " + "specified when applying the hmac-sha-256 algorithm", + algorithm != TransformAlgorithmEnum::kHmacSha256 || + hmacKeyContainer != boost::none); + hmacKey = std::string(hmacKeyContainer->data(), (size_t)hmacKeyContainer->length()); + } + return ctor(algorithm, hmacKey); +} + +} // namespace + +BSONObj DocumentSourceQueryStats::computeQueryStatsKey(std::shared_ptr<const Key> key) const { + static const auto sha256HmacStringDataHasher = [](std::string key, const StringData& sd) { + auto hashed = SHA256Block::computeHmac( + (const uint8_t*)key.data(), key.size(), (const uint8_t*)sd.rawData(), sd.size()); + return hashed.toString(); + }; + + auto opts = SerializationOptions{}; + opts.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + if (_algorithm == TransformAlgorithmEnum::kHmacSha256) { + opts.transformIdentifiers = true; + opts.transformIdentifiersCallback = [&](StringData sd) { + return sha256HmacStringDataHasher(_hmacKey, sd); + }; + } + return key->toBson(pExpCtx->opCtx, opts); +} + +std::unique_ptr<DocumentSourceQueryStats::LiteParsed> DocumentSourceQueryStats::LiteParsed::parse( + const NamespaceString& nss, const BSONElement& spec) { + return parseSpec(spec, [&](TransformAlgorithmEnum algorithm, std::string hmacKey) { + return std::make_unique<DocumentSourceQueryStats::LiteParsed>( + spec.fieldName(), algorithm, hmacKey); + }); +} + +boost::intrusive_ptr<DocumentSource> DocumentSourceQueryStats::createFromBson( + BSONElement spec, const boost::intrusive_ptr<ExpressionContext>& pExpCtx) { + const NamespaceString& nss = pExpCtx->ns; + + uassert(ErrorCodes::InvalidNamespace, + "$queryStats must be run against the 'admin' database with {aggregate: 1}", + nss.db() == NamespaceString::kAdminDb && nss.isCollectionlessAggregateNS()); + + LOGV2_DEBUG_OPTIONS(7808300, + 1, + {logv2::LogTruncation::Disabled}, + "Logging invocation $queryStats", + "commandSpec"_attr = + spec.Obj().redact(BSONObj::RedactLevel::sensitiveOnly)); + return parseSpec(spec, [&](TransformAlgorithmEnum algorithm, std::string hmacKey) { + return new DocumentSourceQueryStats(pExpCtx, algorithm, hmacKey); + }); +} + +Value DocumentSourceQueryStats::serialize(const SerializationOptions& opts) const { + auto hmacKey = opts.serializeLiteral( + BSONBinData(_hmacKey.c_str(), _hmacKey.size(), BinDataType::Sensitive)); + if (opts.literalPolicy == LiteralSerializationPolicy::kToRepresentativeParseableValue) { + // The default shape for a BinData under this policy is empty and has sub-type 0 (general). + // This doesn't quite work for us since we assert when we parse that it is at least 32 bytes + // and also is sub-type 8 (sensitive). + hmacKey = + Value(BSONBinData("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 32, BinDataType::Sensitive)); + } + return Value{ + Document{{kStageName, + _transformIdentifiers + ? Document{{"transformIdentifiers", + Document{{"algorithm", TransformAlgorithm_serializer(_algorithm)}, + {"hmacKey", hmacKey}}}} + : Document{}}}}; +} + +DocumentSource::GetNextResult DocumentSourceQueryStats::doGetNext() { + const auto shouldLog = _algorithm != TransformAlgorithmEnum::kNone; + /** + * When a CopiedPartition is present (loaded) and contains more elements (QueryStatsEntry), we + * can process and return the next element in the _currentCopiedPartition. + * + * When the current CopiedPartition is exhausted (emptied), we move on to the next + * partition. Once we have iterated to the end of the valid partitions, we are done iteratiing + * over all the queryStatsStore entries. + * + * We iterate over a copied container (CopiedParitition) containing the entries in + * the partition to reduce the time under which the partition lock is held. + */ + auto& queryStatsStore = getQueryStatsStore(getContext()->opCtx); + + while (_currentCopiedPartition.isValidPartitionId(queryStatsStore.numPartitions())) { + if (!_currentCopiedPartition.isLoaded()) { + _currentCopiedPartition.load(queryStatsStore); + } + // CopiedPartition::load() will throw if any errors occur. + // Safe to assume _currentCopiedPartition is now loaded. + + // Exhaust all elements in the current copied partition. + // Use a while loop here to handle cases where toDocument() may fail for a specific + // QueryStatsEntry, in which case we suppress the thrown exception and continue + // iterating to the next available entry. + while (!_currentCopiedPartition.empty()) { + auto& statsEntries = _currentCopiedPartition.statsEntries; + const auto& queryStatsEntry = statsEntries.front(); + ON_BLOCK_EXIT([&statsEntries]() { statsEntries.pop_front(); }); + if (auto doc = + toDocument(_currentCopiedPartition.getReadTimestamp(), queryStatsEntry)) { + if (shouldLog) { + LOGV2_DEBUG_OPTIONS(7808301, + 3, + {logv2::LogTruncation::Disabled}, + "Logging all outputs of $queryStats", + "thisOutput"_attr = *doc); + } + return std::move(*doc); + } + } + // Once we have exhausted entries in this partition, move on to the next partition. + _currentCopiedPartition.incrementPartitionId(); + } + + if (shouldLog) { + LOGV2_DEBUG_OPTIONS( + 7808302, 3, {logv2::LogTruncation::Disabled}, "Finished logging output of $queryStats"); + } + return DocumentSource::GetNextResult::makeEOF(); +} + +boost::optional<Document> DocumentSourceQueryStats::toDocument( + const Date_t& partitionReadTime, const QueryStatsEntry& queryStatsEntry) const { + const auto& key = queryStatsEntry.key; + try { + auto queryStatsKey = computeQueryStatsKey(key); + // We use the representative shape to generate the key hash. This avoids returning duplicate + // hashes if we have bugs that cause two different representative shapes to re-parse into + // the same debug shape. + auto representativeShapeKey = key->toBson( + pExpCtx->opCtx, SerializationOptions::kRepresentativeQueryShapeSerializeOptions); + + // This SHA256 version of the hash is output to aid in data analytics use cases. In these + // cases, we often care about comparing hashes from different hosts, potentially on + // different versions and platforms. The thinking here is that the SHA256 algorithm is more + // stable across these different environments than the quicker 'absl::HashOf' + // implementation. + auto hash = SHA256Block::computeHash((const uint8_t*)representativeShapeKey.objdata(), + representativeShapeKey.objsize()) + .toString(); + return Document{{"key", std::move(queryStatsKey)}, + {"keyHash", hash}, + {"metrics", queryStatsEntry.toBSON()}, + {"asOf", partitionReadTime}}; + } catch (const DBException& ex) { + queryStatsHmacApplicationErrors.increment(); + const auto& hash = absl::Hash<query_stats::Key>{}(*key); + const auto queryShape = key->universalComponents()._queryShape->toBson( + pExpCtx->opCtx, SerializationOptions::kRepresentativeQueryShapeSerializeOptions); + LOGV2_DEBUG(7349403, + 2, + "Error encountered when applying hmac to query shape, will not publish " + "queryStats for this entry.", + "status"_attr = ex.toStatus(), + "hash"_attr = hash, + "debugQueryShape"_attr = queryShape); + + if (kDebugBuild || internalQueryStatsErrorsAreCommandFatal.load()) { + auto keyString = std::to_string(hash); + tasserted(7349401, + str::stream() << "Was not able to re-parse queryStats key when " + "reading queryStats.Status " + << ex.toString() << " Hash: " << keyString + << " Query Shape: " << queryShape.toString()); + } + } + return {}; +} + +/** + * Loads the current CopiedPartition with copies of the QueryStatsEntries located in partition of + * cache corresponding to the partitionId of the current CopiedPartition. This ensures that the + * partition mutex is only held for the duration of copying. + */ +void DocumentSourceQueryStats::CopiedPartition::load(QueryStatsStore& queryStatsStore) { + tassert(7932100, + "Attempted to load invalid partition.", + _partitionId < queryStatsStore.numPartitions()); + tassert(7932101, "Partition was already loaded.", !isLoaded()); + // 'statsEntries' should be empty, clear just in case. + statsEntries.clear(); + + // Capture the time at which reading the partition begins. + _readTimestamp = Date_t::now(); + { + // We only keep the partition (which holds a lock) + // for the time needed to collect the metrics (QueryStatsEntry) + const auto partition = queryStatsStore.getPartition(_partitionId); + + // Note the intentional copy of QueryStatsEntry. + // This will give us a snapshot of all the metrics we want to report. + for (auto&& [hash, metrics] : *partition) { + statsEntries.push_back(metrics); + } + } + _isLoaded = true; +} + +bool DocumentSourceQueryStats::CopiedPartition::isLoaded() const { + return _isLoaded; +} + +void DocumentSourceQueryStats::CopiedPartition::incrementPartitionId() { + // Ensure loaded state is reset when partitionId is incremented. + ++_partitionId; + _isLoaded = false; +} + +bool DocumentSourceQueryStats::CopiedPartition::isValidPartitionId( + QueryStatsStore::PartitionId maxNumPartitions) const { + return _partitionId < maxNumPartitions; +} + +const Date_t& DocumentSourceQueryStats::CopiedPartition::getReadTimestamp() const { + return _readTimestamp; +} + +bool DocumentSourceQueryStats::CopiedPartition::empty() const { + return statsEntries.empty(); +} +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_query_stats.h b/src/mongo/db/pipeline/document_source_query_stats.h new file mode 100644 index 00000000000..bc28a443d52 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats.h @@ -0,0 +1,191 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/document_source_query_stats_gen.h" +#include "mongo/db/pipeline/lite_parsed_document_source.h" +#include "mongo/db/query/query_stats/query_stats.h" +#include "mongo/util/producer_consumer_queue.h" + +namespace mongo { + +using namespace query_stats; + +class DocumentSourceQueryStats final : public DocumentSource { +public: + static constexpr StringData kStageName = "$queryStats"_sd; + + class LiteParsed final : public LiteParsedDocumentSource { + public: + static std::unique_ptr<LiteParsed> parse(const NamespaceString& nss, + const BSONElement& spec); + + LiteParsed(std::string parseTimeName, TransformAlgorithmEnum algorithm, std::string hmacKey) + : LiteParsedDocumentSource(std::move(parseTimeName)), + _algorithm(algorithm), + _hmacKey(hmacKey) {} + + stdx::unordered_set<NamespaceString> getInvolvedNamespaces() const override { + return stdx::unordered_set<NamespaceString>(); + } + + PrivilegeVector requiredPrivileges(bool isMongos, + bool bypassDocumentValidation) const override { + return _algorithm == TransformAlgorithmEnum::kNone + + ? PrivilegeVector{Privilege(ResourcePattern::forClusterResource(), + ActionType::queryStatsReadTransformed), + Privilege(ResourcePattern::forClusterResource(), + ActionType::queryStatsRead)} + : PrivilegeVector{Privilege(ResourcePattern::forClusterResource(), + ActionType::queryStatsReadTransformed)}; + } + + bool allowedToPassthroughFromMongos() const final { + // $queryStats must be run locally on a mongod. + return false; + } + + bool isInitialSource() const final { + return true; + } + + void assertSupportsMultiDocumentTransaction() const { + transactionNotSupported(kStageName); + } + + bool _transformIdentifiers; + + const TransformAlgorithmEnum _algorithm; + + std::string _hmacKey; + }; + + static boost::intrusive_ptr<DocumentSource> createFromBson( + BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); + + virtual ~DocumentSourceQueryStats() = default; + + StageConstraints constraints( + Pipeline::SplitState = Pipeline::SplitState::kUnsplit) const override { + StageConstraints constraints{StreamType::kStreaming, + PositionRequirement::kFirst, + HostTypeRequirement::kLocalOnly, + DiskUseRequirement::kNoDiskUse, + FacetRequirement::kNotAllowed, + TransactionRequirement::kNotAllowed, + LookupRequirement::kNotAllowed, + UnionRequirement::kNotAllowed}; + + constraints.requiresInputDocSource = false; + constraints.isIndependentOfAnyCollection = true; + return constraints; + } + + boost::optional<DistributedPlanLogic> distributedPlanLogic() final { + return boost::none; + } + + const char* getSourceName() const override { + return kStageName.rawData(); + } + + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + +private: + /* + * CopiedPartition: This struct is representative of a copied ("materialized") partition + * which should be loaded from the QueryStatsStore. It is used to hold a copy of the + * QueryStatsEntries corresponding to the provided partitionId. + * Once a CopiedPartition has been loaded from QueryStatsStore, it provides access to the + * QueryStatsEntries of the partition without requiring holding the lock over the partition in + * the partitioned cache. + */ + struct CopiedPartition { + CopiedPartition(QueryStatsStore::PartitionId partitionId) + : statsEntries(), _readTimestamp(), _partitionId(partitionId) {} + + ~CopiedPartition() = default; + + bool isLoaded() const; + + void incrementPartitionId(); + + bool isValidPartitionId(QueryStatsStore::PartitionId maxNumPartitions) const; + + const Date_t& getReadTimestamp() const; + + bool empty() const; + + void load(QueryStatsStore& queryStatsStore); + + std::deque<QueryStatsEntry> statsEntries; + + private: + Date_t _readTimestamp; + QueryStatsStore::PartitionId _partitionId; + bool _isLoaded{false}; + }; + + DocumentSourceQueryStats(const boost::intrusive_ptr<ExpressionContext>& expCtx, + TransformAlgorithmEnum algorithm = TransformAlgorithmEnum::kNone, + std::string hmacKey = {}) + : DocumentSource(kStageName, expCtx), + _currentCopiedPartition(0), + _transformIdentifiers(algorithm != TransformAlgorithmEnum::kNone), + _algorithm(algorithm), + _hmacKey(hmacKey) {} + + BSONObj computeQueryStatsKey(std::shared_ptr<const Key> key) const; + + GetNextResult doGetNext() final; + + boost::optional<Document> toDocument(const Date_t& partitionReadTime, + const QueryStatsEntry& queryStatsEntry) const; + + // The current partition copied from query stats store to avoid holding lock during reads. + CopiedPartition _currentCopiedPartition; + + // When true, apply hmac to field names from returned query shapes. + bool _transformIdentifiers; + + // The type of algorithm to use for transform identifiers as an enum, currently only + // kHmacSha256 + // ("hmac-sha-256") is supported. + const TransformAlgorithmEnum _algorithm; + + /** + * Key used for SHA-256 HMAC application on field names. + */ + std::string _hmacKey; +}; + +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_query_stats.idl b/src/mongo/db/pipeline/document_source_query_stats.idl new file mode 100644 index 00000000000..5a4fdb3cda0 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats.idl @@ -0,0 +1,63 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# +global: + cpp_namespace: "mongo" + cpp_includes: + - "mongo/db/pipeline/document_source_query_stats_validators.h" +imports: + - "mongo/idl/basic_types.idl" + - "mongo/db/query/query_stats/transform_algorithm.idl" + +structs: + TransformIdentifiersSpec: + description: "Specification for the transformIdentifiers field of $queryStats." + strict: true + fields: + algorithm: + description: "The algorithm to use to transform the identifiers." + type: TransformAlgorithm + optional: false + validator: + {callback: "validateAlgo"} + hmacKey: + description: > + The hmac key to be used to transform the identifiers. This is considered sensitive + data and should be redacted from data dumps, output I/O, logs, and other persistent + storages. + type: bindata_sensitive + optional: true + validator: + {callback: "validateHmac"} + DocumentSourceQueryStatsSpec: + description: "Specification for a $queryStats stage." + strict: true + fields: + transformIdentifiers: + description: "The algorithm and hmac key to use when transforming identifiers." + type: TransformIdentifiersSpec + optional: true diff --git a/src/mongo/db/pipeline/document_source_query_stats_test.cpp b/src/mongo/db/pipeline/document_source_query_stats_test.cpp new file mode 100644 index 00000000000..8d08c0f2bc2 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats_test.cpp @@ -0,0 +1,170 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/bsontypes.h" +#include "mongo/db/exec/document_value/document.h" +#include "mongo/db/exec/document_value/document_value_test_util.h" +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_query_stats.h" +#include "mongo/idl/server_parameter_test_util.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/str.h" + +namespace mongo { +namespace { + +/** + * Subclass AggregationContextFixture to set the ExpressionContext's namespace to 'admin' with + * {aggregate: 1} by default, so that parsing tests other than those which validate the namespace do + * not need to explicitly set it. + */ +class DocumentSourceQueryStatsTest : public AggregationContextFixture { +public: + DocumentSourceQueryStatsTest() + : AggregationContextFixture(NamespaceString::makeCollectionlessAggregateNSS("admin")) {} +}; + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfSpecIsNotObject) { + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( + fromjson("{$queryStats: 1}").firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::FailedToParse); +} + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfNotRunOnAdmin) { + getExpCtx()->ns = NamespaceString::makeCollectionlessAggregateNSS("foo"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( + fromjson("{$queryStats: {}}").firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::InvalidNamespace); +} + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfNotRunWithAggregateOne) { + getExpCtx()->ns = NamespaceString("admin.foo"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( + fromjson("{$queryStats: {}}").firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::InvalidNamespace); +} + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfUnrecognisedParameterSpecified) { + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( + fromjson("{$queryStats: {foo: true}}").firstElement(), getExpCtx()), + AssertionException, + 40415); +} + +TEST_F(DocumentSourceQueryStatsTest, ParseAndSerialize) { + const auto obj = fromjson("{$queryStats: {}}"); + const auto doc = DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()); + const auto queryStatsOp = static_cast<DocumentSourceQueryStats*>(doc.get()); + const auto expected = Document{{"$queryStats", Document{}}}; + const auto serialized = queryStatsOp->serialize().getDocument(); + ASSERT_DOCUMENT_EQ(expected, serialized); + + // Also make sure that we can parse out own serialization output. + + ASSERT_DOES_NOT_THROW( + DocumentSourceQueryStats::createFromBson(serialized.toBson().firstElement(), getExpCtx())); +} + +TEST_F(DocumentSourceQueryStatsTest, ParseAndSerializeShouldIncludeHmacKey) { + const auto obj = fromjson(R"({ + $queryStats: { + transformIdentifiers: { + algorithm: "hmac-sha-256", + hmacKey: { + $binary: "YW4gYXJiaXRyYXJ5IEhNQUNrZXkgZm9yIHRlc3Rpbmc=", + $type: "08" + } + } + } + })"); + const auto doc = DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()); + const auto queryStatsOp = static_cast<DocumentSourceQueryStats*>(doc.get()); + const auto expected = + Document{{"$queryStats", + Document{{"transformIdentifiers", + Document{{"algorithm", "hmac-sha-256"_sd}, + {"hmacKey", + BSONBinData("an arbitrary HMACkey for testing", + 32, + BinDataType::Sensitive)}}}}}}; + const auto serialized = queryStatsOp->serialize().getDocument(); + ASSERT_DOCUMENT_EQ(serialized, expected); + + // Also make sure that we can parse out own serialization output. + + ASSERT_DOES_NOT_THROW( + DocumentSourceQueryStats::createFromBson(serialized.toBson().firstElement(), getExpCtx())); +} + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfAlgorithmIsNotSupported) { + auto obj = fromjson(R"({ + $queryStats: { + transformIdentifiers: { + algorithm: "randomalgo" + } + } + })"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::BadValue); +} + +TEST_F(DocumentSourceQueryStatsTest, + ShouldFailToParseIfTransformIdentifiersSpecifiedButEmptyAlgorithm) { + auto obj = fromjson(R"({ + $queryStats: { + transformIdentifiers: { + algorithm: "" + } + } + })"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::BadValue); +} + +TEST_F(DocumentSourceQueryStatsTest, + ShouldFailToParseIfTransformIdentifiersSpecifiedButNoAlgorithm) { + auto obj = fromjson(R"({ + $queryStats: { + transformIdentifiers: { + } + } + })"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()), + AssertionException, + 40414); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/query/plan_cache_size_parameter.cpp b/src/mongo/db/pipeline/document_source_query_stats_validators.cpp index 46e42efafbf..8a3f234e325 100644 --- a/src/mongo/db/query/plan_cache_size_parameter.cpp +++ b/src/mongo/db/pipeline/document_source_query_stats_validators.cpp @@ -27,48 +27,30 @@ * it in the license file. */ -#include "mongo/db/query/plan_cache_size_parameter.h" +#include "mongo/platform/basic.h" -#include <pcrecpp.h> +#include "mongo/db/pipeline/document_source_query_stats_validators.h" +#include "mongo/db/query/query_stats/transform_algorithm_gen.h" +#include "mongo/util/str.h" +#include <set> -#include "mongo/db/query/query_knobs_gen.h" +namespace mongo { -namespace mongo::plan_cache_util { - -StatusWith<PlanCacheSizeUnits> parseUnitString(const std::string& strUnit) { - if (strUnit.empty()) { - return Status(ErrorCodes::Error{6007010}, "Unit value cannot be empty"); - } - - if (strUnit[0] == '%') { - return PlanCacheSizeUnits::kPercent; - } else if (strUnit[0] == 'M' || strUnit[0] == 'm') { - return PlanCacheSizeUnits::kMB; - } else if (strUnit[0] == 'G' || strUnit[0] == 'g') { - return PlanCacheSizeUnits::kGB; +Status validateAlgo(TransformAlgorithmEnum algorithm) { + if (algorithm == TransformAlgorithmEnum::kNone) { + return {ErrorCodes::FailedToParse, + str::stream() << "Algorithm specified but found no valid algorithm"}; } - - return Status(ErrorCodes::Error{6007011}, "Incorrect unit value"); + return Status::OK(); } -StatusWith<PlanCacheSizeParameter> PlanCacheSizeParameter::parse(const std::string& str) { - pcrecpp::RE_Options opt; - opt.set_caseless(true); - // Looks for a floating point number with followed by a unit suffix (MB, GB, %). - pcrecpp::RE re("\\s*(\\d+\\.?\\d*)\\s*(MB|GB|%)\\s*", opt); - - double size{}; - std::string strUnit{}; - if (!re.FullMatch(str, &size, &strUnit)) { - return {ErrorCodes::Error{6007012}, "Unable to parse plan cache size string"}; +Status validateHmac(std::vector<uint8_t> hmacKey) { + if (hmacKey.size() < 32) { + return {ErrorCodes::BadValue, + str::stream() << "hmacKey must be greater than or equal to 32 bytes, found length: " + << hmacKey.size()}; } - - auto statusWithUnit = parseUnitString(strUnit); - if (!statusWithUnit.isOK()) { - return statusWithUnit.getStatus(); - } - - return PlanCacheSizeParameter{size, statusWithUnit.getValue()}; + // length check + return Status::OK(); } - -} // namespace mongo::plan_cache_util +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_query_stats_validators.h b/src/mongo/db/pipeline/document_source_query_stats_validators.h new file mode 100644 index 00000000000..077552da0b3 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats_validators.h @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/query/query_stats/transform_algorithm_gen.h" + +namespace mongo { +/** + * Validate properties of the algorithm field of $queryStats.transformIdentifiers + */ +Status validateAlgo(TransformAlgorithmEnum algorithm); + +/** + * Validate properties of the hmac key field of $queryStats.transformIdentifiers + */ +Status validateHmac(std::vector<uint8_t> hmacKey); +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_queue.cpp b/src/mongo/db/pipeline/document_source_queue.cpp index c724db5ae10..a8cf1af797d 100644 --- a/src/mongo/db/pipeline/document_source_queue.cpp +++ b/src/mongo/db/pipeline/document_source_queue.cpp @@ -80,12 +80,13 @@ DocumentSource::GetNextResult DocumentSourceQueue::doGetNext() { return next; } -Value DocumentSourceQueue::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceQueue::serialize(const SerializationOptions& opts) const { ValueArrayStream vals; for (auto elem : _queue) { vals << elem.getDocument().getOwned(); } - return Value(DOC(kStageName << vals.done())); + // We treat the queue's documents as one literal in the context of redaction. + return Value(DOC(kStageName << opts.serializeLiteral(vals.done()))); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_queue.h b/src/mongo/db/pipeline/document_source_queue.h index e7eb6452d06..f170cca27d9 100644 --- a/src/mongo/db/pipeline/document_source_queue.h +++ b/src/mongo/db/pipeline/document_source_queue.h @@ -55,7 +55,7 @@ public: const char* getSourceName() const override; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const override; StageConstraints constraints(Pipeline::SplitState pipeState) const override { StageConstraints constraints{StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_queue_test.cpp b/src/mongo/db/pipeline/document_source_queue_test.cpp index b7014bd3166..2833826ddbf 100644 --- a/src/mongo/db/pipeline/document_source_queue_test.cpp +++ b/src/mongo/db/pipeline/document_source_queue_test.cpp @@ -73,10 +73,19 @@ TEST_F(QueueStageTest, QueueStageSerialize) { ASSERT_TRUE(queueStage); - auto res = queueStage->serialize(boost::none); + auto res = queueStage->serialize(); ASSERT_VALUE_EQ(res, Value{DOC("$queue" << DOC_ARRAY(DOC("a1" << 1) << DOC("a2" << 2)))}); } +TEST_F(QueueStageTest, RedactsCorrectly) { + auto queueDoc = BSON("$queue" << BSON_ARRAY(BSON("a" << 1))); + auto queueStage = DocumentSourceQueue::createFromBson(queueDoc.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$queue":"?array<?object>"})", + redact(*queueStage)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_redact.cpp b/src/mongo/db/pipeline/document_source_redact.cpp index 80dc406c620..c14651b55c7 100644 --- a/src/mongo/db/pipeline/document_source_redact.cpp +++ b/src/mongo/db/pipeline/document_source_redact.cpp @@ -143,8 +143,11 @@ boost::optional<Document> DocumentSourceRedact::redactObject(const Document& roo return boost::optional<Document>(); } else if (simpleValueCmp.evaluate(expressionResult == descendVal)) { const Document in = variables.getDocument(_currentId, root); + in.loadIntoCache(); + MutableDocument out; out.copyMetaDataFrom(in); + FieldIterator fields(in); while (fields.more()) { const Document::FieldPair field(fields.next()); @@ -169,8 +172,8 @@ intrusive_ptr<DocumentSource> DocumentSourceRedact::optimize() { return this; } -Value DocumentSourceRedact::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << _expression.get()->serialize(static_cast<bool>(explain)))); +Value DocumentSourceRedact::serialize(const SerializationOptions& opts) const { + return Value(DOC(getSourceName() << _expression.get()->serialize(opts))); } intrusive_ptr<DocumentSource> DocumentSourceRedact::createFromBson( diff --git a/src/mongo/db/pipeline/document_source_redact.h b/src/mongo/db/pipeline/document_source_redact.h index b566f2c7adf..84f1f262f5b 100644 --- a/src/mongo/db/pipeline/document_source_redact.h +++ b/src/mongo/db/pipeline/document_source_redact.h @@ -66,7 +66,7 @@ public: static boost::intrusive_ptr<DocumentSource> createFromBson( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<Expression> getExpression() { return _expression; diff --git a/src/mongo/db/pipeline/document_source_replace_root.h b/src/mongo/db/pipeline/document_source_replace_root.h index f288aa10321..a796cb069f4 100644 --- a/src/mongo/db/pipeline/document_source_replace_root.h +++ b/src/mongo/db/pipeline/document_source_replace_root.h @@ -60,9 +60,9 @@ public: _newRoot->optimize(); } - Document serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const final { - return Document{{"newRoot", _newRoot->serialize(static_cast<bool>(explain))}}; + Document serializeTransformation(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options = {}) const final { + return Document{{"newRoot", _newRoot->serialize(options)}}; } DepsTracker::State addDependencies(DepsTracker* deps) const final { diff --git a/src/mongo/db/pipeline/document_source_sample.cpp b/src/mongo/db/pipeline/document_source_sample.cpp index 24e86b38cba..40d5f47e3ad 100644 --- a/src/mongo/db/pipeline/document_source_sample.cpp +++ b/src/mongo/db/pipeline/document_source_sample.cpp @@ -83,8 +83,8 @@ DocumentSource::GetNextResult DocumentSourceSample::doGetNext() { return _sortStage->getNext(); } -Value DocumentSourceSample::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(kStageName << DOC("size" << _size))); +Value DocumentSourceSample::serialize(const SerializationOptions& opts) const { + return Value(DOC(kStageName << DOC("size" << opts.serializeLiteral(_size)))); } namespace { diff --git a/src/mongo/db/pipeline/document_source_sample.h b/src/mongo/db/pipeline/document_source_sample.h index 7dd31db3c41..577043c3ec0 100644 --- a/src/mongo/db/pipeline/document_source_sample.h +++ b/src/mongo/db/pipeline/document_source_sample.h @@ -41,7 +41,7 @@ public: const char* getSourceName() const final { return kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { return {StreamType::kBlocking, diff --git a/src/mongo/db/pipeline/document_source_sample_from_random_cursor.cpp b/src/mongo/db/pipeline/document_source_sample_from_random_cursor.cpp index 1105966e3ac..08716b4fac5 100644 --- a/src/mongo/db/pipeline/document_source_sample_from_random_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_sample_from_random_cursor.cpp @@ -143,9 +143,8 @@ DocumentSource::GetNextResult DocumentSourceSampleFromRandomCursor::getNextNonDu "sporadic failure, please try again."); } -Value DocumentSourceSampleFromRandomCursor::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << DOC("size" << _size))); +Value DocumentSourceSampleFromRandomCursor::serialize(const SerializationOptions& opts) const { + return Value(DOC(getSourceName() << DOC("size" << opts.serializeLiteral(_size)))); } DepsTracker::State DocumentSourceSampleFromRandomCursor::getDependencies(DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/document_source_sample_from_random_cursor.h b/src/mongo/db/pipeline/document_source_sample_from_random_cursor.h index 65521488158..4e7a73f0445 100644 --- a/src/mongo/db/pipeline/document_source_sample_from_random_cursor.h +++ b/src/mongo/db/pipeline/document_source_sample_from_random_cursor.h @@ -42,7 +42,7 @@ class DocumentSourceSampleFromRandomCursor final : public DocumentSource { public: static constexpr StringData kStageName = "$sampleFromRandomCursor"_sd; const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DepsTracker::State getDependencies(DepsTracker* deps) const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final { diff --git a/src/mongo/db/pipeline/document_source_sample_test.cpp b/src/mongo/db/pipeline/document_source_sample_test.cpp index 68cd93c4698..ade80752010 100644 --- a/src/mongo/db/pipeline/document_source_sample_test.cpp +++ b/src/mongo/db/pipeline/document_source_sample_test.cpp @@ -188,6 +188,17 @@ TEST_F(SampleBasics, ShouldPropagatePauses) { assertEOF(); } +TEST_F(SampleBasics, RedactsCorrectly) { + createSample(10); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$sample": { + "size": "?number" + } + })", + redact(*sample())); +} + /** * Fixture to test error cases of the $sample stage. */ @@ -402,5 +413,12 @@ DEATH_TEST_REGEX_F(SampleFromRandomCursorBasics, sample()->getNext(); } +TEST_F(SampleFromRandomCursorBasics, RedactsCorrectly) { + createSample(2); + ASSERT_VALUE_EQ_AUTO( // NOLINT + "{ $sampleFromRandomCursor: { size: \"?number\" } }", + redact(*sample())); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp b/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp index 39a13272df6..535c3f5d2be 100644 --- a/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp +++ b/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp @@ -80,9 +80,10 @@ DocumentSource::GetNextResult DocumentSourceSequentialDocumentCache::doGetNext() Pipeline::SourceContainer::iterator DocumentSourceSequentialDocumentCache::doOptimizeAt( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { - // The DocumentSourceSequentialDocumentCache should always be the last stage in the pipeline - // pre-optimization. By the time optimization reaches this point, all preceding stages are in - // the final positions which they would have occupied if no cache stage was present. + // The DocumentSourceSequentialDocumentCache relies on all other stages in the pipeline being at + // the final positions which they would have occupied if no cache stage was present. This should + // be the case when we reach this function. The cache should always be the last stage in the + // pipeline pre-optimizing. invariant(_hasOptimizedPos || std::next(itr) == container->end()); invariant((*itr).get() == this); @@ -152,12 +153,12 @@ Pipeline::SourceContainer::iterator DocumentSourceSequentialDocumentCache::doOpt return container->end(); } -Value DocumentSourceSequentialDocumentCache::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +Value DocumentSourceSequentialDocumentCache::serialize(const SerializationOptions& opts) const { + if (opts.verbosity) { return Value(Document{ {kStageName, - Document{{"maxSizeBytes"_sd, Value(static_cast<long long>(_cache->maxSizeBytes()))}, + Document{{"maxSizeBytes"_sd, + opts.serializeLiteral(static_cast<long long>(_cache->maxSizeBytes()))}, {"status"_sd, _cache->isBuilding() ? "kBuilding"_sd diff --git a/src/mongo/db/pipeline/document_source_sequential_document_cache.h b/src/mongo/db/pipeline/document_source_sequential_document_cache.h index 4e81390d4ef..ea6c99a5f9e 100644 --- a/src/mongo/db/pipeline/document_source_sequential_document_cache.h +++ b/src/mongo/db/pipeline/document_source_sequential_document_cache.h @@ -97,6 +97,10 @@ public: return newStage; } + bool hasOptimizedPos() const { + return _hasOptimizedPos; + } + protected: GetNextResult doGetNext() final; Pipeline::SourceContainer::iterator doOptimizeAt(Pipeline::SourceContainer::iterator itr, @@ -106,7 +110,7 @@ private: DocumentSourceSequentialDocumentCache(const boost::intrusive_ptr<ExpressionContext>& expCtx, SequentialDocumentCache* cache); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; SequentialDocumentCache* _cache; diff --git a/src/mongo/db/pipeline/document_source_sequential_document_cache_test.cpp b/src/mongo/db/pipeline/document_source_sequential_document_cache_test.cpp index d4480ca7c57..36b0f043faf 100644 --- a/src/mongo/db/pipeline/document_source_sequential_document_cache_test.cpp +++ b/src/mongo/db/pipeline/document_source_sequential_document_cache_test.cpp @@ -34,9 +34,11 @@ #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/json.h" +#include "mongo/db/exec/document_value/document_value_test_util.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" #include "mongo/db/pipeline/document_source_mock.h" #include "mongo/db/pipeline/document_source_sequential_document_cache.h" +#include "mongo/unittest/bson_test_util.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -74,5 +76,27 @@ TEST_F(DocumentSourceSequentialDocumentCacheTest, ReturnsEOFAfterCacheExhausted) ASSERT(documentCache->getNext().isEOF()); ASSERT(documentCache->getNext().isEOF()); } + +TEST_F(DocumentSourceSequentialDocumentCacheTest, Redaction) { + SequentialDocumentCache cache(kDefaultMaxCacheSize); + cache.add(DOC("_id" << 0)); + cache.add(DOC("_id" << 1)); + auto documentCache = DocumentSourceSequentialDocumentCache::create(getExpCtx(), &cache); + std::vector<Value> vals; + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$sequentialCache":{"maxSizeBytes":"?number","status":"kBuilding"}})", + redact(*documentCache, true, ExplainOptions::Verbosity::kQueryPlanner)); + + cache.freeze(); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$sequentialCache":{"maxSizeBytes":"?number","status":"kServing"}})", + redact(*documentCache, true, ExplainOptions::Verbosity::kQueryPlanner)); + + cache.abandon(); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$sequentialCache":{"maxSizeBytes":"?number","status":"kAbandoned"}})", + redact(*documentCache, true, ExplainOptions::Verbosity::kQueryPlanner)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.cpp b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.cpp index 3d8b330cec3..d9c5455f271 100644 --- a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.cpp +++ b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.cpp @@ -54,13 +54,12 @@ REGISTER_INTERNAL_DOCUMENT_SOURCE( // to check the FCV. feature_flags::gFeatureFlagSearchShardedFacets.isEnabledAndIgnoreFCV()); -Value DocumentSourceSetVariableFromSubPipeline::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceSetVariableFromSubPipeline::serialize(const SerializationOptions& opts) const { const auto var = "$$" + Variables::getBuiltinVariableName(_variableID); SetVariableFromSubPipelineSpec spec; tassert(625298, "SubPipeline cannot be null during serialization", _subPipeline); - spec.setSetVariable(var); - spec.setPipeline(_subPipeline->serializeToBson(explain)); + spec.setSetVariable(opts.serializeIdentifier(var)); + spec.setPipeline(_subPipeline->serializeToBson(opts)); return Value(DOC(getSourceName() << spec.toBSON())); } diff --git a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.h b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.h index 7d366d9a912..e06cc9a2333 100644 --- a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.h +++ b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.h @@ -111,7 +111,7 @@ protected: private: GetNextResult doGetNext() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; std::unique_ptr<Pipeline, PipelineDeleter> _subPipeline; Variables::Id _variableID; // $setVariableFromSubPipeline sets the value of $$SEARCH_META only on the first call to diff --git a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline_test.cpp b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline_test.cpp index e7cc8d71a27..8b30d3c95b7 100644 --- a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline_test.cpp +++ b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline_test.cpp @@ -158,6 +158,41 @@ TEST_F(DocumentSourceSetVariableFromSubPipelineTest, testDoGetNext) { Value((BSON("d" << 1))), nullptr) == 0); } +TEST_F(DocumentSourceSetVariableFromSubPipelineTest, QueryShape) { + const auto inputDocs = + std::vector{Document{{"a", 1}}, Document{{"b", 1}}, Document{{"c", 1}}, Document{{"d", 1}}}; + auto expCtx = getExpCtx(); + const auto mockSourceForSetVarStage = DocumentSourceMock::createForTest(inputDocs[1], expCtx); + auto ctxForSubPipeline = expCtx->copyForSubPipeline(expCtx->ns); + const auto mockSourceForSubPipeline = + DocumentSourceMock::createForTest(inputDocs, ctxForSubPipeline); + auto setVariableFromSubPipeline = DocumentSourceSetVariableFromSubPipeline::create( + expCtx, + Pipeline::create({DocumentSourceMatch::create(BSON("d" << 1), ctxForSubPipeline)}, + ctxForSubPipeline), + Variables::kSearchMetaId); + setVariableFromSubPipeline->addSubPipelineInitialSource(mockSourceForSubPipeline); + setVariableFromSubPipeline->setSource(mockSourceForSetVarStage.get()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$setVariableFromSubPipeline": { + "setVariable": "HASH<$$SEARCH_META>", + "pipeline": [ + { + "mock": {} + }, + { + "$match": { + "HASH<d>": { + "$eq": "?number" + } + } + } + ] + } + })", + redact(*setVariableFromSubPipeline)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_set_window_fields.cpp b/src/mongo/db/pipeline/document_source_set_window_fields.cpp index 73806992561..e32fc78c04b 100644 --- a/src/mongo/db/pipeline/document_source_set_window_fields.cpp +++ b/src/mongo/db/pipeline/document_source_set_window_fields.cpp @@ -144,8 +144,8 @@ WindowFunctionStatement WindowFunctionStatement::parse(BSONElement elem, window_function::Expression::parse(elem.embeddedObject(), sortBy, expCtx)); } void WindowFunctionStatement::serialize(MutableDocument& outputFields, - boost::optional<ExplainOptions::Verbosity> explain) const { - outputFields[fieldName] = expr->serialize(explain); + const SerializationOptions& opts) const { + outputFields[opts.serializeFieldPathFromString(fieldName)] = expr->serialize(opts); } list<intrusive_ptr<DocumentSource>> document_source_set_window_fields::create( @@ -218,12 +218,10 @@ list<intrusive_ptr<DocumentSource>> document_source_set_window_fields::create( } else { // In DocumentSource we don't have a mechanism for generating non-colliding field names, // so we have to choose the tmp name carefully to make a collision unlikely in practice. - std::array<unsigned char, 16> nonce = UUID::gen().data(); - // We encode as a base64 string for a shorter, more performant field name (length 22). - std::string tmpField = base64::encode(nonce.data(), sizeof(nonce)); - simplePartitionBy = FieldPath{tmpField}; + auto tmp = "__internal_setWindowFields_partition_key"; + simplePartitionBy = FieldPath{tmp}; simplePartitionByExpr = ExpressionFieldPath::createPathFromString( - expCtx.get(), tmpField, expCtx->variablesParseState); + expCtx.get(), tmp, expCtx->variablesParseState); complexPartitionBy = partitionBy; } } @@ -288,39 +286,38 @@ intrusive_ptr<DocumentSource> DocumentSourceInternalSetWindowFields::optimize() return this; } -Value DocumentSourceInternalSetWindowFields::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalSetWindowFields::serialize(const SerializationOptions& opts) const { MutableDocument spec; spec[SetWindowFieldsSpec::kPartitionByFieldName] = - _partitionBy ? (*_partitionBy)->serialize(false) : Value(); + _partitionBy ? (*_partitionBy)->serialize(opts) : Value(); - auto sortKeySerialization = explain + auto sortKeySerialization = opts.verbosity ? SortPattern::SortKeySerialization::kForExplain : SortPattern::SortKeySerialization::kForPipelineSerialization; spec[SetWindowFieldsSpec::kSortByFieldName] = - _sortBy ? Value(_sortBy->serialize(sortKeySerialization)) : Value(); + _sortBy ? Value(_sortBy->serialize(sortKeySerialization, opts)) : Value(); MutableDocument output; for (auto&& stmt : _outputFields) { - stmt.serialize(output, explain); + stmt.serialize(output, opts); } spec[SetWindowFieldsSpec::kOutputFieldName] = output.freezeToValue(); MutableDocument out; out[getSourceName()] = Value(spec.freeze()); - if (explain && *explain >= ExplainOptions::Verbosity::kExecStats) { + if (opts.verbosity && *opts.verbosity >= ExplainOptions::Verbosity::kExecStats) { MutableDocument md; for (auto&& [fieldName, function] : _executableOutputs) { - md[fieldName] = - Value(static_cast<long long>(_memoryTracker[fieldName].maxMemoryBytes())); + md[opts.serializeFieldPathFromString(fieldName)] = opts.serializeLiteral( + static_cast<long long>(_memoryTracker[fieldName].maxMemoryBytes())); } out["maxFunctionMemoryUsageBytes"] = Value(md.freezeToValue()); out["maxTotalMemoryUsageBytes"] = - Value(static_cast<long long>(_memoryTracker.maxMemoryBytes())); - out["usedDisk"] = Value(_iterator.usedDisk()); + opts.serializeLiteral(static_cast<long long>(_memoryTracker.maxMemoryBytes())); + out["usedDisk"] = opts.serializeLiteral(_iterator.usedDisk()); } return Value(out.freezeToValue()); @@ -477,13 +474,15 @@ DocumentSource::GetNextResult DocumentSourceInternalSetWindowFields::doGetNext() // Populate the output document with the result from each window function. auto projSpec = std::make_unique<projection_executor::InclusionNode>( ProjectionPolicies{ProjectionPolicies::DefaultIdPolicy::kIncludeId}); - for (auto&& [fieldName, function] : _executableOutputs) { + for (auto&& outputField : _outputFields) { try { // If we hit a uassert while evaluating expressions on user data, delete the temporary // table before aborting the operation. + auto& fieldName = outputField.fieldName; projSpec->addExpressionForPath( FieldPath(fieldName), - ExpressionConstant::create(pExpCtx.get(), function->getNext())); + ExpressionConstant::create(pExpCtx.get(), + _executableOutputs[fieldName]->getNext())); } catch (const DBException&) { _iterator.finalize(); throw; diff --git a/src/mongo/db/pipeline/document_source_set_window_fields.h b/src/mongo/db/pipeline/document_source_set_window_fields.h index f0f9b0742ce..43af58e2dad 100644 --- a/src/mongo/db/pipeline/document_source_set_window_fields.h +++ b/src/mongo/db/pipeline/document_source_set_window_fields.h @@ -70,8 +70,7 @@ struct WindowFunctionStatement { } } - void serialize(MutableDocument& outputFields, - boost::optional<ExplainOptions::Verbosity> explain) const; + void serialize(MutableDocument& outputFields, const SerializationOptions& opts) const; }; /** @@ -167,7 +166,7 @@ public: boost::intrusive_ptr<DocumentSource> optimize() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DocumentSource::GetNextResult doGetNext(); diff --git a/src/mongo/db/pipeline/document_source_set_window_fields_test.cpp b/src/mongo/db/pipeline/document_source_set_window_fields_test.cpp index 287497db445..2f35d4a6b94 100644 --- a/src/mongo/db/pipeline/document_source_set_window_fields_test.cpp +++ b/src/mongo/db/pipeline/document_source_set_window_fields_test.cpp @@ -212,5 +212,342 @@ TEST_F(DocumentSourceSetWindowFieldsTest, ReportsModifiedFields) { ASSERT_EQUALS(modified.paths.count("b"), 1U); ASSERT_TRUE(modified.renames.empty()); } + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnShiftOperator) { + auto spec = fromjson( + R"({ + $setWindowFields: { + partitionBy: '$foo', + sortBy: { + bar: 1 + }, + output: { + x: { + $shift: { + output: '$y', + by: 1, + default: 'BAZ' + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": "$HASH<foo>", + "sortBy": { + "HASH<bar>": 1 + }, + "output": { + "HASH<x>": { + "$shift": { + "by": "?number", + "output": "$HASH<y>", + "default": "?string" + } + } + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnExpMovingAvgOperator) { + auto spec = fromjson( + R"({ + $setWindowFields: { + partitionBy: '$foo.bar', + sortBy: { + bar: 1 + }, + output: { + x: { + $expMovingAvg: { + alpha: 0.5, + input: '$y' + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": "$HASH<foo>.HASH<bar>", + "sortBy": { + "HASH<bar>": 1 + }, + "output": { + "HASH<x>": { + "$expMovingAvg": { + "alpha": "?number", + "input": "$HASH<y>" + } + } + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnExpressionNOperator) { + auto spec = fromjson( + R"({ + $setWindowFields: { + partitionBy: '$a', + output: { + b: { + $minN: { + n: 3, + input: '$y' + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": "$HASH<a>", + "output": { + "HASH<b>": { + "$minN": { + "n": "?number", + "input": "$HASH<y>" + }, + "window": { + "documents": [ + "unbounded", + "unbounded" + ] + } + } + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnSumWithDocumentsWindow) { + auto spec = fromjson(R"({ + $setWindowFields: { + partitionBy: { + $year: '$x' + }, + sortBy: { + a: 1, + b: -1 + }, + output: { + cumulative: { + $sum: '$baz', + window: { + documents: [ + 'unbounded', + 'current' + ] + } + }, + maximum: { + $max: '$baz', + window: { + documents: [ + 'unbounded', + 'unbounded' + ] + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": { + "$year": { + "date": "$HASH<x>" + } + }, + "sortBy": { + "HASH<a>": 1, + "HASH<b>": -1 + }, + "output": { + "HASH<cumulative>": { + "$sum": "$HASH<baz>", + "window": { + "documents": [ + "unbounded", + "current" + ] + } + }, + "HASH<maximum>": { + "$max": "$HASH<baz>", + "window": { + "documents": [ + "unbounded", + "unbounded" + ] + } + } + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnPushWithRangeWindowWithUnit) { + auto spec = fromjson(R"({ + $setWindowFields: { + partitionBy: '$foo', + sortBy: { + bar: 1 + }, + output: { + a: { + $push: '$b', + window: { + range: [ + 'unbounded', + -10 + ], + unit: 'month' + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": "$HASH<foo>", + "sortBy": { + "HASH<bar>": 1 + }, + "output": { + "HASH<a>": { + "$push": "$HASH<b>", + "window": { + "range": [ + "unbounded", + "?number" + ], + "unit": "month" + } + } + } + } + })", + redact(*docSource)); +} + +/** + * Helper function that parses the $setWindowFields aggregation stage from the input, serializes it + * to its representative shape, re-parses the representative shape, and compares to the original. + */ +void assertRepresentativeShapeIsStable(const boost::intrusive_ptr<ExpressionContext>& expCtx, + BSONObj inputStage, + BSONObj expectedRepresentativeStage) { + auto parsedStage = + DocumentSourceInternalSetWindowFields::createFromBson(inputStage.firstElement(), expCtx); + std::vector<Value> serialization; + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + parsedStage->serializeToArray(serialization, opts); + + auto serializedStage = serialization[0].getDocument().toBson(); + ASSERT_BSONOBJ_EQ(serializedStage, expectedRepresentativeStage); + + auto roundTripped = DocumentSourceInternalSetWindowFields::createFromBson( + serializedStage.firstElement(), expCtx); + + std::vector<Value> newSerialization; + roundTripped->serializeToArray(newSerialization, opts); + ASSERT_EQ(newSerialization.size(), 1UL); + ASSERT_VALUE_EQ(newSerialization[0], serialization[0]); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RoundTripSerializationDocumentWindowBounds) { + assertRepresentativeShapeIsStable(getExpCtx(), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {documents: [-10, 10]}}}}})"), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {documents: [0, 1]}}}}})")); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RoundTripSerializationRangeWindowBounds) { + assertRepresentativeShapeIsStable(getExpCtx(), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {range: [-10, 10]}}}}})"), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {range: [0, 1]}}}}})")); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RoundTripSerializationRangeWindowBoundsWithUnit) { + assertRepresentativeShapeIsStable(getExpCtx(), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {range: [-10, 10], unit: 'second'}}}}})"), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {range: [0, 1], unit: 'second'}}}}})")); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RoundTripSerializationExpMovingAvg) { + assertRepresentativeShapeIsStable(getExpCtx(), + fromjson( + R"({ + $setWindowFields: { + partitionBy: '$foo.bar', + sortBy: { + bar: 1 + }, + output: { + x: { + $expMovingAvg: { + alpha: 0.5, + input: '$y' + } + } + } + } + })"), + fromjson( + R"({ + $_internalSetWindowFields: { + partitionBy: '$foo.bar', + sortBy: { + bar: 1 + }, + output: { + x: { + $expMovingAvg: { + alpha: 0.1, + input: '$y' + } + } + } + } + })")); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_single_document_transformation.cpp b/src/mongo/db/pipeline/document_source_single_document_transformation.cpp index 3e6e8373b56..33820b0ceaf 100644 --- a/src/mongo/db/pipeline/document_source_single_document_transformation.cpp +++ b/src/mongo/db/pipeline/document_source_single_document_transformation.cpp @@ -89,10 +89,11 @@ void DocumentSourceSingleDocumentTransformation::doDispose() { } Value DocumentSourceSingleDocumentTransformation::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{getSourceName(), - _parsedTransform ? _parsedTransform->serializeTransformation(explain) - : _cachedStageOptions}}); + const SerializationOptions& opts) const { + return Value( + Document{{getSourceName(), + _parsedTransform ? _parsedTransform->serializeTransformation(opts.verbosity, opts) + : _cachedStageOptions}}); } Pipeline::SourceContainer::iterator DocumentSourceSingleDocumentTransformation::doOptimizeAt( diff --git a/src/mongo/db/pipeline/document_source_single_document_transformation.h b/src/mongo/db/pipeline/document_source_single_document_transformation.h index eebe9adad22..b3cc1538bfe 100644 --- a/src/mongo/db/pipeline/document_source_single_document_transformation.h +++ b/src/mongo/db/pipeline/document_source_single_document_transformation.h @@ -62,7 +62,7 @@ public: const char* getSourceName() const final; boost::intrusive_ptr<DocumentSource> optimize() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DepsTracker::State getDependencies(DepsTracker* deps) const final; GetModPathsReturn getModifiedPaths() const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final { diff --git a/src/mongo/db/pipeline/document_source_skip.cpp b/src/mongo/db/pipeline/document_source_skip.cpp index df7537fd1dd..1a9ea0cd6c9 100644 --- a/src/mongo/db/pipeline/document_source_skip.cpp +++ b/src/mongo/db/pipeline/document_source_skip.cpp @@ -72,8 +72,8 @@ DocumentSource::GetNextResult DocumentSourceSkip::doGetNext() { return pSource->getNext(); } -Value DocumentSourceSkip::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << _nToSkip)); +Value DocumentSourceSkip::serialize(const SerializationOptions& opts) const { + return Value(DOC(getSourceName() << opts.serializeLiteral(_nToSkip))); } intrusive_ptr<DocumentSource> DocumentSourceSkip::optimize() { diff --git a/src/mongo/db/pipeline/document_source_skip.h b/src/mongo/db/pipeline/document_source_skip.h index 89dbeb95b11..667ab7cd85b 100644 --- a/src/mongo/db/pipeline/document_source_skip.h +++ b/src/mongo/db/pipeline/document_source_skip.h @@ -73,7 +73,7 @@ public: Pipeline::SourceContainer::iterator doOptimizeAt(Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<DocumentSource> optimize() final; diff --git a/src/mongo/db/pipeline/document_source_skip_test.cpp b/src/mongo/db/pipeline/document_source_skip_test.cpp index 1de0df02324..7e54cbdde7c 100644 --- a/src/mongo/db/pipeline/document_source_skip_test.cpp +++ b/src/mongo/db/pipeline/document_source_skip_test.cpp @@ -103,5 +103,12 @@ TEST_F(DocumentSourceSkipTest, SkipsChainedTogetherShouldNotOverFlowWhenOptimizi ASSERT_EQUALS(containerOptimized.size(), 1U); ASSERT_EQUALS(skipFirst->getSkip(), 2); } + +TEST_F(DocumentSourceSkipTest, Redaction) { + auto stage = DocumentSourceSkip::create(getExpCtx(), 1337); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$skip":"?number"})", + redact(*stage)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_sort.cpp b/src/mongo/db/pipeline/document_source_sort.cpp index 84b28d6fde9..235f98039df 100644 --- a/src/mongo/db/pipeline/document_source_sort.cpp +++ b/src/mongo/db/pipeline/document_source_sort.cpp @@ -44,6 +44,7 @@ #include "mongo/db/pipeline/lite_parsed_document_source.h" #include "mongo/db/pipeline/skip_and_limit.h" #include "mongo/db/query/collation/collation_index_key.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/db/stats/resource_consumption_metrics.h" #include "mongo/logv2/log.h" #include "mongo/platform/overflow_arithmetic.h" @@ -68,10 +69,10 @@ struct BoundMakerMin { doc.metadata().getTimeseriesBucketMinTime().toMillisSinceEpoch() + offset)}; } - Document serialize() const { + Document serialize(const SerializationOptions& opts) const { // Convert from millis to seconds. return Document{{{"base"_sd, DocumentSourceSort::kMin}, - {DocumentSourceSort::kOffset, (offset / 1000)}}}; + {DocumentSourceSort::kOffset, opts.serializeLiteral(offset / 1000)}}}; } }; @@ -84,10 +85,10 @@ struct BoundMakerMax { doc.metadata().getTimeseriesBucketMaxTime().toMillisSinceEpoch() + offset)}; } - Document serialize() const { + Document serialize(const SerializationOptions& opts) const { // Convert from millis to seconds. return Document{{{"base"_sd, DocumentSourceSort::kMax}, - {DocumentSourceSort::kOffset, (offset / 1000)}}}; + {DocumentSourceSort::kOffset, opts.serializeLiteral(offset / 1000)}}}; } }; struct CompAsc { @@ -286,8 +287,10 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceSort::clone( _sortExecutor->getMaxMemoryBytes()); } -void DocumentSourceSort::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { +void DocumentSourceSort::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { + auto explain = opts.verbosity; + if (_timeSorter) { tassert(6369900, "$_internalBoundedSort should not absorb a $limit", @@ -295,20 +298,22 @@ void DocumentSourceSort::serializeToArray( // {$_internalBoundedSort: {sortKey, bound}} auto sortKey = _sortExecutor->sortPattern().serialize( - SortPattern::SortKeySerialization::kForPipelineSerialization); + SortPattern::SortKeySerialization::kForPipelineSerialization, opts); MutableDocument mutDoc{Document{{ {"$_internalBoundedSort"_sd, Document{{{"sortKey"_sd, std::move(sortKey)}, - {"bound"_sd, _timeSorter->serializeBound()}, - {"limit"_sd, static_cast<long long>(_timeSorter->limit())}}}}, + {"bound"_sd, _timeSorter->serializeBound(opts)}, + {"limit"_sd, + opts.serializeLiteral(static_cast<long long>(_timeSorter->limit()))}}}}, }}}; if (explain >= ExplainOptions::Verbosity::kExecStats) { mutDoc["totalDataSizeSortedBytesEstimate"] = - Value(static_cast<long long>(_timeSorter->totalDataSizeBytes())); - mutDoc["usedDisk"] = Value(_timeSorter->stats().spilledRanges() > 0); - mutDoc["spills"] = Value(static_cast<long long>(_timeSorter->stats().spilledRanges())); + opts.serializeLiteral(static_cast<long long>(_timeSorter->totalDataSizeBytes())); + mutDoc["usedDisk"] = opts.serializeLiteral(_timeSorter->stats().spilledRanges() > 0); + mutDoc["spills"] = + opts.serializeLiteral(static_cast<long long>(_timeSorter->stats().spilledRanges())); } array.push_back(Value{mutDoc.freeze()}); @@ -319,31 +324,31 @@ void DocumentSourceSort::serializeToArray( if (!explain) { // one Value for $sort and maybe a Value for $limit MutableDocument inner(_sortExecutor->sortPattern().serialize( - SortPattern::SortKeySerialization::kForPipelineSerialization)); + SortPattern::SortKeySerialization::kForPipelineSerialization, opts)); array.push_back(Value(DOC(kStageName << inner.freeze()))); if (_sortExecutor->hasLimit()) { auto limitSrc = DocumentSourceLimit::create(pExpCtx, limit); - limitSrc->serializeToArray(array); + limitSrc->serializeToArray(array, opts); } return; } - MutableDocument mutDoc( - DOC(kStageName << DOC("sortKey" - << _sortExecutor->sortPattern().serialize( - SortPattern::SortKeySerialization::kForExplain) - << "limit" - << (_sortExecutor->hasLimit() ? Value(static_cast<long long>(limit)) - : Value())))); + MutableDocument mutDoc(DOC( + kStageName << DOC("sortKey" << _sortExecutor->sortPattern().serialize( + SortPattern::SortKeySerialization::kForExplain, opts) + << "limit" + << (_sortExecutor->hasLimit() + ? opts.serializeLiteral(static_cast<long long>(limit)) + : Value())))); if (explain >= ExplainOptions::Verbosity::kExecStats) { auto& stats = _sortExecutor->stats(); mutDoc["totalDataSizeSortedBytesEstimate"] = - Value(static_cast<long long>(stats.totalDataSizeBytes)); - mutDoc["usedDisk"] = Value(stats.spills > 0); - mutDoc["spills"] = Value(static_cast<long long>(stats.spills)); + opts.serializeLiteral(static_cast<long long>(stats.totalDataSizeBytes)); + mutDoc["usedDisk"] = opts.serializeLiteral(stats.spills > 0); + mutDoc["spills"] = opts.serializeLiteral(static_cast<long long>(stats.spills)); } array.push_back(Value(mutDoc.freeze())); diff --git a/src/mongo/db/pipeline/document_source_sort.h b/src/mongo/db/pipeline/document_source_sort.h index ba087ee4a3f..d28e29a77ea 100644 --- a/src/mongo/db/pipeline/document_source_sort.h +++ b/src/mongo/db/pipeline/document_source_sort.h @@ -72,7 +72,7 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<DocumentSource> clone( const boost::intrusive_ptr<ExpressionContext>& newExpCtx) const final; @@ -201,8 +201,8 @@ private: uint64_t limit, uint64_t maxMemoryUsageBytes); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { - MONGO_UNREACHABLE; // Should call serializeToArray instead. + Value serialize(const SerializationOptions& opts) const final override { + MONGO_UNREACHABLE_TASSERT(7484302); // Should call serializeToArray instead. } /** diff --git a/src/mongo/db/pipeline/document_source_sort_by_count_test.cpp b/src/mongo/db/pipeline/document_source_sort_by_count_test.cpp index d4ab7f15c55..df92e9b5e07 100644 --- a/src/mongo/db/pipeline/document_source_sort_by_count_test.cpp +++ b/src/mongo/db/pipeline/document_source_sort_by_count_test.cpp @@ -70,7 +70,8 @@ public: // Serialize the DocumentSourceGroup and DocumentSourceSort from $sortByCount so that we can // check the explain output to make sure $group and $sort have the correct fields. - const auto explain = ExplainOptions::Verbosity::kQueryPlanner; + const auto explain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; vector<Value> explainedStages; groupStage->serializeToArray(explainedStages, explain); sortStage->serializeToArray(explainedStages, explain); diff --git a/src/mongo/db/pipeline/document_source_sort_test.cpp b/src/mongo/db/pipeline/document_source_sort_test.cpp index 013ef9b6a53..31c6c9323a5 100644 --- a/src/mongo/db/pipeline/document_source_sort_test.cpp +++ b/src/mongo/db/pipeline/document_source_sort_test.cpp @@ -27,6 +27,7 @@ * it in the license file. */ +#include "mongo/db/query/explain_options.h" #include "mongo/platform/basic.h" #include <boost/intrusive_ptr.hpp> @@ -538,5 +539,79 @@ TEST_F(DocumentSourceSortExecutionTest, ShouldCorrectlyTrackMemoryUsageBetweenPa sort->getNext(), AssertionException, ErrorCodes::QueryExceededMemoryLimitNoDiskUseAllowed); } +TEST_F(DocumentSourceSortTest, Redaction) { + createSort(BSON("a" << 1)); + auto boundedSort = DocumentSourceSort::createBoundedSort( + sort()->getSortKeyPattern(), DocumentSourceSort::kMin, 1337, 10, getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"$sort":{"HASH<a>":1}})", + redact(*sort(), true)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalBoundedSort": { + "sortKey": { + "HASH<a>": 1 + }, + "bound": { + "base": "min", + "offsetSeconds": "?number" + }, + "limit": "?number" + } + })", + redact(*boundedSort, true)); + + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"$sort":{"sortKey":{"HASH<a>":1}}})", + redact(*sort(), true, ExplainOptions::Verbosity::kQueryPlanner)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalBoundedSort": { + "sortKey": { + "HASH<a>": 1 + }, + "bound": { + "base": "min", + "offsetSeconds": "?number" + }, + "limit": "?number" + } + })", + redact(*boundedSort, true, ExplainOptions::Verbosity::kQueryPlanner)); + + ASSERT_BSONOBJ_EQ_AUTO( // + R"({ + "$sort": { + "sortKey": { + "HASH<a>": 1 + } + }, + "totalDataSizeSortedBytesEstimate": "?number", + "usedDisk": "?bool", + "spills": "?number" + })", + redact(*sort(), true, ExplainOptions::Verbosity::kExecStats)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalBoundedSort": { + "sortKey": { + "HASH<a>": 1 + }, + "bound": { + "base": "min", + "offsetSeconds": "?number" + }, + "limit": "?number" + }, + "totalDataSizeSortedBytesEstimate": "?number", + "usedDisk": "?bool", + "spills": "?number" + })", + redact(*boundedSort, true, ExplainOptions::Verbosity::kExecStats)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_streaming_group.cpp b/src/mongo/db/pipeline/document_source_streaming_group.cpp index 44702ab550d..0aeae62201a 100644 --- a/src/mongo/db/pipeline/document_source_streaming_group.cpp +++ b/src/mongo/db/pipeline/document_source_streaming_group.cpp @@ -153,13 +153,13 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceStreamingGroup::createFromBso } void DocumentSourceStreamingGroup::serializeAdditionalFields( - MutableDocument& out, boost::optional<ExplainOptions::Verbosity> explain) const { + MutableDocument& out, const SerializationOptions& opts) const { std::vector<Value> monotonicIdFields; if (_idFieldNames.empty()) { - monotonicIdFields.emplace_back("_id"_sd); + monotonicIdFields.emplace_back(opts.serializeFieldPath("_id")); } else { for (size_t i : _monotonicExpressionIndexes) { - monotonicIdFields.emplace_back(_idFieldNames[i]); + monotonicIdFields.emplace_back(opts.serializeFieldPathFromString(_idFieldNames[i])); } } out[kMonotonicIdFieldsSpecField] = Value(std::move(monotonicIdFields)); diff --git a/src/mongo/db/pipeline/document_source_streaming_group.h b/src/mongo/db/pipeline/document_source_streaming_group.h index bd72a3a7e7d..e69db4ef026 100644 --- a/src/mongo/db/pipeline/document_source_streaming_group.h +++ b/src/mongo/db/pipeline/document_source_streaming_group.h @@ -90,8 +90,9 @@ protected: GetNextResult doGetNext() final; bool isSpecFieldReserved(StringData fieldName) final; - void serializeAdditionalFields(MutableDocument& out, - boost::optional<ExplainOptions::Verbosity> explain) const final; + void serializeAdditionalFields( + MutableDocument& out, + const SerializationOptions& opts = SerializationOptions{}) const final; private: static constexpr StringData kMonotonicIdFieldsSpecField = "$monotonicIdFields"_sd; diff --git a/src/mongo/db/pipeline/document_source_tee_consumer.cpp b/src/mongo/db/pipeline/document_source_tee_consumer.cpp index 739cec3498c..47539fb491c 100644 --- a/src/mongo/db/pipeline/document_source_tee_consumer.cpp +++ b/src/mongo/db/pipeline/document_source_tee_consumer.cpp @@ -71,9 +71,8 @@ void DocumentSourceTeeConsumer::doDispose() { _bufferSource->dispose(_facetId); } -Value DocumentSourceTeeConsumer::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceTeeConsumer::serialize(const SerializationOptions& opts) const { // We only serialize this stage in the context of explain. - return explain ? Value(DOC(_stageName << Document())) : Value(); + return opts.verbosity ? Value(DOC(_stageName << Document())) : Value(); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_tee_consumer.h b/src/mongo/db/pipeline/document_source_tee_consumer.h index da56c76a37b..26a8ee02a2f 100644 --- a/src/mongo/db/pipeline/document_source_tee_consumer.h +++ b/src/mongo/db/pipeline/document_source_tee_consumer.h @@ -79,7 +79,7 @@ public: const char* getSourceName() const override; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; protected: GetNextResult doGetNext() final; diff --git a/src/mongo/db/pipeline/document_source_test_optimizations.h b/src/mongo/db/pipeline/document_source_test_optimizations.h index 2f6e484be10..d7f8f4bec13 100644 --- a/src/mongo/db/pipeline/document_source_test_optimizations.h +++ b/src/mongo/db/pipeline/document_source_test_optimizations.h @@ -71,8 +71,9 @@ public: } private: - virtual Value serialize(boost::optional<ExplainOptions::Verbosity>) const override { - MONGO_UNREACHABLE; + virtual Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { + MONGO_UNREACHABLE_TASSERT(7484301); } }; diff --git a/src/mongo/db/pipeline/document_source_union_with.cpp b/src/mongo/db/pipeline/document_source_union_with.cpp index 2f08fdfc0c6..fdf1f724787 100644 --- a/src/mongo/db/pipeline/document_source_union_with.cpp +++ b/src/mongo/db/pipeline/document_source_union_with.cpp @@ -51,21 +51,21 @@ REGISTER_DOCUMENT_SOURCE(unionWith, AllowedWithApiStrict::kAlways); namespace { +void validatorCallback(const Pipeline& pipeline) { + const auto& sources = pipeline.getSources(); + std::for_each(sources.begin(), sources.end(), [](auto& src) { + uassert(31441, + str::stream() << src->getSourceName() + << " is not allowed within a $unionWith's sub-pipeline", + src->constraints().isAllowedInUnionPipeline()); + }); +} + std::unique_ptr<Pipeline, PipelineDeleter> buildPipelineFromViewDefinition( const boost::intrusive_ptr<ExpressionContext>& expCtx, ExpressionContext::ResolvedNamespace resolvedNs, std::vector<BSONObj> currentPipeline) { - auto validatorCallback = [](const Pipeline& pipeline) { - const auto& sources = pipeline.getSources(); - std::for_each(sources.begin(), sources.end(), [](auto& src) { - uassert(31441, - str::stream() << src->getSourceName() - << " is not allowed within a $unionWith's sub-pipeline", - src->constraints().isAllowedInUnionPipeline()); - }); - }; - MakePipelineOptions opts; opts.attachCursorSource = false; // Only call optimize() here if we actually have a pipeline to resolve in the view definition. @@ -78,6 +78,22 @@ std::unique_ptr<Pipeline, PipelineDeleter> buildPipelineFromViewDefinition( } // namespace +DocumentSourceUnionWith::DocumentSourceUnionWith( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + std::unique_ptr<Pipeline, PipelineDeleter> pipeline) + : DocumentSource(kStageName, expCtx), _pipeline(std::move(pipeline)) {} + +DocumentSourceUnionWith::DocumentSourceUnionWith( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + NamespaceString unionNss, + std::vector<BSONObj> pipeline) + : DocumentSourceUnionWith(expCtx, + buildPipelineFromViewDefinition( + expCtx, expCtx->getResolvedNamespace(unionNss), pipeline)) { + _userNss = std::move(unionNss); + _userPipeline = std::move(pipeline); +} + DocumentSourceUnionWith::~DocumentSourceUnionWith() { if (_pipeline && _pipeline->getContext()->explain) { _pipeline->dispose(pExpCtx->opCtx); @@ -197,9 +213,7 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceUnionWith::createFromBson( pipeline = unionWithSpec.getPipeline().value_or(std::vector<BSONObj>{}); } return make_intrusive<DocumentSourceUnionWith>( - expCtx, - buildPipelineFromViewDefinition( - expCtx, expCtx->getResolvedNamespace(std::move(unionNss)), std::move(pipeline))); + expCtx, std::move(unionNss), std::move(pipeline)); } DocumentSource::GetNextResult DocumentSourceUnionWith::doGetNext() { @@ -278,11 +292,7 @@ Pipeline::SourceContainer::iterator DocumentSourceUnionWith::doOptimizeAt( _pipeline->addFinalSource(nextStage->clone(_pipeline->getContext())); // Apply the same rewrite to the cached pipeline if available. if (pExpCtx->explain >= ExplainOptions::Verbosity::kExecStats) { - auto cloneForExplain = nextStage->clone(_pipeline->getContext()); - if (!_cachedPipeline.empty()) { - cloneForExplain->setSource(_cachedPipeline.back().get()); - } - _cachedPipeline.push_back(std::move(cloneForExplain)); + _pushedDownStages.push_back(nextStage->serialize().getDocument().toBson()); } auto newStageItr = container->insert(itr, std::move(nextStage)); container->erase(std::next(itr)); @@ -315,14 +325,16 @@ void DocumentSourceUnionWith::doDispose() { if (!_pipeline->getContext()->explain) { _pipeline->dispose(pExpCtx->opCtx); + _userPipeline.clear(); + _pushedDownStages.clear(); _pipeline.reset(); } } } -Value DocumentSourceUnionWith::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceUnionWith::serialize(const SerializationOptions& opts) const { auto collectionless = _pipeline->getContext()->ns.isCollectionlessAggregateNS(); - if (explain) { + if (opts.verbosity) { // There are several different possible states depending on the explain verbosity as well as // the other stages in the pipeline: // * If verbosity is queryPlanner, then the sub-pipeline should be untouched and we can @@ -332,44 +344,65 @@ Value DocumentSourceUnionWith::serialize(boost::optional<ExplainOptions::Verbosi // $limit stage after the $unionWith which results in only reading from the base collection // branch and not the sub-pipeline. Pipeline* pipeCopy = nullptr; - if (*explain == ExplainOptions::Verbosity::kQueryPlanner) { + if (*opts.verbosity == ExplainOptions::Verbosity::kQueryPlanner) { pipeCopy = Pipeline::create(_pipeline->getSources(), _pipeline->getContext()).release(); - } else if (*explain >= ExplainOptions::Verbosity::kExecStats && + } else if (*opts.verbosity >= ExplainOptions::Verbosity::kExecStats && _executionState > ExecutionProgress::kIteratingSource) { + std::vector<BSONObj> recoveredPipeline; // We've either exhausted the sub-pipeline or at least started iterating it. Use the - // cached pipeline to get the explain output since the '_pipeline' may have been - // modified for any optimizations or pushdowns into the initial $cursor stage. - pipeCopy = Pipeline::create(_cachedPipeline, _pipeline->getContext()).release(); + // cached user pipeline and pushed down stages to get the explain output since the + // '_pipeline' may have been modified for any optimizations or pushdowns into the + // initial $cursor stage. + recoveredPipeline.reserve(_userPipeline.size() + _pushedDownStages.size()); + std::move( + _userPipeline.begin(), _userPipeline.end(), std::back_inserter(recoveredPipeline)); + std::move(_pushedDownStages.begin(), + _pushedDownStages.end(), + std::back_inserter(recoveredPipeline)); + pipeCopy = + Pipeline::parse(recoveredPipeline, _pipeline->getContext(), validatorCallback) + .release(); } else { // The plan does not require reading from the sub-pipeline, so just include the // serialization in the explain output. BSONArrayBuilder bab; - for (auto&& stage : _pipeline->serialize(explain)) + for (auto&& stage : _pipeline->serialize(opts)) bab << stage; auto spec = collectionless ? DOC("pipeline" << bab.arr()) - : DOC("coll" << _pipeline->getContext()->ns.coll() << "pipeline" << bab.arr()); + : DOC("coll" << opts.serializeIdentifier(_pipeline->getContext()->ns.coll()) + << "pipeline" << bab.arr()); return Value(DOC(getSourceName() << spec)); } invariant(pipeCopy); BSONObj explainLocal = - pExpCtx->mongoProcessInterface->preparePipelineAndExplain(pipeCopy, *explain); + pExpCtx->mongoProcessInterface->preparePipelineAndExplain(pipeCopy, *opts.verbosity); LOGV2_DEBUG(4553501, 3, "$unionWith attached cursor to pipeline for explain"); // We expect this to be an explanation of a pipeline -- there should only be one field. invariant(explainLocal.nFields() == 1); - auto spec = collectionless ? DOC("pipeline" << explainLocal.firstElement()) - : DOC("coll" << _pipeline->getContext()->ns.coll() << "pipeline" - << explainLocal.firstElement()); + auto spec = collectionless + ? DOC("pipeline" << explainLocal.firstElement()) + : DOC("coll" << opts.serializeIdentifier(_pipeline->getContext()->ns.coll()) + << "pipeline" << explainLocal.firstElement()); return Value(DOC(getSourceName() << spec)); } else { - BSONArrayBuilder bab; - for (auto&& stage : _pipeline->serialize()) - bab << stage; - auto spec = collectionless - ? DOC("pipeline" << bab.arr()) - : DOC("coll" << _pipeline->getContext()->ns.coll() << "pipeline" << bab.arr()); + // Query shapes must reflect the original, unresolved and unoptimized pipeline, so we need a + // special case here if we are serializing the stage for that purpose. Otherwise, we should + // return the current (optimized) pipeline for introspection with explain, etc. + auto serializedPipeline = [&]() -> std::vector<BSONObj> { + if (opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + return Pipeline::parse(_userPipeline, _pipeline->getContext(), validatorCallback) + ->serializeToBson(opts); + } + return _pipeline->serializeToBson(opts); + }(); + + auto spec = collectionless ? DOC("pipeline" << serializedPipeline) + : DOC("coll" << opts.serializeIdentifier(_userNss.coll()) + << "pipeline" << serializedPipeline); return Value(DOC(getSourceName() << spec)); } } diff --git a/src/mongo/db/pipeline/document_source_union_with.h b/src/mongo/db/pipeline/document_source_union_with.h index 741f2e13e68..9d9f9aae9c8 100644 --- a/src/mongo/db/pipeline/document_source_union_with.h +++ b/src/mongo/db/pipeline/document_source_union_with.h @@ -61,18 +61,19 @@ public: }; DocumentSourceUnionWith(const boost::intrusive_ptr<ExpressionContext>& expCtx, - std::unique_ptr<Pipeline, PipelineDeleter> pipeline) - : DocumentSource(kStageName, expCtx), _pipeline(std::move(pipeline)) { - // If this pipeline is being run as part of explain, then cache a copy to use later during - // serialization. - if (expCtx->explain >= ExplainOptions::Verbosity::kExecStats) { - _cachedPipeline = _pipeline->getSources(); - } - } + NamespaceString unionNss, + std::vector<BSONObj> pipeline); + + // Expose a constructor that skips the parsing step for testing purposes. + DocumentSourceUnionWith(const boost::intrusive_ptr<ExpressionContext>& expCtx, + std::unique_ptr<Pipeline, PipelineDeleter> pipeline); DocumentSourceUnionWith(const DocumentSourceUnionWith& original, const boost::intrusive_ptr<ExpressionContext>& newExpCtx) - : DocumentSource(kStageName, newExpCtx), _pipeline(original._pipeline->clone()) {} + : DocumentSource(kStageName, newExpCtx), + _pipeline(original._pipeline->clone()), + _userNss(original._userNss), + _userPipeline(original._userPipeline) {} ~DocumentSourceUnionWith(); @@ -180,7 +181,7 @@ private: kFinished }; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; void addViewDefinition(NamespaceString nss, std::vector<BSONObj> viewPipeline); @@ -189,7 +190,15 @@ private: const ExceptionFor<ErrorCodes::CommandOnShardedViewNotSupportedOnMongod>& e); std::unique_ptr<Pipeline, PipelineDeleter> _pipeline; - Pipeline::SourceContainer _cachedPipeline; + // The original, unresolved namespace to union. + NamespaceString _userNss; + // The aggregation pipeline defined with the user request, prior to optimization and view + // resolution. + std::vector<BSONObj> _userPipeline; + // Match and/or project stages after a $unionWith can be pushed down into the $unionWith (and + // the head of the pipeline). If we're doing an explain with execution stats, we will need + // copies of these stages as they may be pushed down to the find layer. + std::vector<BSONObj> _pushedDownStages; ExecutionProgress _executionState = ExecutionProgress::kIteratingSource; UnionWithStats _stats; }; diff --git a/src/mongo/db/pipeline/document_source_union_with.idl b/src/mongo/db/pipeline/document_source_union_with.idl index 77b3ea64a82..11bbaa499dc 100644 --- a/src/mongo/db/pipeline/document_source_union_with.idl +++ b/src/mongo/db/pipeline/document_source_union_with.idl @@ -31,6 +31,7 @@ global: imports: - "mongo/idl/basic_types.idl" + - "mongo/db/pipeline/aggregate_command.idl" structs: UnionWithSpec: @@ -44,4 +45,4 @@ structs: pipeline: description: An optional pipeline to apply to the collection being unioned. optional: true - type: array<object> + type: pipeline diff --git a/src/mongo/db/pipeline/document_source_union_with_test.cpp b/src/mongo/db/pipeline/document_source_union_with_test.cpp index 2f2b819174d..6dff25c53b1 100644 --- a/src/mongo/db/pipeline/document_source_union_with_test.cpp +++ b/src/mongo/db/pipeline/document_source_union_with_test.cpp @@ -572,5 +572,62 @@ TEST_F(DocumentSourceUnionWithTest, StricterConstraintsFromSubSubPipelineAreInhe StageConstraints::UnionRequirement::kAllowed); ASSERT_TRUE(unionStage.constraints(Pipeline::SplitState::kUnsplit) == expectedConstraints); } + +TEST_F(DocumentSourceUnionWithTest, RedactsCorrectlyBasic) { + auto expCtx = getExpCtx(); + NamespaceString nsToUnionWith(expCtx->ns.db(), "coll"); + expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {nsToUnionWith.coll().toString(), {nsToUnionWith, std::vector<BSONObj>()}}}); + + auto docSource = DocumentSourceUnionWith::createFromBson( + BSON("$unionWith" << nsToUnionWith.coll()).firstElement(), expCtx); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$unionWith": { + "coll": "HASH<coll>", + "pipeline": [] + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceUnionWithTest, RedactsCorrectlyWithPipeline) { + auto expCtx = getExpCtx(); + NamespaceString nsToUnionWith(expCtx->ns.db(), "coll"); + expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {nsToUnionWith.coll().toString(), {nsToUnionWith, std::vector<BSONObj>()}}}); + + BSONArrayBuilder pipeline; + pipeline << BSON("$match" << BSON("a" << 15)); + pipeline << BSON("$project" << BSON("a" << 1 << "b" << 1)); + auto docSource = DocumentSourceUnionWith::createFromBson( + BSON("$unionWith" << BSON("coll" << nsToUnionWith.coll() << "pipeline" << pipeline.arr())) + .firstElement(), + expCtx); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$unionWith": { + "coll": "HASH<coll>", + "pipeline": [ + { + "$match": { + "HASH<a>": { + "$eq": "?number" + } + } + }, + { + "$project": { + "HASH<_id>": true, + "HASH<a>": true, + "HASH<b>": true + } + } + ] + } + })", + redact(*docSource)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_unwind.cpp b/src/mongo/db/pipeline/document_source_unwind.cpp index 2161b0b9bbb..45ba4aed308 100644 --- a/src/mongo/db/pipeline/document_source_unwind.cpp +++ b/src/mongo/db/pipeline/document_source_unwind.cpp @@ -91,6 +91,10 @@ private: // Index into the _inputArray to return next. size_t _index = 0; + + // True if we are including the array index and it's path is a parent of the unwind path. If + // this is true, we will just return the array indices and ignore the array values. + bool _conflictingPaths; }; DocumentSourceUnwind::Unwinder::Unwinder(const FieldPath& unwindPath, @@ -100,7 +104,8 @@ DocumentSourceUnwind::Unwinder::Unwinder(const FieldPath& unwindPath, : _unwindPath(unwindPath), _preserveNullAndEmptyArrays(preserveNullAndEmptyArrays), _indexPath(indexPath), - _strict(strict) {} + _strict(strict), + _conflictingPaths(indexPath ? indexPath->isPrefixOf(unwindPath) : false) {} void DocumentSourceUnwind::Unwinder::resetDocument(const Document& document) { // Reset document specific attributes. @@ -141,8 +146,11 @@ DocumentSource::GetNextResult DocumentSourceUnwind::Unwinder::getNext() { // across documents that have come out of this pipeline operator. This is a partial deep // clone. Because the value at the end will be replaced, everything along the path // leading to that will be replaced in order not to share that change with any other - // clones (or the original). - _output.setNestedField(_unwindPathFieldIndexes, _inputArray[_index]); + // clones (or the original). If the array index path is a parent of the unwind path, we + // ignore the array value since it would be overwritten by the index. + if (!_conflictingPaths) { + _output.setNestedField(_unwindPathFieldIndexes, _inputArray[_index]); + } indexForOutput = _index; _index++; _haveNext = _index < length; @@ -232,19 +240,16 @@ bool DocumentSourceUnwind::canPushSortBack(const DocumentSourceSort* sort) const // otherwise when we swap the limit and unwind, we could end up providing fewer results to the // user than expected. if (!sort->hasLimit() || _preserveNullAndEmptyArrays) { - auto unwindPath = _unwindPath.fullPath(); + auto modifiedPaths = getModifiedPaths(); // Checks if any of the $sort's paths depend on the unwind path (or vice versa). SortPattern sortKeyPattern = sort->getSortKeyPattern(); - bool sortPathMatchesUnwindPath = + bool sortDependsOnUnwind = std::any_of(sortKeyPattern.begin(), sortKeyPattern.end(), [&](auto& sortKey) { // If 'sortKey' is a $meta expression, we can do the swap. - if (!sortKey.fieldPath) - return false; - return expression::bidirectionalPathPrefixOf(unwindPath, - sortKey.fieldPath->fullPath()); + return sortKey.fieldPath && modifiedPaths.canModify(*sortKey.fieldPath); }); - return !sortPathMatchesUnwindPath; + return !sortDependsOnUnwind; } return false; } @@ -293,12 +298,13 @@ Pipeline::SourceContainer::iterator DocumentSourceUnwind::doOptimizeAt( return std::next(itr); } -Value DocumentSourceUnwind::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << DOC( - "path" << _unwindPath.fullPathWithPrefix() << "preserveNullAndEmptyArrays" - << (_preserveNullAndEmptyArrays ? Value(true) : Value()) - << "includeArrayIndex" - << (_indexPath ? Value((*_indexPath).fullPath()) : Value())))); +Value DocumentSourceUnwind::serialize(const SerializationOptions& opts) const { + return Value(DOC( + getSourceName() << DOC( + "path" << opts.serializeFieldPathWithPrefix(_unwindPath) << "preserveNullAndEmptyArrays" + << (_preserveNullAndEmptyArrays ? opts.serializeLiteral(true) : Value()) + << "includeArrayIndex" + << (_indexPath ? Value(opts.serializeFieldPath(*_indexPath)) : Value())))); } DepsTracker::State DocumentSourceUnwind::getDependencies(DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/document_source_unwind.h b/src/mongo/db/pipeline/document_source_unwind.h index d7f401dd99c..81a2fb58882 100644 --- a/src/mongo/db/pipeline/document_source_unwind.h +++ b/src/mongo/db/pipeline/document_source_unwind.h @@ -43,7 +43,7 @@ public: // virtuals from DocumentSource const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Returns the unwound path, and the 'includeArrayIndex' path, if specified. diff --git a/src/mongo/db/pipeline/document_source_unwind_test.cpp b/src/mongo/db/pipeline/document_source_unwind_test.cpp index aa8a550ec06..c8c7b46f0c3 100644 --- a/src/mongo/db/pipeline/document_source_unwind_test.cpp +++ b/src/mongo/db/pipeline/document_source_unwind_test.cpp @@ -734,6 +734,61 @@ TEST_F(UnwindStageTest, UnwindIncludesIndexPathWhenIncludingIndex) { ASSERT_EQUALS(1U, modifiedPaths.paths.count("arrIndex")); } +TEST_F(UnwindStageTest, UnwindIndexPathIsSamePathAsArrayPath) { + const bool includeNullIfEmptyOrMissing = false; + const boost::optional<std::string> includeArrayIndex = std::string("array"); + auto unwind = DocumentSourceUnwind::create( + getExpCtx(), "array", includeNullIfEmptyOrMissing, includeArrayIndex); + auto source = DocumentSourceMock::createForTest( + {Document{{"array", vector<Value>{Value(10), Value(20)}}}, + Document{{"array", vector<Value>{Value(30), Value(40)}}}}, + getExpCtx()); + + unwind->setSource(source.get()); + + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(0)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(1)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(0)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(1)); +} + +TEST_F(UnwindStageTest, UnwindIndexPathIsParentOfArrayPath) { + const bool includeNullIfEmptyOrMissing = false; + const boost::optional<std::string> includeArrayIndex = std::string("obj"); + auto unwind = DocumentSourceUnwind::create( + getExpCtx(), "obj.array", includeNullIfEmptyOrMissing, includeArrayIndex); + auto source = DocumentSourceMock::createForTest( + {Document{{"obj", Document{{"array", vector<Value>{Value(10), Value(20)}}}}}, + Document{{"obj", Document{{"array", vector<Value>{Value(30), Value(40)}}}}}}, + getExpCtx()); + + unwind->setSource(source.get()); + + Document res; + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["obj"], Value(0)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["obj"], Value(1)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["obj"], Value(0)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["obj"], Value(1)); +} + +TEST_F(UnwindStageTest, UnwindIndexPathIsChildOfArrayPath) { + const bool includeNullIfEmptyOrMissing = false; + const boost::optional<std::string> includeArrayIndex = std::string("array.index"); + auto unwind = DocumentSourceUnwind::create( + getExpCtx(), "array", includeNullIfEmptyOrMissing, includeArrayIndex); + auto source = DocumentSourceMock::createForTest( + {Document{{"array", vector<Value>{Value(10), Value(20)}}}, + Document{{"array", vector<Value>{Value(30), Value(40)}}}}, + getExpCtx()); + + unwind->setSource(source.get()); + + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(BSON("index" << 0))); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(BSON("index" << 1))); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(BSON("index" << 0))); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(BSON("index" << 1))); +} + // // Error cases. // @@ -816,6 +871,26 @@ TEST_F(UnwindStageTest, ShouldRejectUnrecognizedOption) { 28811); } +TEST_F(UnwindStageTest, Redaction) { + auto spec = fromjson(R"({ + $unwind: { + path: "$foo.bar", + includeArrayIndex: "foo.baz", + preserveNullAndEmptyArrays: true + } + })"); + auto docSource = DocumentSourceUnwind::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$unwind": { + "path": "$HASH<foo>.HASH<bar>", + "preserveNullAndEmptyArrays": "?bool", + "includeArrayIndex": "HASH<foo>.HASH<baz>" + } + })", + redact(*docSource)); +} + class All : public OldStyleSuiteSpecification { public: All() : OldStyleSuiteSpecification("DocumentSourceUnwindTests") {} diff --git a/src/mongo/db/pipeline/exchange_spec.idl b/src/mongo/db/pipeline/exchange_spec.idl index 5f07f822693..0bc67b96d20 100644 --- a/src/mongo/db/pipeline/exchange_spec.idl +++ b/src/mongo/db/pipeline/exchange_spec.idl @@ -46,25 +46,30 @@ enums: structs: ExchangeSpec: description: "exchange aggregation request specification" + query_shape_component: true fields: policy: type: ExchangePolicy description: A string indicating a policy of how documents are distributed to consumers. unstable: false + query_shape: parameter consumers: type: int description: Number of consumers. unstable: false + query_shape: literal orderPreserving: type: bool default: false description: A flag indicating documents are merged while preserving the order. unstable: false + query_shape: parameter bufferSize: type: int default: 16777216 description: The size of exchange buffers. unstable: false + query_shape: literal key: type: object default: "BSONObj()" @@ -74,14 +79,17 @@ structs: field listed here, or if any prefix of any path is multikey (i.e. an array is encountered while traversing a path listed here), then it is by definition sent to consumer 0. + query_shape: literal boundaries: type: array<object> optional: true description: Range/hash split points. unstable: false + query_shape: literal consumerIds: type: array<int> optional: true description: Mapping from a range index to a consumer id. unstable: false + query_shape: literal diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp index 2d4c9a61aaa..dc2aeeb7e0e 100644 --- a/src/mongo/db/pipeline/expression.cpp +++ b/src/mongo/db/pipeline/expression.cpp @@ -72,13 +72,25 @@ using std::pair; using std::string; using std::vector; -/// Helper function to easily wrap constants with $const. -static Value serializeConstant(Value val) { +Value ExpressionConstant::serializeConstant(const SerializationOptions& opts, + Value val, + bool wrapRepresentativeValue) { if (val.missing()) { return Value("$$REMOVE"_sd); } + // It's safer to wrap constants in $const when generating representative shapes to avoid + // ambiguity when re-parsing (SERVER-88296, SERVER-85376). However, we allow certain expressions + // to override this behavior in order to reduce shape verbosity if the expression takes many + // constant arguments (e.g. variadic expressions - SERVER-84159). + // Debug shapes never wrap constants in $const to reduce shape size (and because re-parsing + // support is not a consideration there). + if ((opts.literalPolicy == LiteralSerializationPolicy::kUnchanged) || + (wrapRepresentativeValue && + opts.literalPolicy == LiteralSerializationPolicy::kToRepresentativeParseableValue)) { + return Value(DOC("$const" << opts.serializeLiteral(val))); + } - return Value(DOC("$const" << val)); + return opts.serializeLiteral(val); } /* --------------------------- Expression ------------------------------ */ @@ -643,11 +655,16 @@ Value ExpressionArray::evaluate(const Document& root, Variables* variables) cons return Value(std::move(values)); } -Value ExpressionArray::serialize(bool explain) const { +Value ExpressionArray::serialize(const SerializationOptions& options) const { + if (options.literalPolicy != LiteralSerializationPolicy::kUnchanged && + selfAndChildrenAreConstant()) { + return ExpressionConstant::serializeConstant( + options, evaluate(Document{}, &(getExpressionContext()->variables))); + } vector<Value> expressions; expressions.reserve(_children.size()); for (auto&& expr : _children) { - expressions.push_back(expr->serialize(explain)); + expressions.push_back(expr->serialize(options)); } return Value(std::move(expressions)); } @@ -670,6 +687,15 @@ intrusive_ptr<Expression> ExpressionArray::optimize() { return this; } +bool ExpressionArray::selfAndChildrenAreConstant() const { + for (auto&& exprPointer : _children) { + if (!exprPointer->selfAndChildrenAreConstant()) { + return false; + } + } + return true; +} + const char* ExpressionArray::getOpName() const { // This should never be called, but is needed to inherit from ExpressionNary. return "$array"; @@ -970,11 +996,11 @@ Value ExpressionCoerceToBool::evaluate(const Document& root, Variables* variable return Value(false); } -Value ExpressionCoerceToBool::serialize(bool explain) const { +Value ExpressionCoerceToBool::serialize(const SerializationOptions& options) const { // When not explaining, serialize to an $and expression. When parsed, the $and expression // will be optimized back into a ExpressionCoerceToBool. - const char* name = explain ? "$coerceToBool" : "$and"; - return Value(DOC(name << DOC_ARRAY(pExpression->serialize(explain)))); + const char* name = options.verbosity ? "$coerceToBool" : "$and"; + return Value(DOC(name << DOC_ARRAY(pExpression->serialize(options)))); } /* ----------------------- ExpressionCompare --------------------------- */ @@ -1223,8 +1249,8 @@ Value ExpressionConstant::evaluate(const Document& root, Variables* variables) c return _value; } -Value ExpressionConstant::serialize(bool explain) const { - return serializeConstant(_value); +Value ExpressionConstant::serialize(const SerializationOptions& options) const { + return ExpressionConstant::serializeConstant(options, _value); } REGISTER_STABLE_EXPRESSION(const, ExpressionConstant::parse); @@ -1438,20 +1464,20 @@ intrusive_ptr<Expression> ExpressionDateFromParts::optimize() { return this; } -Value ExpressionDateFromParts::serialize(bool explain) const { +Value ExpressionDateFromParts::serialize(const SerializationOptions& options) const { return Value(Document{ {"$dateFromParts", - Document{{"year", _year ? _year->serialize(explain) : Value()}, - {"month", _month ? _month->serialize(explain) : Value()}, - {"day", _day ? _day->serialize(explain) : Value()}, - {"hour", _hour ? _hour->serialize(explain) : Value()}, - {"minute", _minute ? _minute->serialize(explain) : Value()}, - {"second", _second ? _second->serialize(explain) : Value()}, - {"millisecond", _millisecond ? _millisecond->serialize(explain) : Value()}, - {"isoWeekYear", _isoWeekYear ? _isoWeekYear->serialize(explain) : Value()}, - {"isoWeek", _isoWeek ? _isoWeek->serialize(explain) : Value()}, - {"isoDayOfWeek", _isoDayOfWeek ? _isoDayOfWeek->serialize(explain) : Value()}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}}}}); + Document{{"year", _year ? _year->serialize(options) : Value()}, + {"month", _month ? _month->serialize(options) : Value()}, + {"day", _day ? _day->serialize(options) : Value()}, + {"hour", _hour ? _hour->serialize(options) : Value()}, + {"minute", _minute ? _minute->serialize(options) : Value()}, + {"second", _second ? _second->serialize(options) : Value()}, + {"millisecond", _millisecond ? _millisecond->serialize(options) : Value()}, + {"isoWeekYear", _isoWeekYear ? _isoWeekYear->serialize(options) : Value()}, + {"isoWeek", _isoWeek ? _isoWeek->serialize(options) : Value()}, + {"isoDayOfWeek", _isoDayOfWeek ? _isoDayOfWeek->serialize(options) : Value()}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}}}}); } bool ExpressionDateFromParts::evaluateNumberWithDefault(const Document& root, @@ -1695,14 +1721,14 @@ intrusive_ptr<Expression> ExpressionDateFromString::optimize() { return this; } -Value ExpressionDateFromString::serialize(bool explain) const { +Value ExpressionDateFromString::serialize(const SerializationOptions& options) const { return Value( Document{{"$dateFromString", - Document{{"dateString", _dateString->serialize(explain)}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}, - {"format", _format ? _format->serialize(explain) : Value()}, - {"onNull", _onNull ? _onNull->serialize(explain) : Value()}, - {"onError", _onError ? _onError->serialize(explain) : Value()}}}}); + Document{{"dateString", _dateString->serialize(options)}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}, + {"format", _format ? _format->serialize(options) : Value()}, + {"onNull", _onNull ? _onNull->serialize(options) : Value()}, + {"onError", _onError ? _onError->serialize(options) : Value()}}}}); } Value ExpressionDateFromString::evaluate(const Document& root, Variables* variables) const { @@ -1853,12 +1879,12 @@ intrusive_ptr<Expression> ExpressionDateToParts::optimize() { return this; } -Value ExpressionDateToParts::serialize(bool explain) const { +Value ExpressionDateToParts::serialize(const SerializationOptions& options) const { return Value( Document{{"$dateToParts", - Document{{"date", _date->serialize(explain)}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}, - {"iso8601", _iso8601 ? _iso8601->serialize(explain) : Value()}}}}); + Document{{"date", _date->serialize(options)}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}, + {"iso8601", _iso8601 ? _iso8601->serialize(options) : Value()}}}}); } boost::optional<int> ExpressionDateToParts::evaluateIso8601Flag(const Document& root, @@ -2011,13 +2037,13 @@ intrusive_ptr<Expression> ExpressionDateToString::optimize() { return this; } -Value ExpressionDateToString::serialize(bool explain) const { +Value ExpressionDateToString::serialize(const SerializationOptions& options) const { return Value( Document{{"$dateToString", - Document{{"date", _date->serialize(explain)}, - {"format", _format ? _format->serialize(explain) : Value()}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}, - {"onNull", _onNull ? _onNull->serialize(explain) : Value()}}}}); + Document{{"date", _date->serialize(options)}, + {"format", _format ? _format->serialize(options) : Value()}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}, + {"onNull", _onNull ? _onNull->serialize(options) : Value()}}}}); } Value ExpressionDateToString::evaluate(const Document& root, Variables* variables) const { @@ -2158,14 +2184,14 @@ boost::intrusive_ptr<Expression> ExpressionDateDiff::optimize() { return this; }; -Value ExpressionDateDiff::serialize(bool explain) const { +Value ExpressionDateDiff::serialize(const SerializationOptions& options) const { return Value{Document{ {"$dateDiff"_sd, - Document{{"startDate"_sd, _startDate->serialize(explain)}, - {"endDate"_sd, _endDate->serialize(explain)}, - {"unit"_sd, _unit->serialize(explain)}, - {"timezone"_sd, _timeZone ? _timeZone->serialize(explain) : Value{}}, - {"startOfWeek"_sd, _startOfWeek ? _startOfWeek->serialize(explain) : Value{}}}}}}; + Document{{"startDate"_sd, _startDate->serialize(options)}, + {"endDate"_sd, _endDate->serialize(options)}, + {"unit"_sd, _unit->serialize(options)}, + {"timezone"_sd, _timeZone ? _timeZone->serialize(options) : Value{}}, + {"startOfWeek"_sd, _startOfWeek ? _startOfWeek->serialize(options) : Value{}}}}}}; }; Date_t ExpressionDateDiff::convertToDate(const Value& value, StringData parameterName) { @@ -2379,10 +2405,24 @@ Value ExpressionObject::evaluate(const Document& root, Variables* variables) con return outputDoc.freezeToValue(); } -Value ExpressionObject::serialize(bool explain) const { +bool ExpressionObject::selfAndChildrenAreConstant() const { + for (auto&& [_, exprPointer] : _expressions) { + if (!exprPointer->selfAndChildrenAreConstant()) { + return false; + } + } + return true; +} + +Value ExpressionObject::serialize(const SerializationOptions& options) const { + if (options.literalPolicy != LiteralSerializationPolicy::kUnchanged && + selfAndChildrenAreConstant()) { + return ExpressionConstant::serializeConstant(options, Value(Document{})); + } MutableDocument outputDoc; for (auto&& pair : _expressions) { - outputDoc.addField(pair.first, pair.second->serialize(explain)); + outputDoc.addField(options.serializeFieldPathFromString(pair.first), + pair.second->serialize(options)); } return outputDoc.freezeToValue(); } @@ -2557,14 +2597,34 @@ Value ExpressionFieldPath::evaluate(const Document& root, Variables* variables) } } -Value ExpressionFieldPath::serialize(bool explain) const { - if (_fieldPath.getFieldName(0) == "CURRENT" && _fieldPath.getPathLength() > 1) { +namespace { +// Shared among expressions that need to serialize dotted paths and redact the path components. +auto getPrefixAndPath(FieldPath path) { + if (path.getFieldName(0) == "CURRENT" && path.getPathLength() > 1) { // use short form for "$$CURRENT.foo" but not just "$$CURRENT" - return Value("$" + _fieldPath.tail().fullPath()); + return std::make_pair(std::string("$"), path.tail()); } else { - return Value("$$" + _fieldPath.fullPath()); + return std::make_pair(std::string("$$"), path); } } +} // namespace + +Value ExpressionFieldPath::serialize(const SerializationOptions& options) const { + auto [prefix, path] = getPrefixAndPath(_fieldPath); + // First handles special cases for redaction of system variables. User variables will fall + // through to the default full redaction case. + if (options.transformIdentifiers && prefix.length() == 2) { + if (path.getPathLength() == 1 && Variables::isBuiltin(_variable)) { + // Nothing to redact for builtin variables. + return Value(prefix + path.fullPath()); + } else if (path.getPathLength() > 1 && Variables::isBuiltin(_variable)) { + // The first component of this path is a system variable, so keep that and redact + // the rest. + return Value(prefix + path.front() + "." + options.serializeFieldPath(path.tail())); + } + } + return Value(prefix + options.serializeFieldPath(path)); +} Expression::ComputedPaths ExpressionFieldPath::getComputedPaths(const std::string& exprFieldPath, Variables::Id renamingVar) const { @@ -2705,14 +2765,14 @@ intrusive_ptr<Expression> ExpressionFilter::optimize() { return this; } -Value ExpressionFilter::serialize(bool explain) const { +Value ExpressionFilter::serialize(const SerializationOptions& options) const { if (_limit) { - return Value(DOC("$filter" << DOC("input" << _input->serialize(explain) << "as" << _varName - << "cond" << _cond->serialize(explain) << "limit" - << (*_limit)->serialize(explain)))); + return Value(DOC("$filter" << DOC("input" << _input->serialize(options) << "as" << _varName + << "cond" << _cond->serialize(options) << "limit" + << (*_limit)->serialize(options)))); } - return Value(DOC("$filter" << DOC("input" << _input->serialize(explain) << "as" << _varName - << "cond" << _cond->serialize(explain)))); + return Value(DOC("$filter" << DOC("input" << _input->serialize(options) << "as" << _varName + << "cond" << _cond->serialize(options)))); } Value ExpressionFilter::evaluate(const Document& root, Variables* variables) const { @@ -2904,15 +2964,19 @@ intrusive_ptr<Expression> ExpressionLet::optimize() { return this; } -Value ExpressionLet::serialize(bool explain) const { +Value ExpressionLet::serialize(const SerializationOptions& options) const { MutableDocument vars; for (VariableMap::const_iterator it = _variables.begin(), end = _variables.end(); it != end; ++it) { - vars[it->second.name] = it->second.expression->serialize(explain); + auto key = it->second.name; + if (options.transformIdentifiers) { + key = options.transformIdentifiersCallback(key); + } + vars[key] = it->second.expression->serialize(options); } return Value( - DOC("$let" << DOC("vars" << vars.freeze() << "in" << _subExpression->serialize(explain)))); + DOC("$let" << DOC("vars" << vars.freeze() << "in" << _subExpression->serialize(options)))); } Value ExpressionLet::evaluate(const Document& root, Variables* variables) const { @@ -3006,9 +3070,9 @@ intrusive_ptr<Expression> ExpressionMap::optimize() { return this; } -Value ExpressionMap::serialize(bool explain) const { - return Value(DOC("$map" << DOC("input" << _input->serialize(explain) << "as" << _varName << "in" - << _each->serialize(explain)))); +Value ExpressionMap::serialize(const SerializationOptions& options) const { + return Value(DOC("$map" << DOC("input" << _input->serialize(options) << "as" << _varName << "in" + << _each->serialize(options)))); } Value ExpressionMap::evaluate(const Document& root, Variables* variables) const { @@ -3150,7 +3214,7 @@ ExpressionMeta::ExpressionMeta(ExpressionContext* const expCtx, MetaType metaTyp expCtx->sbeCompatible = false; } -Value ExpressionMeta::serialize(bool explain) const { +Value ExpressionMeta::serialize(const SerializationOptions& options) const { const auto nameIter = kMetaTypeToMetaName.find(_metaType); invariant(nameIter != kMetaTypeToMetaName.end()); return Value(DOC("$meta" << nameIter->second)); @@ -3786,7 +3850,7 @@ Value ExpressionLn::evaluateNumericArg(const Value& numericArg) const { if (numericArg.getType() == NumberDecimal) { Decimal128 argDecimal = numericArg.getDecimal(); if (argDecimal.isGreater(Decimal128::kNormalizedZero)) - return Value(argDecimal.logarithm()); + return Value(argDecimal.naturalLogarithm()); // Fall through for error case. } double argDouble = numericArg.coerceToDouble(); @@ -3940,9 +4004,9 @@ Value toValue(const std::array<std::uint8_t, 32>& buf) { return Value(BSONBinData(vec.data(), vec.size(), BinDataType::Encrypt)); } -Value ExpressionInternalFLEEqual::serialize(bool explain) const { +Value ExpressionInternalFLEEqual::serialize(const SerializationOptions& options) const { return Value(Document{{kInternalFleEq, - Document{{"field", _children[0]->serialize(explain)}, + Document{{"field", _children[0]->serialize(options)}, {"edc", toValue(_edcToken)}, {"counter", Value(static_cast<long long>(_contentionFactor))}, {"server", toValue(_serverToken)}}}}); @@ -4097,13 +4161,22 @@ void ExpressionNary::addOperand(const intrusive_ptr<Expression>& pExpression) { _children.push_back(pExpression); } -Value ExpressionNary::serialize(bool explain) const { +Value ExpressionNary::serialize(const SerializationOptions& options) const { const size_t nOperand = _children.size(); vector<Value> array; /* build up the array */ - for (size_t i = 0; i < nOperand; i++) - array.push_back(_children[i]->serialize(explain)); - + for (size_t i = 0; i < nOperand; i++) { + // If this input is a constant, bypass the standard serialization that wraps the + // representative value in $const. This does not lead to ambiguity for variadic operators + // but avoids bloating the representative shape for operators that have many inputs. + ExpressionConstant const* exprConst = dynamic_cast<ExpressionConstant*>(_children[i].get()); + if (exprConst) { + array.push_back(exprConst->serializeConstant( + options, exprConst->getValue(), false /* wrapRepresentativeValue */)); + } else { + array.push_back(_children[i]->serialize(options)); + } + } return Value(DOC(getOpName() << array)); } @@ -4568,11 +4641,11 @@ void ExpressionReduce::_doAddDependencies(DepsTracker* deps) const { _in->addDependencies(deps); } -Value ExpressionReduce::serialize(bool explain) const { +Value ExpressionReduce::serialize(const SerializationOptions& options) const { return Value(Document{{"$reduce", - Document{{"input", _input->serialize(explain)}, - {"initialValue", _initial->serialize(explain)}, - {"in", _in->serialize(explain)}}}}); + Document{{"input", _input->serialize(options)}, + {"initialValue", _initial->serialize(options)}, + {"in", _in->serialize(options)}}}}); } /* ------------------------ ExpressionReplaceBase ------------------------ */ @@ -4583,11 +4656,11 @@ void ExpressionReplaceBase::_doAddDependencies(DepsTracker* deps) const { _replacement->addDependencies(deps); } -Value ExpressionReplaceBase::serialize(bool explain) const { +Value ExpressionReplaceBase::serialize(const SerializationOptions& options) const { return Value(Document{{getOpName(), - Document{{"input", _input->serialize(explain)}, - {"find", _find->serialize(explain)}, - {"replacement", _replacement->serialize(explain)}}}}); + Document{{"input", _input->serialize(options)}, + {"find", _find->serialize(options)}, + {"replacement", _replacement->serialize(options)}}}}); } namespace { @@ -4880,9 +4953,9 @@ void ExpressionSortArray::_doAddDependencies(DepsTracker* deps) const { _input->addDependencies(deps); } -Value ExpressionSortArray::serialize(bool explain) const { +Value ExpressionSortArray::serialize(const SerializationOptions& options) const { return Value(Document{{kName, - Document{{"input", _input->serialize(explain)}, + Document{{"input", _input->serialize(options)}, {"sortBy", _sortBy.getOriginalElement()}}}}); } @@ -5871,20 +5944,20 @@ boost::intrusive_ptr<Expression> ExpressionSwitch::optimize() { return this; } -Value ExpressionSwitch::serialize(bool explain) const { +Value ExpressionSwitch::serialize(const SerializationOptions& options) const { std::vector<Value> serializedBranches; serializedBranches.reserve(numBranches()); for (int i = 0; i < numBranches(); ++i) { auto [caseExpr, thenExpr] = getBranch(i); - serializedBranches.push_back(Value(Document{{"case", caseExpr->serialize(explain)}, - {"then", thenExpr->serialize(explain)}})); + serializedBranches.push_back(Value(Document{{"case", caseExpr->serialize(options)}, + {"then", thenExpr->serialize(options)}})); } if (defaultExpr()) { return Value(Document{{"$switch", Document{{"branches", Value(serializedBranches)}, - {"default", defaultExpr()->serialize(explain)}}}}); + {"default", defaultExpr()->serialize(options)}}}}); } return Value(Document{{"$switch", Document{{"branches", Value(serializedBranches)}}}}); @@ -6126,11 +6199,11 @@ boost::intrusive_ptr<Expression> ExpressionTrim::optimize() { return this; } -Value ExpressionTrim::serialize(bool explain) const { +Value ExpressionTrim::serialize(const SerializationOptions& options) const { return Value( Document{{_name, - Document{{"input", _input->serialize(explain)}, - {"chars", _characters ? _characters->serialize(explain) : Value()}}}}); + Document{{"input", _input->serialize(options)}, + {"chars", _characters ? _characters->serialize(options) : Value()}}}}); } void ExpressionTrim::_doAddDependencies(DepsTracker* deps) const { @@ -6428,17 +6501,17 @@ boost::intrusive_ptr<Expression> ExpressionZip::optimize() { return this; } -Value ExpressionZip::serialize(bool explain) const { +Value ExpressionZip::serialize(const SerializationOptions& options) const { vector<Value> serializedInput; vector<Value> serializedDefaults; Value serializedUseLongestLength = Value(_useLongestLength); for (auto&& expr : _inputs) { - serializedInput.push_back(expr.get()->serialize(explain)); + serializedInput.push_back(expr.get()->serialize(options)); } for (auto&& expr : _defaults) { - serializedDefaults.push_back(expr.get()->serialize(explain)); + serializedDefaults.push_back(expr.get()->serialize(options)); } return Value(DOC("$zip" << DOC("inputs" << Value(serializedInput) << "defaults" @@ -7019,14 +7092,6 @@ boost::intrusive_ptr<Expression> ExpressionConvert::optimize() { return this; } -Value ExpressionConvert::serialize(bool explain) const { - return Value(Document{{"$convert", - Document{{"input", _input->serialize(explain)}, - {"to", _to->serialize(explain)}, - {"onError", _onError ? _onError->serialize(explain) : Value()}, - {"onNull", _onNull ? _onNull->serialize(explain) : Value()}}}}); -} - void ExpressionConvert::_doAddDependencies(DepsTracker* deps) const { _input->addDependencies(deps); _to->addDependencies(deps); @@ -7038,6 +7103,29 @@ void ExpressionConvert::_doAddDependencies(DepsTracker* deps) const { } } +Value ExpressionConvert::serialize(const SerializationOptions& options) const { + // Since the 'to' field is a parameter from a set of valid values and not free user input, + // we want to avoid boiling it down to the representative value in the query shape. The first + // condition is so that we can keep serializing correctly whenever the 'to' field is an + // expression that gets resolved down to a string of a valid type, or its corresponding + // numerical value. If it's just the constant, we want to wrap it in a $const except when the + // serialization policy is debug. + auto constExpr = dynamic_cast<ExpressionConstant*>(_to.get()); + Value toField = Value(); + if (!constExpr) { + toField = _to->serialize(options); + } else if (options.literalPolicy == LiteralSerializationPolicy::kToDebugTypeString) { + toField = constExpr->getValue(); + } else { + toField = Value(DOC("$const" << constExpr->getValue())); + } + return Value(Document{{"$convert", + Document{{"input", _input->serialize(options)}, + {"to", toField}, + {"onError", _onError ? _onError->serialize(options) : Value()}, + {"onNull", _onNull ? _onNull->serialize(options) : Value()}}}}); +} + BSONType ExpressionConvert::computeTargetType(Value targetTypeName) const { BSONType targetType; if (targetTypeName.getType() == BSONType::String) { @@ -7300,12 +7388,12 @@ void ExpressionRegex::_compile(RegexExecutionState* executionState) const { executionState->capturesBuffer.resize((1 + executionState->numCaptures) * 3); } -Value ExpressionRegex::serialize(bool explain) const { +Value ExpressionRegex::serialize(const SerializationOptions& options) const { return Value( Document{{_opName, - Document{{"input", _input->serialize(explain)}, - {"regex", _regex->serialize(explain)}, - {"options", _options ? _options->serialize(explain) : Value()}}}}); + Document{{"input", _input->serialize(options)}, + {"regex", _regex->serialize(options)}, + {"options", _options ? _options->serialize(options) : Value()}}}}); } void ExpressionRegex::_extractInputField(RegexExecutionState* executionState, @@ -7568,7 +7656,7 @@ void ExpressionRandom::_doAddDependencies(DepsTracker* deps) const { deps->needRandomGenerator = true; } -Value ExpressionRandom::serialize(const bool explain) const { +Value ExpressionRandom::serialize(const SerializationOptions& options) const { return Value(DOC(getOpName() << Document())); } @@ -7591,8 +7679,8 @@ Value ExpressionToHashedIndexKey::evaluate(const Document& root, Variables* vari BSONElementHasher::DEFAULT_HASH_SEED)); } -Value ExpressionToHashedIndexKey::serialize(bool explain) const { - return Value(DOC("$toHashedIndexKey" << _children[0]->serialize(explain))); +Value ExpressionToHashedIndexKey::serialize(const SerializationOptions& options) const { + return Value(DOC("$toHashedIndexKey" << _children[0]->serialize(options))); } void ExpressionToHashedIndexKey::_doAddDependencies(DepsTracker* deps) const { @@ -7668,13 +7756,13 @@ boost::intrusive_ptr<Expression> ExpressionDateArithmetics::optimize() { return intrusive_ptr<Expression>(this); } -Value ExpressionDateArithmetics::serialize(bool explain) const { +Value ExpressionDateArithmetics::serialize(const SerializationOptions& options) const { return Value( Document{{_opName, - Document{{"startDate", _startDate->serialize(explain)}, - {"unit", _unit->serialize(explain)}, - {"amount", _amount->serialize(explain)}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}}}}); + Document{{"startDate", _startDate->serialize(options)}, + {"unit", _unit->serialize(options)}, + {"amount", _amount->serialize(options)}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}}}}); } Value ExpressionDateArithmetics::evaluate(const Document& root, Variables* variables) const { @@ -7867,14 +7955,14 @@ boost::intrusive_ptr<Expression> ExpressionDateTrunc::optimize() { return this; }; -Value ExpressionDateTrunc::serialize(bool explain) const { +Value ExpressionDateTrunc::serialize(const SerializationOptions& options) const { return Value{Document{ {"$dateTrunc"_sd, - Document{{"date"_sd, _date->serialize(explain)}, - {"unit"_sd, _unit->serialize(explain)}, - {"binSize"_sd, _binSize ? _binSize->serialize(explain) : Value{}}, - {"timezone"_sd, _timeZone ? _timeZone->serialize(explain) : Value{}}, - {"startOfWeek"_sd, _startOfWeek ? _startOfWeek->serialize(explain) : Value{}}}}}}; + Document{{"date"_sd, _date->serialize(options)}, + {"unit"_sd, _unit->serialize(options)}, + {"binSize"_sd, _binSize ? _binSize->serialize(options) : Value{}}, + {"timezone"_sd, _timeZone ? _timeZone->serialize(options) : Value{}}, + {"startOfWeek"_sd, _startOfWeek ? _startOfWeek->serialize(options) : Value{}}}}}}; }; Date_t ExpressionDateTrunc::convertToDate(const Value& value) { @@ -8056,7 +8144,6 @@ Value ExpressionGetField::evaluate(const Document& root, Variables* variables) c return Value(); } - return inputValue.getDocument().getField(fieldValue.getString()); } @@ -8069,10 +8156,22 @@ void ExpressionGetField::_doAddDependencies(DepsTracker* deps) const { _field->addDependencies(deps); } -Value ExpressionGetField::serialize(const bool explain) const { +Value ExpressionGetField::serialize(const SerializationOptions& options) const { + // The parser guarantees that the '_field' expression evaluates to a constant string. + auto strPath = static_cast<ExpressionConstant*>(_field.get())->getValue().getString(); + + Value maybeRedactedPath{options.serializeFieldPathFromString(strPath)}; + // This is a pretty unique option to serialize. It is both a constant and a field path, which + // means that it: + // - should be redacted (if that option is set). + // - should *not* be wrapped in $const iff we are serializing for a debug string + if (options.literalPolicy != LiteralSerializationPolicy::kToDebugTypeString) { + maybeRedactedPath = Value(Document{{"$const"_sd, maybeRedactedPath}}); + } + return Value(Document{{"$getField"_sd, - Document{{"field"_sd, _field->serialize(explain)}, - {"input"_sd, _input->serialize(explain)}}}}); + Document{{"field"_sd, std::move(maybeRedactedPath)}, + {"input"_sd, _input->serialize(options)}}}}); } /* -------------------------- ExpressionSetField ------------------------------ */ @@ -8129,46 +8228,11 @@ intrusive_ptr<Expression> ExpressionSetField::parse(ExpressionContext* const exp uassert(4161103, str::stream() << name << " requires 'value' to be specified", valueExpr); uassert(4161109, str::stream() << name << " requires 'input' to be specified", inputExpr); - // The 'field' argument to '$setField' must evaluate to a constant string, for example, - // {$const: "$a.b"}. In case the user has forgotten to wrap the value into a '$const' or - // '$literal' expression, we will raise an error with a more meaningful description. - if (auto fieldPathExpr = dynamic_cast<ExpressionFieldPath*>(fieldExpr.get()); fieldPathExpr) { - auto fp = fieldPathExpr->getFieldPath().fullPathWithPrefix(); - uasserted(4161108, - str::stream() << "'" << fp - << "' is a field path reference which is not allowed " - "in this context. Did you mean {$literal: '" - << fp << "'}?"); - } - - auto constFieldExpr = dynamic_cast<ExpressionConstant*>(fieldExpr.get()); - uassert(4161106, - str::stream() << name - << " requires 'field' to evaluate to a constant, " - "but got a non-constant argument", - constFieldExpr); - uassert(4161107, - str::stream() << name - << " requires 'field' to evaluate to type String, " - "but got " - << typeName(constFieldExpr->getValue().getType()), - constFieldExpr->getValue().getType() == BSONType::String); - - - return make_intrusive<ExpressionSetField>(expCtx, fieldExpr, inputExpr, valueExpr); + return make_intrusive<ExpressionSetField>( + expCtx, std::move(fieldExpr), std::move(inputExpr), std::move(valueExpr)); } Value ExpressionSetField::evaluate(const Document& root, Variables* variables) const { - auto field = _field->evaluate(root, variables); - - // The parser guarantees that the '_field' expression evaluates to a constant string. - tassert(4161104, - str::stream() << kExpressionName - << " requires 'field' to evaluate to type String, " - "but got " - << typeName(field.getType()), - field.getType() == BSONType::String); - auto input = _input->evaluate(root, variables); if (input.nullish()) { return Value(BSONNULL); @@ -8182,7 +8246,7 @@ Value ExpressionSetField::evaluate(const Document& root, Variables* variables) c // Build output document and modify 'field'. MutableDocument outputDoc(input.getDocument()); - outputDoc.setField(field.getString(), value); + outputDoc.setField(_fieldName, value); return outputDoc.freezeToValue(); } @@ -8196,11 +8260,59 @@ void ExpressionSetField::_doAddDependencies(DepsTracker* deps) const { _value->addDependencies(deps); } -Value ExpressionSetField::serialize(const bool explain) const { +Value ExpressionSetField::serialize(const SerializationOptions& options) const { + // The parser guarantees that the '_field' expression evaluates to a constant string. + auto strPath = static_cast<ExpressionConstant*>(_field.get())->getValue().getString(); + + Value maybeRedactedPath{options.serializeFieldPathFromString(strPath)}; + // This is a pretty unique option to serialize. It is both a constant and a field path, which + // means that it: + // - should be redacted (if that option is set). + // - should *not* be wrapped in $const iff we are serializing for a debug string + if (options.literalPolicy != LiteralSerializationPolicy::kToDebugTypeString) { + maybeRedactedPath = Value(Document{{"$const"_sd, maybeRedactedPath}}); + } + return Value(Document{{"$setField"_sd, - Document{{"field"_sd, _field->serialize(explain)}, - {"input"_sd, _input->serialize(explain)}, - {"value"_sd, _value->serialize(explain)}}}}); + Document{{"field"_sd, std::move(maybeRedactedPath)}, + {"input"_sd, _input->serialize(options)}, + {"value"_sd, _value->serialize(options)}}}}); +} + +std::string ExpressionSetField::getValidFieldName(boost::intrusive_ptr<Expression> fieldExpr) { + tassert(9534701, + str::stream() << kExpressionName << " requires 'field' to be specified", + fieldExpr); + + // The 'field' argument to '$setField' must evaluate to a constant string, for example, + // {$const: "$a.b"}. In case the user has forgotten to wrap the value into a '$const' or + // '$literal' expression, we will raise an error with a more meaningful description. + if (auto fieldPathExpr = dynamic_cast<ExpressionFieldPath*>(fieldExpr.get()); fieldPathExpr) { + auto fp = fieldPathExpr->getFieldPath().fullPathWithPrefix(); + uasserted(4161108, + str::stream() << "'" << fp + << "' is a field path reference which is not allowed " + "in this context. Did you mean {$literal: '" + << fp << "'}?"); + } + + auto constFieldExpr = dynamic_cast<ExpressionConstant*>(fieldExpr.get()); + uassert(4161106, + str::stream() << kExpressionName + << " requires 'field' to evaluate to a constant, " + "but got a non-constant argument", + constFieldExpr); + uassert(4161107, + str::stream() << kExpressionName + << " requires 'field' to evaluate to type String, " + "but got " + << typeName(constFieldExpr->getValue().getType()), + constFieldExpr->getValue().getType() == BSONType::String); + uassert(9534700, + str::stream() << kExpressionName << ": 'field' cannot contain an embedded null byte", + constFieldExpr->getValue().getStringData().find('\0') == std::string::npos); + + return constFieldExpr->getValue().getString(); } /* ------------------------- ExpressionTsSecond ----------------------------- */ @@ -8289,12 +8401,12 @@ boost::intrusive_ptr<Expression> ExpressionInternalKeyStringValue::parse( return make_intrusive<ExpressionInternalKeyStringValue>(expCtx, inputExpr, collationExpr); } -Value ExpressionInternalKeyStringValue::serialize(bool explain) const { +Value ExpressionInternalKeyStringValue::serialize(const SerializationOptions& options) const { return Value( Document{{getOpName(), - Document{{"input", _children[_kInput]->serialize(explain)}, + Document{{"input", _children[_kInput]->serialize(options)}, {"collation", - _children[_kCollation] ? _children[_kCollation]->serialize(explain) + _children[_kCollation] ? _children[_kCollation]->serialize(options) : Value()}}}}); } diff --git a/src/mongo/db/pipeline/expression.h b/src/mongo/db/pipeline/expression.h index 26f04430e0b..c5140186fca 100644 --- a/src/mongo/db/pipeline/expression.h +++ b/src/mongo/db/pipeline/expression.h @@ -54,6 +54,7 @@ #include "mongo/db/query/allowed_contexts.h" #include "mongo/db/query/datetime/date_time_support.h" #include "mongo/db/query/query_feature_flags_gen.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/db/query/sort_pattern.h" #include "mongo/db/server_options.h" #include "mongo/db/update/pattern_cmp.h" @@ -223,7 +224,7 @@ public: * If 'explain' is false, the returned Value must result in the same Expression when parsed by * parseOperand(). */ - virtual Value serialize(bool explain) const = 0; + virtual Value serialize(const SerializationOptions& options = {}) const = 0; /** * Evaluate the expression with respect to the Document given by 'root' and the Variables given @@ -352,6 +353,13 @@ public: return monotonic::State::NonMonotonic; } + /** + * Helper to determine whether this expression always evaluates to the same value. + */ + virtual bool selfAndChildrenAreConstant() const { + return false; + } + protected: using ExpressionVector = std::vector<boost::intrusive_ptr<Expression>>; @@ -384,12 +392,105 @@ private: }; /** + * A constant expression. Repeated calls to evaluate() will always return the same thing. + */ +class ExpressionConstant final : public Expression { +public: + ExpressionConstant(ExpressionContext* expCtx, const Value& value); + + boost::intrusive_ptr<Expression> optimize() final; + Value evaluate(const Document& root, Variables* variables) const final; + Value serialize(const SerializationOptions& options = {}) const final; + + const char* getOpName() const; + + /** + * Creates a new ExpressionConstant with value 'value'. + */ + static boost::intrusive_ptr<ExpressionConstant> create(ExpressionContext* expCtx, + const Value& value); + + static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, + BSONElement bsonExpr, + const VariablesParseState& vps); + + /** + * Returns true if 'expression' is nullptr or if 'expression' is an instance of an + * ExpressionConstant. + */ + static bool isNullOrConstant(boost::intrusive_ptr<Expression> expression) { + return !expression || dynamic_cast<ExpressionConstant*>(expression.get()); + } + + /** + * Returns true if 'expression' is an instance of an ExpressionConstant. + */ + static bool isConstant(boost::intrusive_ptr<Expression> expression) { + return dynamic_cast<ExpressionConstant*>(expression.get()); + } + + static Value serializeConstant(const SerializationOptions& opts, + Value val, + bool wrapRepresentativeValue = true); + + bool selfAndChildrenAreConstant() const override final { + return true; + } + + /** + * Returns true if every expression in 'expressions' is either a nullptr or an instance of an + * ExpressionConstant. + */ + static bool allNullOrConstant( + const std::initializer_list<boost::intrusive_ptr<Expression>>& expressions) { + return std::all_of(expressions.begin(), expressions.end(), [](auto exp) { + return ExpressionConstant::isNullOrConstant(exp); + }); + } + template <typename ExpressionContainer> + static bool allConstant(const ExpressionContainer& expressions) { + return std::all_of(expressions.begin(), expressions.end(), [](auto exp) { + return ExpressionConstant::isConstant(exp); + }); + } + + /** + * Returns the constant value represented by this Expression. + */ + Value getValue() const { + return _value; + } + + void setValue(const Value& value) { + _value = value; + }; + + void acceptVisitor(ExpressionMutableVisitor* visitor) final { + return visitor->visit(this); + } + + void acceptVisitor(ExpressionConstVisitor* visitor) const final { + return visitor->visit(this); + } + +protected: + void _doAddDependencies(DepsTracker* deps) const override; + +private: + monotonic::State getMonotonicState(const FieldPath& sortedFieldPath) const final { + return monotonic::State::Constant; + } + + Value _value; +}; + +/** * Inherit from ExpressionVariadic or ExpressionFixedArity instead of directly from this class. */ class ExpressionNary : public Expression { public: boost::intrusive_ptr<Expression> optimize() override; - Value serialize(bool explain) const override; + Value serialize(const SerializationOptions& options = {}) const override; /* Add an operand to the n-ary expression. @@ -454,6 +555,33 @@ public: : ExpressionNaryBase<SubClass>(expCtx) {} ExpressionVariadic(ExpressionContext* const expCtx, Expression::ExpressionVector&& children) : ExpressionNaryBase<SubClass>(expCtx, std::move(children)) {} + + Value serialize(const SerializationOptions& options = {}) const { + // As a special case, we would like to serialize a variadic number of children as + // "?array<?subtype>" if they are all constant. Check for that here, otherwise default to + // the normal one-by-one serialization of the children. + if (options.literalPolicy == LiteralSerializationPolicy::kToDebugTypeString && + ExpressionConstant::allConstant(this->_children)) { + // We could evaluate the expression right here and now and end up with just the one + // constant answer, but this is not an optimization funciton, it is meant to just + // serialize what we have, so let's preserve the array of constants. + auto args = [&]() { + std::vector<Value> values; + const auto& constants = this->_children; + values.reserve(constants.size()); + std::transform(constants.begin(), + constants.end(), + std::back_inserter(values), + [](const auto& exp) { + return static_cast<ExpressionConstant*>(exp.get())->getValue(); + }); + return values; + }(); + return Value(Document{ + {this->getOpName(), ExpressionConstant::serializeConstant(options, Value(args))}}); + } + return ExpressionNary::serialize(options); + } }; /** @@ -569,9 +697,9 @@ public: return AccumulatorN::kName.rawData(); } - Value serialize(bool explain) const { + Value serialize(const SerializationOptions& options = {}) const { MutableDocument md; - AccumulatorN::serializeHelper(_n, _output, explain, md); + AccumulatorN::serializeHelper(_n, _output, options, md); return Value(DOC(getOpName() << md.freeze())); } @@ -686,78 +814,6 @@ public: }; /** - * A constant expression. Repeated calls to evaluate() will always return the same thing. - */ -class ExpressionConstant final : public Expression { -public: - ExpressionConstant(ExpressionContext* expCtx, const Value& value); - - boost::intrusive_ptr<Expression> optimize() final; - Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; - - const char* getOpName() const; - - /** - * Creates a new ExpressionConstant with value 'value'. - */ - static boost::intrusive_ptr<ExpressionConstant> create(ExpressionContext* expCtx, - const Value& value); - - static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, - BSONElement bsonExpr, - const VariablesParseState& vps); - - /** - * Returns true if 'expression' is nullptr or if 'expression' is an instance of an - * ExpressionConstant. - */ - static bool isNullOrConstant(boost::intrusive_ptr<Expression> expression) { - return !expression || dynamic_cast<ExpressionConstant*>(expression.get()); - } - - /** - * Returns true if every expression in 'expressions' is either a nullptr or an instance of an - * ExpressionConstant. - */ - static bool allNullOrConstant( - const std::initializer_list<boost::intrusive_ptr<Expression>>& expressions) { - return std::all_of(expressions.begin(), expressions.end(), [](auto exp) { - return ExpressionConstant::isNullOrConstant(exp); - }); - } - - /** - * Returns the constant value represented by this Expression. - */ - Value getValue() const { - return _value; - } - - void setValue(const Value& value) { - _value = value; - }; - - void acceptVisitor(ExpressionMutableVisitor* visitor) final { - return visitor->visit(this); - } - - void acceptVisitor(ExpressionConstVisitor* visitor) const final { - return visitor->visit(this); - } - -protected: - void _doAddDependencies(DepsTracker* deps) const override; - -private: - monotonic::State getMonotonicState(const FieldPath& sortedFieldPath) const final { - return monotonic::State::Constant; - } - - Value _value; -}; - -/** * Inherit from this class if your expression works with date types, and accepts either a single * argument which is a date, or an object {date: <date>, timezone: <string>}. */ @@ -798,11 +854,11 @@ public: * Always serializes to the full {date: <date arg>, timezone: <timezone arg>} format, leaving * off the timezone if not specified. */ - Value serialize(bool explain) const final { - auto timezone = _timeZone ? _timeZone->serialize(explain) : Value(); + Value serialize(const SerializationOptions& options = {}) const final { + auto timezone = _timeZone ? _timeZone->serialize(options) : Value(); return Value(Document{ {_opName, - Document{{"date", _date->serialize(explain)}, {"timezone", std::move(timezone)}}}}); + Document{{"date", _date->serialize(options)}, {"timezone", std::move(timezone)}}}}); } boost::intrusive_ptr<Expression> optimize() final { @@ -1056,7 +1112,7 @@ public: } Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; static boost::intrusive_ptr<ExpressionArray> create( ExpressionContext* const expCtx, std::vector<boost::intrusive_ptr<Expression>>&& children) { @@ -1073,6 +1129,8 @@ public: void acceptVisitor(ExpressionConstVisitor* visitor) const final { return visitor->visit(this); } + + bool selfAndChildrenAreConstant() const override final; }; @@ -1222,7 +1280,7 @@ class ExpressionCoerceToBool final : public Expression { public: boost::intrusive_ptr<Expression> optimize() final; Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; static boost::intrusive_ptr<ExpressionCoerceToBool> create( ExpressionContext* expCtx, boost::intrusive_ptr<Expression> pExpression); @@ -1385,8 +1443,8 @@ public: boost::intrusive_ptr<Expression> onNull, boost::intrusive_ptr<Expression> onError); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -1427,8 +1485,8 @@ public: boost::intrusive_ptr<Expression> isoDayOfWeek, boost::intrusive_ptr<Expression> timeZone); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -1507,8 +1565,8 @@ public: boost::intrusive_ptr<Expression> timeZone, boost::intrusive_ptr<Expression> iso8601); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -1541,8 +1599,8 @@ public: boost::intrusive_ptr<Expression> date, boost::intrusive_ptr<Expression> timeZone, boost::intrusive_ptr<Expression> onNull); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -1653,8 +1711,8 @@ public: boost::intrusive_ptr<Expression> unit, boost::intrusive_ptr<Expression> timezone, boost::intrusive_ptr<Expression> startOfWeek); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, @@ -1786,7 +1844,7 @@ public: boost::intrusive_ptr<Expression> optimize() final; Value evaluate(const Document& root, Variables* variables) const; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; /* Create a field path expression using old semantics (rooted off of CURRENT). @@ -1887,8 +1945,8 @@ private: class ExpressionFilter final : public Expression { public: - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -2124,8 +2182,8 @@ public: class ExpressionLet final : public Expression { public: - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -2241,7 +2299,7 @@ public: ConstDataRange serverToken, int64_t contentionFactor, ConstDataRange edcToken); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; const char* getOpName() const; @@ -2275,8 +2333,8 @@ public: boost::intrusive_ptr<Expression> input, // yields array to iterate boost::intrusive_ptr<Expression> each); // yields results to be added to output array - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -2308,7 +2366,7 @@ class ExpressionMeta final : public Expression { public: ExpressionMeta(ExpressionContext* expCtx, DocumentMetadataFields::MetaType metaType); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -2502,7 +2560,7 @@ class ExpressionObject final : public Expression { public: boost::intrusive_ptr<Expression> optimize() final; Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; static boost::intrusive_ptr<ExpressionObject> create( ExpressionContext* expCtx, @@ -2534,8 +2592,10 @@ public: return visitor->visit(this); } + protected: void _doAddDependencies(DepsTracker* deps) const final; + bool selfAndChildrenAreConstant() const override final; private: ExpressionObject( @@ -2645,7 +2705,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vps); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -2681,8 +2741,8 @@ public: virtual const char* getOpName() const = 0; Value evaluate(const Document& root, Variables* variables) const final; - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; protected: void _doAddDependencies(DepsTracker* deps) const final; @@ -2962,7 +3022,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vps); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3078,10 +3138,14 @@ private: FieldPath getFieldPath() const { auto inputConstExpression = dynamic_cast<ExpressionConstant*>(_children[0].get()); uassert(5511201, - "Expected const expression as argument to _internalUnwindAllAlongPath", + "Expected const expression as argument to _internalFindAllValuesAtPath", inputConstExpression); auto constVal = inputConstExpression->getValue(); - // getString asserts if type != string, which is the correct behavior for what we want. + + uassert(9567004, + str::stream() << getOpName() << " requires argument to be a string", + constVal.getType() == BSONType::String); + return FieldPath(constVal.getString()); } }; @@ -3337,7 +3401,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vpsIn); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3452,7 +3516,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vpsIn); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3703,7 +3767,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vpsIn); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3742,8 +3806,8 @@ public: const VariablesParseState& vpsIn); Value evaluate(const Document& root, Variables* variables) const final; - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3848,7 +3912,7 @@ public: boost::optional<std::pair<boost::optional<std::string>, std::string>> getConstantPatternAndOptions() const; - Value serialize(bool explain) const; + Value serialize(const SerializationOptions& options = {}) const; const std::string& getOpName() const { return _opName; @@ -3965,7 +4029,7 @@ public: BSONElement exprElement, const VariablesParseState& vps); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; @@ -4011,7 +4075,7 @@ public: } Value evaluate(const Document& root, Variables* variables) const; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; protected: void _doAddDependencies(DepsTracker* deps) const final; @@ -4034,8 +4098,8 @@ public: _timeZone(_children[3]), _opName(opName) {} - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; protected: @@ -4162,8 +4226,8 @@ public: boost::intrusive_ptr<Expression> binSize, boost::intrusive_ptr<Expression> timezone, boost::intrusive_ptr<Expression> startOfWeek); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -4253,7 +4317,7 @@ public: expCtx->sbeCompatible = false; } - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; @@ -4294,11 +4358,12 @@ public: : Expression(expCtx, {std::move(field), std::move(input), std::move(value)}), _field(_children[0]), _input(_children[1]), - _value(_children[2]) { + _value(_children[2]), + _fieldName(getValidFieldName(_children[0])) { expCtx->sbeCompatible = false; } - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; @@ -4318,9 +4383,18 @@ protected: void _doAddDependencies(DepsTracker* deps) const final override; private: + /** + * Ensures 'fieldExpr' is a constant string representing a valid field name and returns it as a + * string. If 'fieldExpr' is not valid, this function will throw a 'uassert()'. + */ + std::string getValidFieldName(boost::intrusive_ptr<Expression> fieldExpr); + boost::intrusive_ptr<Expression>& _field; boost::intrusive_ptr<Expression>& _input; boost::intrusive_ptr<Expression>& _value; + + // This is pre-validated by the constructor. + const std::string _fieldName; }; class ExpressionTsSecond final : public ExpressionFixedArity<ExpressionTsSecond, 1> { @@ -4429,7 +4503,7 @@ public: BSONElement expr, const VariablesParseState& vps); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; diff --git a/src/mongo/db/pipeline/expression_and_test.cpp b/src/mongo/db/pipeline/expression_and_test.cpp index cbfa37a77d1..9671c08b2db 100644 --- a/src/mongo/db/pipeline/expression_and_test.cpp +++ b/src/mongo/db/pipeline/expression_and_test.cpp @@ -76,8 +76,8 @@ static BSONObj toBson(const Value& value) { } /** Convert Expression to BSON. */ -static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); +static BSONObj expressionToBson(const boost::intrusive_ptr<Expression>& expression) { + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } /** Convert Document to BSON. */ diff --git a/src/mongo/db/pipeline/expression_compare_test.cpp b/src/mongo/db/pipeline/expression_compare_test.cpp index 11836d77267..941eceeaa59 100644 --- a/src/mongo/db/pipeline/expression_compare_test.cpp +++ b/src/mongo/db/pipeline/expression_compare_test.cpp @@ -75,7 +75,7 @@ static BSONObj toBson(const Value& value) { /** Convert Expression to BSON. */ static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } /** Convert Document to BSON. */ diff --git a/src/mongo/db/pipeline/expression_context.cpp b/src/mongo/db/pipeline/expression_context.cpp index 7258456a1ac..626db2f6016 100644 --- a/src/mongo/db/pipeline/expression_context.cpp +++ b/src/mongo/db/pipeline/expression_context.cpp @@ -49,6 +49,49 @@ ExpressionContext::ResolvedNamespace::ResolvedNamespace(NamespaceString ns, : ns(std::move(ns)), pipeline(std::move(pipeline)), uuid(collUUID) {} ExpressionContext::ExpressionContext(OperationContext* opCtx, + const FindCommandRequest& findCmd, + std::unique_ptr<CollatorInterface> collator, + bool mayDbProfile, + boost::optional<ExplainOptions::Verbosity> verbosity, + bool allowDiskUseDefault) + // Although both 'find' and 'aggregate' commands have an ExpressionContext, some of the data + // members in the ExpressionContext are used exclusively by the aggregation subsystem. This + // includes the following fields which here we simply initialize to some meaningless default + // value: + // - explain + // - fromMongos + // - needsMerge + // - bypassDocumentValidation + // - mongoProcessInterface + // - resolvedNamespaces + // - uuid + // + // As we change the code to make the find and agg systems more tightly coupled, it would make + // sense to start initializing these fields for find operations as well. + : ExpressionContext(opCtx, + verbosity, + false, // fromMongos + false, // needsMerge + findCmd.getAllowDiskUse().value_or(allowDiskUseDefault), + false, // bypassDocumentValidation + false, // isMapReduceCommand + findCmd.getNamespaceOrUUID().nss().has_value() + ? *findCmd.getNamespaceOrUUID().nss() + : NamespaceString{}, + findCmd.getLegacyRuntimeConstants(), + std::move(collator), + nullptr, // mongoProcessInterface + {}, // resolvedNamespaces + [&findCmd]() -> boost::optional<UUID> { + if (findCmd.getNamespaceOrUUID().uuid().has_value()) { + return findCmd.getNamespaceOrUUID().uuid(); + } + return boost::none; + }(), + findCmd.getLet(), + mayDbProfile) {} + +ExpressionContext::ExpressionContext(OperationContext* opCtx, const AggregateCommandRequest& request, std::unique_ptr<CollatorInterface> collator, std::shared_ptr<MongoProcessInterface> processInterface, @@ -159,6 +202,44 @@ ExpressionContext::ExpressionContext( variables.seedVariablesWithLetParameters(this, *letParameters); } +ExpressionContext::ExpressionContext(OperationContext* opCtx, + const NamespaceString& nss, + const boost::optional<BSONObj>& letParameters) + : explain(boost::none), + allowDiskUse(false), + ns(nss), + opCtx(opCtx), + jsHeapLimitMB(internalQueryJavaScriptHeapSizeLimitMB.load()), + mongoProcessInterface(std::make_shared<StubMongoProcessInterface>()), + timeZoneDatabase(opCtx && opCtx->getServiceContext() + ? TimeZoneDatabase::get(opCtx->getServiceContext()) + : nullptr), + variablesParseState(variables.useIdGenerator()), + maxFeatureCompatibilityVersion(boost::none), // Ensure all features are allowed. + mayDbProfile(true), + _collator(nullptr), + _documentComparator(_collator.get()), + _valueComparator(_collator.get()) { + // This is a shortcut to avoid reading the clock and the vector clock, since we don't actually + // care about their values for this 'blank' ExpressionContext codepath. + variables.setLegacyRuntimeConstants({Date_t::min(), Timestamp()}); + // Expression counters are reported in serverStatus to indicate how often clients use certain + // expressions/stages, so it's a side effect tied to parsing. We must stop expression counters + // before re-parsing to avoid adding to the counters more than once per a given query. + stopExpressionCounters(); + if (letParameters) + variables.seedVariablesWithLetParameters(this, *letParameters); +} + +boost::intrusive_ptr<ExpressionContext> ExpressionContext::makeBlankExpressionContext( + OperationContext* opCtx, + const NamespaceStringOrUUID& nssOrUUID, + boost::optional<BSONObj> shapifiedLet) { + const auto nss = nssOrUUID.nss().has_value() ? *nssOrUUID.nss() : NamespaceString{}; + // This constructor is private, so we can't use `boost::make_instrusive()`. + return new ExpressionContext(opCtx, nss, shapifiedLet); +} + void ExpressionContext::checkForInterruptSlow() { // This check could be expensive, at least in relative terms, so don't check every time. invariant(opCtx); diff --git a/src/mongo/db/pipeline/expression_context.h b/src/mongo/db/pipeline/expression_context.h index c5ce9b74f27..aa2d212f53d 100644 --- a/src/mongo/db/pipeline/expression_context.h +++ b/src/mongo/db/pipeline/expression_context.h @@ -114,6 +114,16 @@ public: }; /** + * Constructs an ExpressionContext to be used for find command parsing and evaluation. + */ + ExpressionContext(OperationContext* opCtx, + const FindCommandRequest& findCmd, + std::unique_ptr<CollatorInterface> collator, + bool mayDbProfile, + boost::optional<ExplainOptions::Verbosity> verbosity = boost::none, + bool allowDiskUseByDefault = false); + + /** * Constructs an ExpressionContext to be used for Pipeline parsing and evaluation. * 'resolvedNamespaces' maps collection names (not full namespaces) to ResolvedNamespaces. */ @@ -161,6 +171,22 @@ public: boost::optional<ExplainOptions::Verbosity> explain = boost::none); /** + * Constructs a blank ExpressionContext suitable for creating Query Shapes, but it could be + * applied to other use cases as well. + * + * The process for creating a Query Shape sometimes requires re-parsing the BSON into a proper + * AST, and for that you need an ExpressionContext. + * + * Note: this is meant for introspection and is not suitable for using to execute queries - + * since it does not contain for example a collation argument or a real MongoProcessInterface + * for execution. + */ + static boost::intrusive_ptr<ExpressionContext> makeBlankExpressionContext( + OperationContext* opCtx, + const NamespaceStringOrUUID& nssOrUUID, + boost::optional<BSONObj> shapifiedLet = boost::none); + + /** * Used by a pipeline to check for interrupts so that killOp() works. Throws a UserAssertion if * this aggregation pipeline has been interrupted. */ @@ -278,7 +304,8 @@ public: */ const ResolvedNamespace& getResolvedNamespace(const NamespaceString& nss) const { auto it = _resolvedNamespaces.find(nss.coll()); - invariant(it != _resolvedNamespaces.end()); + invariant(it != _resolvedNamespaces.end(), + str::stream() << "No resolved namespace provided for " << nss.toString()); return it->second; }; @@ -323,10 +350,8 @@ public: * created yet. Initializes the Scope with the 'jsScope' variables from the runtimeConstants. * Loads the Scope with the functions stored in system.js if the expression isn't executed on * mongos and is called from a MapReduce command or `forceLoadOfStoredProcedures` is true. - * - * Returns a JsExec and a boolean indicating whether the Scope was created as part of this call. */ - auto getJsExecWithScope(bool forceLoadOfStoredProcedures = false) const { + JsExecution* getJsExecWithScope(bool forceLoadOfStoredProcedures = false) const { uassert(31264, "Cannot run server-side javascript without the javascript engine enabled", getGlobalScriptEngine()); @@ -348,9 +373,17 @@ public: "$where."); } - auto scopeObj = BSONObj(); + // If there is a cached JsExecution object, return it. This is a performance optimization + // that avoids potentially copying the scope object, which is not needed if a cached exec + // object already exists. + JsExecution* jsExec = JsExecution::getCached(opCtx, loadStoredProcedures); + if (jsExec) { + return jsExec; + } + + BSONObj scopeObj = BSONObj(); if (variables.hasValue(Variables::kJsScopeId)) { - auto scopeVar = variables.getValue(Variables::kJsScopeId); + Value scopeVar = variables.getValue(Variables::kJsScopeId); invariant(scopeVar.isObject()); scopeObj = scopeVar.getDocument().toBson(); } @@ -558,6 +591,14 @@ protected: bool _requiresTimeseriesExtendedRangeSupport = false; private: + // Instantiates an ExpressionContext which does not increment expression counters and does not + // enforce FCV restrictions. It is used for implementing the `makeBlankExpressionContext()` + // factory method. Please also note that the runtime constants are not given real/accurate + // values of '$$NOW' and '$$CLUSTER_TIME', in the name of efficiency. + ExpressionContext(OperationContext* opCtx, + const NamespaceString& ns, + const boost::optional<BSONObj>& letParameters = boost::none); + boost::optional<ExpressionCounters> _expressionCounters = boost::none; }; diff --git a/src/mongo/db/pipeline/expression_context_test.cpp b/src/mongo/db/pipeline/expression_context_test.cpp index b105dce531b..2ff0ef20b1e 100644 --- a/src/mongo/db/pipeline/expression_context_test.cpp +++ b/src/mongo/db/pipeline/expression_context_test.cpp @@ -39,15 +39,6 @@ #include "mongo/db/vector_clock_mutable.h" #include "mongo/unittest/unittest.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString(); \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace mongo { namespace { diff --git a/src/mongo/db/pipeline/expression_convert_test.cpp b/src/mongo/db/pipeline/expression_convert_test.cpp index 678c29b5b16..c1cfd639ce1 100644 --- a/src/mongo/db/pipeline/expression_convert_test.cpp +++ b/src/mongo/db/pipeline/expression_convert_test.cpp @@ -67,10 +67,11 @@ TEST_F(ExpressionConvertTest, ParseAndSerializeWithoutOptionalArguments) { auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}}}")), - convertExp->serialize(false)); + convertExp->serialize()); ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}}}")), - convertExp->serialize(true)); + convertExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})); } TEST_F(ExpressionConvertTest, ParseAndSerializeWithOnError) { @@ -85,11 +86,12 @@ TEST_F(ExpressionConvertTest, ParseAndSerializeWithOnError) { ASSERT_VALUE_EQ( Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}, onError: {$const: 0}}}")), - convertExp->serialize(false)); + convertExp->serialize()); ASSERT_VALUE_EQ( Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}, onError: {$const: 0}}}")), - convertExp->serialize(true)); + convertExp->serialize( + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})); } TEST_F(ExpressionConvertTest, ParseAndSerializeWithOnNull) { @@ -104,11 +106,12 @@ TEST_F(ExpressionConvertTest, ParseAndSerializeWithOnNull) { ASSERT_VALUE_EQ( Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}, onNull: {$const: 0}}}")), - convertExp->serialize(false)); + convertExp->serialize()); ASSERT_VALUE_EQ( Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}, onNull: {$const: 0}}}")), - convertExp->serialize(true)); + convertExp->serialize( + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})); } TEST_F(ExpressionConvertTest, ConvertWithoutInputFailsToParse) { @@ -141,6 +144,32 @@ TEST_F(ExpressionConvertTest, ConvertWithoutToFailsToParse) { }); } +TEST_F(ExpressionConvertTest, RoundTripSerialization) { + auto expCtx = getExpCtx(); + + // Round-trip serialization of an argument that *looks* like an expression. + auto spec = BSON("$convert" << BSON("input" << BSON("$literal" << BSON("$toString" + << "this is a string")) + << "to" + << "string")); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + auto serialized = convertExp->serialize(opts); + ASSERT_VALUE_EQ(Value(BSON("$convert" << BSON("input" << BSON("$const" << BSON("?" + << "?")) + << "to" + << BSON("$const" + << "string")))), + serialized); + + auto roundTrip = Expression::parseExpression(expCtx.get(), + serialized.getDocument().toBson(), + expCtx->variablesParseState) + ->serialize(opts); + ASSERT_VALUE_EQ(roundTrip, serialized); +} + TEST_F(ExpressionConvertTest, InvalidTypeNameFails) { auto expCtx = getExpCtx(); @@ -3445,4 +3474,82 @@ TEST_F(ExpressionConvertShortcutsTest, ThrowsOnConversionFailure) { } } // namespace ExpressionConvertShortcutsTest + +namespace ExpressionConvertSerializationTest { + +using ExpressionConvertSerializationTest = AggregationContextFixture; + +TEST_F(ExpressionConvertSerializationTest, ConvertToTypeSerializesCorrectly) { + auto expCtx = getExpCtx(); + auto spec = fromjson("{$convert: {input: '$funnyTest', to: 'bool'}}"); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'bool'}}}")), + convertExp->serialize(SerializationOptions{})); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'bool'}}}")), + convertExp->serialize(SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: 'bool'}}")), + convertExp->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +TEST_F(ExpressionConvertSerializationTest, ConvertToDollarSerializesCorrectly) { + auto expCtx = getExpCtx(); + auto spec = fromjson("{$convert: {input: '$funnyTest', to: {$add: [7, 2]}}}"); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + ASSERT_VALUE_EQ( + Value( + fromjson("{$convert: {input: '$funnyTest', to: {$add: [{$const: 7}, {$const: 2}]}}}")), + convertExp->serialize(SerializationOptions{})); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$add: [1, 1]}}}")), + convertExp->serialize(SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$add: '?array<?number>'}}}")), + convertExp->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +TEST_F(ExpressionConvertSerializationTest, ToStringSerializesCorrectly) { + auto expCtx = getExpCtx(); + + auto spec = BSON("$toString" + << "$funnyTest"); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'string'}}}")), + convertExp->serialize(SerializationOptions{})); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'string'}}}")), + convertExp->serialize(SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: 'string'}}")), + convertExp->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +TEST_F(ExpressionConvertSerializationTest, ToBoolSerializesCorrectly) { + auto expCtx = getExpCtx(); + + auto spec = BSON("$toBool" + << "$funnyTest"); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'bool'}}}")), + convertExp->serialize(SerializationOptions{})); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'bool'}}}")), + convertExp->serialize(SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: 'bool'}}")), + convertExp->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +} // namespace ExpressionConvertSerializationTest + } // namespace mongo diff --git a/src/mongo/db/pipeline/expression_date_test.cpp b/src/mongo/db/pipeline/expression_date_test.cpp index 9205ea29633..c4b7a01db42 100644 --- a/src/mongo/db/pipeline/expression_date_test.cpp +++ b/src/mongo/db/pipeline/expression_date_test.cpp @@ -64,8 +64,10 @@ TEST_F(ExpressionDateFromPartsTest, SerializesToObjectSyntax) { {"second", Document{{"$const", 15}}}, {"millisecond", Document{{"$const", 414}}}, {"timezone", Document{{"$const", "America/Los_Angeles"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } TEST_F(ExpressionDateFromPartsTest, OptimizesToConstantIfAllInputsAreConstant) { @@ -183,8 +185,10 @@ TEST_F(ExpressionDateToPartsTest, SerializesToObjectSyntax) { Document{{"date", Document{{"$const", Date_t{}}}}, {"timezone", Document{{"$const", "Europe/London"_sd}}}, {"iso8601", Document{{"$const", false}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } TEST_F(ExpressionDateToPartsTest, OptimizesToConstantIfAllInputsAreConstant) { @@ -430,22 +434,28 @@ TEST_F(DateExpressionTest, SerializesToObjectSyntax) { Value(Document{{expName, Document{{"date", Document{{"$const", Date_t{}}}}, {"timezone", Document{{"$const", "Europe/London"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); // Test that it serializes to the full format if given a date. spec = BSON(expName << Date_t{}); expectedSerialization = Value(Document{{expName, Document{{"date", Document{{"$const", Date_t{}}}}}}}); dateExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); // Test that it serializes to the full format if given a date within an array. spec = BSON(expName << BSON_ARRAY(Date_t{})); dateExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } } @@ -598,8 +608,10 @@ TEST_F(ExpressionDateToStringTest, SerializesToObjectSyntax) { {"timezone", Document{{"$const", "Europe/London"_sd}}}, {"onNull", Document{{"$const", "nullDefault"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } TEST_F(ExpressionDateToStringTest, OptimizesToConstantIfAllInputsAreConstant) { @@ -824,8 +836,10 @@ TEST_F(ExpressionDateFromStringTest, SerializesToObjectSyntax) { Document{{"$dateFromString", Document{{"dateString", Document{{"$const", "2017-07-04T13:06:44Z"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); // Test that it serializes to the full format if given an object specification. spec = BSON("$dateFromString" << BSON("dateString" @@ -838,8 +852,10 @@ TEST_F(ExpressionDateFromStringTest, SerializesToObjectSyntax) { Document{{"dateString", Document{{"$const", "2017-07-04T13:06:44Z"_sd}}}, {"timezone", Document{{"$const", "Europe/London"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); spec = BSON("$dateFromString" << BSON("dateString" << "2017-07-04T13:06:44Z" @@ -854,8 +870,10 @@ TEST_F(ExpressionDateFromStringTest, SerializesToObjectSyntax) { {"timezone", Document{{"$const", "Europe/London"_sd}}}, {"format", Document{{"$const", "%Y-%d-%mT%H:%M:%S"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); spec = BSON("$dateFromString" << BSON("dateString" << "2017-07-04T13:06:44Z" @@ -876,8 +894,10 @@ TEST_F(ExpressionDateFromStringTest, SerializesToObjectSyntax) { {"onNull", Document{{"$const", "nullDefault"_sd}}}, {"onError", Document{{"$const", "errorDefault"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } TEST_F(ExpressionDateFromStringTest, OptimizesToConstantIfAllInputsAreConstant) { @@ -1416,13 +1436,15 @@ void assertParsesAndSerializesExpression(boost::intrusive_ptr<ExpressionContextF const auto expression = Expression::parseExpression(expCtx.get(), expressionSpec, expCtx->variablesParseState); const auto expectedSerialization = Value(expectedSerializedExpressionSpec); - ASSERT_VALUE_EQ(expression->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(expression->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(expression->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(expression->serialize(), expectedSerialization); // Verify that parsed and then serialized expression is the same. ASSERT_VALUE_EQ(Expression::parseExpression( expCtx.get(), expectedSerializedExpressionSpec, expCtx->variablesParseState) - ->serialize(false), + ->serialize(), expectedSerialization); } @@ -1893,8 +1915,10 @@ TEST_F(ExpressionDateArithmeticsTest, SerializesToObject) { Document{{"startDate", Document{{"$const", Date_t{}}}}, {"unit", Document{{"$const", "day"_sd}}}, {"amount", Document{{"$const", 1}}}}}}); - ASSERT_VALUE_EQ(dateAddExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateAddExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateAddExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateAddExp->serialize(), expectedSerialization); // with timezone doc = BSON(expName << BSON("startDate" << Date_t{} << "unit" @@ -1908,8 +1932,10 @@ TEST_F(ExpressionDateArithmeticsTest, SerializesToObject) { {"unit", Document{{"$const", "day"_sd}}}, {"amount", Document{{"$const", -1}}}, {"timezone", Document{{"$const", "America/New_York"_sd}}}}}}); - ASSERT_VALUE_EQ(dateAddExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateAddExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateAddExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateAddExp->serialize(), expectedSerialization); } } diff --git a/src/mongo/db/pipeline/expression_field_path_test.cpp b/src/mongo/db/pipeline/expression_field_path_test.cpp index 9fa73620dfe..b04484c47ab 100644 --- a/src/mongo/db/pipeline/expression_field_path_test.cpp +++ b/src/mongo/db/pipeline/expression_field_path_test.cpp @@ -200,6 +200,72 @@ TEST(FieldPath, ScalarVariableWithDottedFieldPathOptimizesToConstantMissingValue ASSERT_VALUE_EQ(Value(), constantExpr->getValue()); } +TEST(FieldPath, SerializeWithRedaction) { + SerializationOptions options = SerializationOptions::kMarkIdentifiers_FOR_TEST; + + auto expCtx = ExpressionContextForTest{}; + intrusive_ptr<Expression> expression = + ExpressionFieldPath::createPathFromString(&expCtx, "bar", expCtx.variablesParseState); + ASSERT_VALUE_EQ_AUTO( // NOLINT + "\"$HASH<bar>\"", + expression->serialize(options)); + + // Repeat with a dotted path. + expression = + ExpressionFieldPath::createPathFromString(&expCtx, "a.b.c", expCtx.variablesParseState); + ASSERT_VALUE_EQ_AUTO( // NOLINT + "\"$HASH<a>.HASH<b>.HASH<c>\"", + expression->serialize(options)); + + auto expr = [&](const std::string& json) { + return Expression::parseExpression(&expCtx, fromjson(json), expCtx.variablesParseState); + }; + + // Expression with multiple field paths. + expression = expr(R"({$and: [{$gt: ["$foo", 5]}, {$lt: ["$foo", 10]}]})"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "$gt": [ + "$HASH<foo>", + { + "$const": 5 + } + ] + }, + { + "$lt": [ + "$HASH<foo>", + { + "$const": 10 + } + ] + } + ] + })", + expression->serialize(options).getDocument()); + + // Test that a variable followed by user fields is properly hashed. + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + + expression = expr(R"({$gt: ["$$ROOT.a.b", 5]})"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$gt":["$$ROOT.HASH<a>.HASH<b>","?number"]})", + expression->serialize(options).getDocument()); + + expression = expr(R"({$gt: ["$foo", "$$NOW"]})"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$gt":["$HASH<foo>","$$NOW"]})", + expression->serialize(options).getDocument()); + + // Repeat the above test with a dotted path. + expression = expr(R"({$gt: ["$foo.a.b", "$$NOW"]})"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$gt":["$HASH<foo>.HASH<a>.HASH<b>","$$NOW"]})", + expression->serialize(options).getDocument()); +} + /** The field path itself is a dependency. */ class Dependencies { public: @@ -424,7 +490,7 @@ public: ExpressionFieldPath::deprecatedCreate(&expCtx, "a.b.c"); ASSERT_BSONOBJ_BINARY_EQ(BSON("foo" << "$a.b.c"), - BSON("foo" << expression->serialize(false))); + BSON("foo" << expression->serialize())); } }; @@ -436,7 +502,7 @@ public: intrusive_ptr<Expression> expression = ExpressionFieldPath::deprecatedCreate(&expCtx, "a.b.c"); BSONArrayBuilder bab; - bab << expression->serialize(false); + bab << expression->serialize(); ASSERT_BSONOBJ_BINARY_EQ(BSON_ARRAY("$a.b.c"), bab.arr()); } }; diff --git a/src/mongo/db/pipeline/expression_find_internal.h b/src/mongo/db/pipeline/expression_find_internal.h index 68d535e5b28..36f88ded4aa 100644 --- a/src/mongo/db/pipeline/expression_find_internal.h +++ b/src/mongo/db/pipeline/expression_find_internal.h @@ -78,7 +78,7 @@ public: return visitor->visit(this); } - Value serialize(bool explain) const final { + Value serialize(const SerializationOptions& options) const final { MONGO_UNREACHABLE; } @@ -153,7 +153,7 @@ public: return visitor->visit(this); } - Value serialize(bool explain) const final { + Value serialize(const SerializationOptions& options) const final { MONGO_UNREACHABLE; } @@ -214,7 +214,7 @@ public: return visitor->visit(this); } - Value serialize(bool explain) const final { + Value serialize(const SerializationOptions& options) const final { MONGO_UNREACHABLE; } diff --git a/src/mongo/db/pipeline/expression_function.cpp b/src/mongo/db/pipeline/expression_function.cpp index c53b0004b31..7bf43ffd1c9 100644 --- a/src/mongo/db/pipeline/expression_function.cpp +++ b/src/mongo/db/pipeline/expression_function.cpp @@ -46,16 +46,18 @@ ExpressionFunction::ExpressionFunction(ExpressionContext* const expCtx, expCtx->sbeCompatible = false; } -Value ExpressionFunction::serialize(bool explain) const { - MutableDocument d; - d["body"] = Value(_funcSource); - d["args"] = Value(_passedArgs->serialize(explain)); - d["lang"] = Value(_lang); +Value ExpressionFunction::serialize(const SerializationOptions& options) const { + MutableDocument innerOpts(Document{{"body"_sd, options.serializeLiteral(_funcSource)}, + {"args"_sd, _passedArgs->serialize(options)}, + // "lang" is purposefully not treated as a literal since it + // is more of a selection of an enum + {"lang"_sd, _lang}}); + // This field will only be seralized when desugaring $where in $expr + $_internalJs if (_assignFirstArgToThis) { - d["_internalSetObjToThis"] = Value(_assignFirstArgToThis); + innerOpts["_internalSetObjToThis"] = options.serializeLiteral(_assignFirstArgToThis); } - return Value(Document{{kExpressionName, d.freezeToValue()}}); + return Value(Document{{kExpressionName, innerOpts.freezeToValue()}}); } void ExpressionFunction::_doAddDependencies(mongo::DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/expression_function.h b/src/mongo/db/pipeline/expression_function.h index b8d6b95e35a..a4d50dcb588 100644 --- a/src/mongo/db/pipeline/expression_function.h +++ b/src/mongo/db/pipeline/expression_function.h @@ -69,7 +69,7 @@ public: Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); diff --git a/src/mongo/db/pipeline/expression_function_test.cpp b/src/mongo/db/pipeline/expression_function_test.cpp new file mode 100644 index 00000000000..aa631a53c77 --- /dev/null +++ b/src/mongo/db/pipeline/expression_function_test.cpp @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/exec/document_value/document_value_test_util.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/pipeline/expression_function.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/dbtests/dbtests.h" + +namespace mongo { + +namespace { + +TEST(ExpressionFunction, SerializeAndRedactArgs) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + + auto expCtx = ExpressionContextForTest(); + auto expr = BSON("$function" << BSON("body" + << "function(age) {return age >= 21;}" + << "args" << BSON_ARRAY("$age") << "lang" + << "js")); + VariablesParseState vps = expCtx.variablesParseState; + auto exprFunc = ExpressionFunction::parse(&expCtx, expr.firstElement(), vps); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$function":{"body":"?string","args":["$HASH<age>"],"lang":"js"}})", + exprFunc->serialize(options).getDocument()); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/pipeline/expression_js_emit.cpp b/src/mongo/db/pipeline/expression_js_emit.cpp index ca6ba238c6e..dd43f587016 100644 --- a/src/mongo/db/pipeline/expression_js_emit.cpp +++ b/src/mongo/db/pipeline/expression_js_emit.cpp @@ -130,10 +130,10 @@ boost::intrusive_ptr<Expression> ExpressionInternalJsEmit::parse(ExpressionConte return new ExpressionInternalJsEmit(expCtx, std::move(thisRef), std::move(funcSourceString)); } -Value ExpressionInternalJsEmit::serialize(bool explain) const { +Value ExpressionInternalJsEmit::serialize(const SerializationOptions& options) const { return Value( Document{{kExpressionName, - Document{{"eval", _funcSource}, {"this", _thisRef->serialize(explain)}}}}); + Document{{"eval", _funcSource}, {"this", _thisRef->serialize(options)}}}}); } Value ExpressionInternalJsEmit::evaluate(const Document& root, Variables* variables) const { diff --git a/src/mongo/db/pipeline/expression_js_emit.h b/src/mongo/db/pipeline/expression_js_emit.h index e19907de84e..5202582d8ee 100644 --- a/src/mongo/db/pipeline/expression_js_emit.h +++ b/src/mongo/db/pipeline/expression_js_emit.h @@ -55,7 +55,7 @@ public: Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); diff --git a/src/mongo/db/pipeline/expression_let_test.cpp b/src/mongo/db/pipeline/expression_let_test.cpp new file mode 100644 index 00000000000..6b851f18523 --- /dev/null +++ b/src/mongo/db/pipeline/expression_let_test.cpp @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/exec/document_value/document_value_test_util.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace ExpressionTests { +namespace { + +TEST(RedactionTest, ExpressionLet) { + SerializationOptions options = SerializationOptions::kMarkIdentifiers_FOR_TEST; + + auto expCtx = ExpressionContextForTest{}; + + auto expression = Expression::parseExpression(&expCtx, + fromjson(R"( + {$let: {vars: {foo: 35}, in: {$gt: ["$$foo", 23]}}} + )"), + expCtx.variablesParseState); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$let": { + "vars": { + "HASH<foo>": { + "$const": 35 + } + }, + "in": { + "$gt": [ + "$$HASH<foo>", + { + "$const": 23 + } + ] + } + } + })", + expression->serialize(options).getDocument()); + + expression = Expression::parseExpression(&expCtx, + fromjson(R"( + {$let: {vars: {foo: 35, myNow: "$$NOW", obj: {hello: 23}}, in: {$gt: ["$$myNow", "$$obj.hello"]}}} + )"), + expCtx.variablesParseState); + + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$let": { + "vars": { + "HASH<foo>": { + "$const": 35 + }, + "HASH<myNow>": "$$NOW", + "HASH<obj>": { + "HASH<hello>": { + "$const": 23 + } + } + }, + "in": { + "$gt": [ + "$$HASH<myNow>", + "$$HASH<obj>.HASH<hello>" + ] + } + } + })", + expression->serialize(options).getDocument()); +} +} // namespace +} // namespace ExpressionTests +} // namespace mongo diff --git a/src/mongo/db/pipeline/expression_nary_test.cpp b/src/mongo/db/pipeline/expression_nary_test.cpp index 15b8e824f7c..cc01d807462 100644 --- a/src/mongo/db/pipeline/expression_nary_test.cpp +++ b/src/mongo/db/pipeline/expression_nary_test.cpp @@ -120,7 +120,7 @@ static BSONObj constify(const BSONObj& obj, bool parentIsArray = false) { /** Convert Expression to BSON. */ static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } class ExpressionBaseTest : public unittest::Test { @@ -249,15 +249,59 @@ TEST_F(ExpressionNaryTest, ValidateObjectExpressionDependency) { TEST_F(ExpressionNaryTest, SerializationToBsonObj) { _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(5))); ASSERT_BSONOBJ_EQ(BSON("foo" << BSON("$testable" << BSON_ARRAY(BSON("$const" << 5)))), - BSON("foo" << _notAssociativeNorCommutative->serialize(false))); + BSON("foo" << _notAssociativeNorCommutative->serialize())); } TEST_F(ExpressionNaryTest, SerializationToBsonArr) { _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(5))); ASSERT_BSONOBJ_EQ(constify(BSON_ARRAY(BSON("$testable" << BSON_ARRAY(5)))), - BSON_ARRAY(_notAssociativeNorCommutative->serialize(false))); + BSON_ARRAY(_notAssociativeNorCommutative->serialize())); } +TEST_F(ExpressionNaryTest, RedactsCorrectlyWithConstantArguments) { + _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(5))); + _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(10))); + _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(15))); + + SerializationOptions opts; + + // The default shape should wrap the constants in $const. + ASSERT_BSONOBJ_EQ( + BSON("foo" << BSON("$testable" << BSON_ARRAY(BSON("$const" << 5) << BSON("$const" << 10) + << BSON("$const" << 15)))), + BSON("foo" << _notAssociativeNorCommutative->serialize(opts))); + + // The representative shape should be an array of raw constants (i.e. not wrapped in $const). + opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue; + ASSERT_BSONOBJ_EQ(BSON("foo" << BSON("$testable" << BSON_ARRAY(1 << 1 << 1))), + BSON("foo" << _notAssociativeNorCommutative->serialize(opts))); +} + +TEST_F(ExpressionNaryTest, RedactsCorrectlyWithMixedArguments) { + VariablesParseState vps = expCtx.variablesParseState; + _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(5))); + _notAssociativeNorCommutative->addOperand( + Expression::parseExpression(&expCtx, BSON("$sum" << BSON_ARRAY(1 << 2)), vps)); + _notAssociativeNorCommutative->addOperand(ExpressionFieldPath::parse(&expCtx, "$b", vps)); + + SerializationOptions opts; + + // The default shape should wrap the constants in $const. + ASSERT_BSONOBJ_EQ(BSON("foo" << BSON("$testable" << BSON_ARRAY( + BSON("$const" << 5) + << BSON("$sum" << BSON_ARRAY(BSON("$const" << 1) + << BSON("$const" << 2))) + << "$b"))), + BSON("foo" << _notAssociativeNorCommutative->serialize(opts))); + + // The representative shape should not wrap the constant in $const. + opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue; + ASSERT_BSONOBJ_EQ(BSON("foo" << BSON("$testable" << BSON_ARRAY( + 1 << BSON("$sum" << BSON_ARRAY(1 << 1)) << "$b"))), + BSON("foo" << _notAssociativeNorCommutative->serialize(opts))); +} + + // Verify that the internal operands are optimized TEST_F(ExpressionNaryTest, InternalOperandOptimizationIsDone) { BSONArray spec = BSON_ARRAY(BSON("$and" << BSONArray()) << "$abc"); diff --git a/src/mongo/db/pipeline/expression_object_test.cpp b/src/mongo/db/pipeline/expression_object_test.cpp index 7246e9569e3..b308b94efaa 100644 --- a/src/mongo/db/pipeline/expression_object_test.cpp +++ b/src/mongo/db/pipeline/expression_object_test.cpp @@ -64,7 +64,7 @@ TEST(ExpressionObjectParse, ShouldAcceptEmptyObject) { auto expCtx = ExpressionContextForTest{}; VariablesParseState vps = expCtx.variablesParseState; auto object = ExpressionObject::parse(&expCtx, BSONObj(), vps); - ASSERT_VALUE_EQ(Value(Document{}), object->serialize(false)); + ASSERT_VALUE_EQ(Value(Document{}), object->serialize()); } TEST(ExpressionObjectParse, ShouldAcceptLiteralsAsValues) { @@ -77,7 +77,7 @@ TEST(ExpressionObjectParse, ShouldAcceptLiteralsAsValues) { vps); auto expectedResult = Value(Document{{"a", literal(5)}, {"b", literal("string"_sd)}, {"c", literal(BSONNULL)}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ExpressionObjectParse, ShouldAccept_idAsFieldName) { @@ -85,7 +85,7 @@ TEST(ExpressionObjectParse, ShouldAccept_idAsFieldName) { VariablesParseState vps = expCtx.variablesParseState; auto object = ExpressionObject::parse(&expCtx, BSON("_id" << 5), vps); auto expectedResult = Value(Document{{"_id", literal(5)}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ExpressionObjectParse, ShouldAcceptFieldNameContainingDollar) { @@ -93,7 +93,7 @@ TEST(ExpressionObjectParse, ShouldAcceptFieldNameContainingDollar) { VariablesParseState vps = expCtx.variablesParseState; auto object = ExpressionObject::parse(&expCtx, BSON("a$b" << 5), vps); auto expectedResult = Value(Document{{"a$b", literal(5)}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ExpressionObjectParse, ShouldAcceptNestedObjects) { @@ -104,7 +104,7 @@ TEST(ExpressionObjectParse, ShouldAcceptNestedObjects) { auto expectedResult = Value(Document{{"a", Document{{"b", literal(1)}}}, {"c", Document{{"d", Document{{"e", literal(1)}, {"f", literal(1)}}}}}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ExpressionObjectParse, ShouldAcceptArrays) { @@ -113,15 +113,14 @@ TEST(ExpressionObjectParse, ShouldAcceptArrays) { auto object = ExpressionObject::parse(&expCtx, fromjson("{a: [1, 2]}"), vps); auto expectedResult = Value(Document{{"a", vector<Value>{Value(literal(1)), Value(literal(2))}}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ObjectParsing, ShouldAcceptExpressionAsValue) { auto expCtx = ExpressionContextForTest{}; VariablesParseState vps = expCtx.variablesParseState; auto object = ExpressionObject::parse(&expCtx, BSON("a" << BSON("$and" << BSONArray())), vps); - ASSERT_VALUE_EQ(object->serialize(false), - Value(Document{{"a", Document{{"$and", BSONArray()}}}})); + ASSERT_VALUE_EQ(object->serialize(), Value(Document{{"a", Document{{"$and", BSONArray()}}}})); } // diff --git a/src/mongo/db/pipeline/expression_or_test.cpp b/src/mongo/db/pipeline/expression_or_test.cpp index ca4f5fdf843..00d8a9deb74 100644 --- a/src/mongo/db/pipeline/expression_or_test.cpp +++ b/src/mongo/db/pipeline/expression_or_test.cpp @@ -77,7 +77,7 @@ static BSONObj toBson(const Value& value) { /** Convert Expression to BSON. */ static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } /** Convert Document to BSON. */ diff --git a/src/mongo/db/pipeline/expression_test.cpp b/src/mongo/db/pipeline/expression_test.cpp index 109fc4acf16..591225258e2 100644 --- a/src/mongo/db/pipeline/expression_test.cpp +++ b/src/mongo/db/pipeline/expression_test.cpp @@ -44,6 +44,7 @@ #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/collation/collator_interface_mock.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/dbtests/dbtests.h" #include "mongo/idl/server_parameter_test_util.h" #include "mongo/logv2/log.h" @@ -123,7 +124,7 @@ static BSONObj toBson(const Value& value) { /** Convert Expression to BSON. */ static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } /** Convert Document to BSON. */ @@ -169,7 +170,6 @@ void parseAndVerifyResults( ASSERT_VALUE_EQ(expr->evaluate({}, &expCtx.variables), expected); } - /** * Generates a random double with a variable number of decimal places between 1 and 15. */ @@ -710,7 +710,7 @@ public: private: static BSONObj toBsonObj(const intrusive_ptr<Expression>& expression) { - return BSON("field" << expression->serialize(false)); + return BSON("field" << expression->serialize()); } }; @@ -729,7 +729,7 @@ public: private: static BSONArray toBsonArray(const intrusive_ptr<Expression>& expression) { BSONArrayBuilder bab; - bab << expression->serialize(false); + bab << expression->serialize(); return bab.arr(); } }; @@ -805,7 +805,7 @@ public: private: static BSONObj toBsonObj(const intrusive_ptr<Expression>& expression) { - return BSON("field" << expression->serialize(false)); + return BSON("field" << expression->serialize()); } }; @@ -822,7 +822,7 @@ public: private: static BSONObj toBsonArray(const intrusive_ptr<Expression>& expression) { BSONArrayBuilder bab; - bab << expression->serialize(false); + bab << expression->serialize(); return bab.obj(); } }; @@ -840,7 +840,26 @@ TEST(ExpressionConstantTest, ConstantOfValueMissingSerializesToRemoveSystemVar) intrusive_ptr<Expression> expression = ExpressionConstant::create(&expCtx, Value()); ASSERT_BSONOBJ_BINARY_EQ(BSON("field" << "$$REMOVE"), - BSON("field" << expression->serialize(false))); + BSON("field" << expression->serialize())); +} + +TEST(ExpressionConstantTest, ConstantRedaction) { + SerializationOptions options; + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + + // Test that a constant is replaced. + auto expCtx = ExpressionContextForTest{}; + intrusive_ptr<Expression> expression = ExpressionConstant::create(&expCtx, Value("my_ssn"_sd)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"field":"?string"})", + BSON("field" << expression->serialize(options))); + + auto expressionBSON = BSON("$and" << BSON_ARRAY(BSON("$gt" << BSON_ARRAY("$foo" << 5)) + << BSON("$lt" << BSON_ARRAY("$foo" << 10)))); + expression = Expression::parseExpression(&expCtx, expressionBSON, expCtx.variablesParseState); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"field":{"$and":[{"$gt":["$foo","?number"]},{"$lt":["$foo","?number"]}]}})", + BSON("field" << expression->serialize(options))); } } // namespace Constant @@ -1583,7 +1602,7 @@ TEST(ParseExpression, ShouldRecognizeConstExpression) { auto resultExpression = parseExpression(BSON("$const" << 5)); auto constExpression = dynamic_cast<ExpressionConstant*>(resultExpression.get()); ASSERT_TRUE(constExpression); - ASSERT_VALUE_EQ(constExpression->serialize(false), Value(Document{{"$const", 5}})); + ASSERT_VALUE_EQ(constExpression->serialize(), Value(Document{{"$const", 5}})); } TEST(ParseExpression, ShouldRejectUnknownExpression) { @@ -1617,15 +1636,14 @@ TEST(ParseExpression, ShouldParseExpressionWithMultipleArguments) { ASSERT_TRUE(strCaseCmpExpression); vector<Value> arguments = {Value(Document{{"$const", "foo"_sd}}), Value(Document{{"$const", "FOO"_sd}})}; - ASSERT_VALUE_EQ(strCaseCmpExpression->serialize(false), - Value(Document{{"$strcasecmp", arguments}})); + ASSERT_VALUE_EQ(strCaseCmpExpression->serialize(), Value(Document{{"$strcasecmp", arguments}})); } TEST(ParseExpression, ShouldParseExpressionWithNoArguments) { auto resultExpression = parseExpression(BSON("$and" << BSONArray())); auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", vector<Value>{}}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", vector<Value>{}}})); } TEST(ParseExpression, ShouldParseExpressionWithOneArgument) { @@ -1633,7 +1651,7 @@ TEST(ParseExpression, ShouldParseExpressionWithOneArgument) { auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", arguments}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", arguments}})); } TEST(ParseExpression, ShouldAcceptArgumentWithoutArrayForVariadicExpressions) { @@ -1641,7 +1659,7 @@ TEST(ParseExpression, ShouldAcceptArgumentWithoutArrayForVariadicExpressions) { auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", arguments}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", arguments}})); } TEST(ParseExpression, ShouldAcceptArgumentWithoutArrayAsSingleArgument) { @@ -1649,7 +1667,7 @@ TEST(ParseExpression, ShouldAcceptArgumentWithoutArrayAsSingleArgument) { auto notExpression = dynamic_cast<ExpressionNot*>(resultExpression.get()); ASSERT_TRUE(notExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(notExpression->serialize(false), Value(Document{{"$not", arguments}})); + ASSERT_VALUE_EQ(notExpression->serialize(), Value(Document{{"$not", arguments}})); } TEST(ParseExpression, ShouldAcceptObjectAsSingleArgument) { @@ -1657,7 +1675,7 @@ TEST(ParseExpression, ShouldAcceptObjectAsSingleArgument) { auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", arguments}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", arguments}})); } TEST(ParseExpression, ShouldAcceptObjectInsideArrayAsSingleArgument) { @@ -1665,7 +1683,7 @@ TEST(ParseExpression, ShouldAcceptObjectInsideArrayAsSingleArgument) { auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", arguments}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", arguments}})); } } // namespace Expression @@ -1691,7 +1709,7 @@ TEST(ParseOperand, ShouldRecognizeFieldPath) { << "$field")); auto fieldPathExpression = dynamic_cast<ExpressionFieldPath*>(resultExpression.get()); ASSERT_TRUE(fieldPathExpression); - ASSERT_VALUE_EQ(fieldPathExpression->serialize(false), Value("$field"_sd)); + ASSERT_VALUE_EQ(fieldPathExpression->serialize(), Value("$field"_sd)); } TEST(ParseOperand, ShouldRecognizeStringLiteral) { @@ -1699,7 +1717,7 @@ TEST(ParseOperand, ShouldRecognizeStringLiteral) { << "foo")); auto constantExpression = dynamic_cast<ExpressionConstant*>(resultExpression.get()); ASSERT_TRUE(constantExpression); - ASSERT_VALUE_EQ(constantExpression->serialize(false), Value(Document{{"$const", "foo"_sd}})); + ASSERT_VALUE_EQ(constantExpression->serialize(), Value(Document{{"$const", "foo"_sd}})); } TEST(ParseOperand, ShouldRecognizeNestedArray) { @@ -1709,21 +1727,21 @@ TEST(ParseOperand, ShouldRecognizeNestedArray) { ASSERT_TRUE(arrayExpression); vector<Value> expectedSerializedArray = {Value(Document{{"$const", "foo"_sd}}), Value("$field"_sd)}; - ASSERT_VALUE_EQ(arrayExpression->serialize(false), Value(expectedSerializedArray)); + ASSERT_VALUE_EQ(arrayExpression->serialize(), Value(expectedSerializedArray)); } TEST(ParseOperand, ShouldRecognizeNumberLiteral) { auto resultExpression = parseOperand(BSON("" << 5)); auto constantExpression = dynamic_cast<ExpressionConstant*>(resultExpression.get()); ASSERT_TRUE(constantExpression); - ASSERT_VALUE_EQ(constantExpression->serialize(false), Value(Document{{"$const", 5}})); + ASSERT_VALUE_EQ(constantExpression->serialize(), Value(Document{{"$const", 5}})); } TEST(ParseOperand, ShouldRecognizeNestedExpression) { auto resultExpression = parseOperand(BSON("" << BSON("$and" << BSONArray()))); auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", vector<Value>{}}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", vector<Value>{}}})); } } // namespace Operand @@ -1764,7 +1782,7 @@ public: if (ValueComparator().evaluate(result != expected)) { string errMsg = str::stream() << "for expression " << field.first.toString() << " with argument " - << args.toString() << " full tree: " << expr->serialize(false).toString() + << args.toString() << " full tree: " << expr->serialize().toString() << " expected: " << expected.toString() << " but got: " << result.toString(); FAIL(errMsg); @@ -2576,7 +2594,7 @@ TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectly) { auto expression = ExpressionFieldPath::parse(&expCtx, "$$REMOVE", vps); ASSERT_BSONOBJ_EQ(BSON("foo" << "$$REMOVE"), - BSON("foo" << expression->serialize(false))); + BSON("foo" << expression->serialize())); } TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectlyWithTrailingPath) { @@ -2585,7 +2603,7 @@ TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectlyWithTrailingPath) { auto expression = ExpressionFieldPath::parse(&expCtx, "$$REMOVE.a.b", vps); ASSERT_BSONOBJ_EQ(BSON("foo" << "$$REMOVE.a.b"), - BSON("foo" << expression->serialize(false))); + BSON("foo" << expression->serialize())); } TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectlyAfterOptimization) { @@ -2596,7 +2614,7 @@ TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectlyAfterOptimization) { ASSERT(dynamic_cast<ExpressionConstant*>(optimizedExpression.get())); ASSERT_BSONOBJ_EQ(BSON("foo" << "$$REMOVE"), - BSON("foo" << optimizedExpression->serialize(false))); + BSON("foo" << optimizedExpression->serialize())); } } // namespace BuiltinRemoveVariable @@ -2814,7 +2832,7 @@ public: if (ValueComparator().evaluate(result != expected)) { string errMsg = str::stream() << "for expression " << field.first.toString() << " with argument " - << args.toString() << " full tree: " << expr->serialize(false).toString() + << args.toString() << " full tree: " << expr->serialize().toString() << " expected: " << expected.toString() << " but got: " << result.toString(); FAIL(errMsg); @@ -3505,7 +3523,7 @@ TEST(ExpressionRandom, Basic) { intrusive_ptr<Expression> expression = Expression::parseExpression(&expCtx, fromjson("{ $rand: {} }"), vps); - const std::string& serialized = expression->serialize(false).getDocument().toString(); + const std::string& serialized = expression->serialize().getDocument().toString(); ASSERT_EQ("{$rand: {}}", serialized); const auto randFn = [&expression, &expCtx]() -> double { @@ -3642,11 +3660,18 @@ TEST(ExpressionGetFieldTest, GetFieldSerializesStringArgumentCorrectly) { VariablesParseState vps = expCtx.variablesParseState; BSONObj expr = fromjson("{$meta: \"foo\"}"); auto expression = ExpressionGetField::parse(&expCtx, expr.firstElement(), vps); - ASSERT_BSONOBJ_EQ(BSON("ignoredField" << BSON("$getField" << BSON("field" << BSON("$const" - << "foo") - << "input" - << "$$CURRENT"))), - BSON("ignoredField" << expression->serialize(false))); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "ignoredField": { + "$getField": { + "field": { + "$const": "foo" + }, + "input": "$$CURRENT" + } + } + })", + BSON("ignoredField" << expression->serialize())); } TEST(ExpressionGetFieldTest, GetFieldSerializesCorrectly) { @@ -3654,12 +3679,220 @@ TEST(ExpressionGetFieldTest, GetFieldSerializesCorrectly) { VariablesParseState vps = expCtx.variablesParseState; BSONObj expr = fromjson("{$meta: {\"field\": \"foo\", \"input\": {a: 1}}}"); auto expression = ExpressionGetField::parse(&expCtx, expr.firstElement(), vps); - ASSERT_BSONOBJ_EQ( - BSON("ignoredField" << BSON( - "$getField" << BSON("field" << BSON("$const" - << "foo") - << "input" << BSON("a" << BSON("$const" << 1))))), - BSON("ignoredField" << expression->serialize(false))); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "ignoredField": { + "$getField": { + "field": { + "$const": "foo" + }, + "input": { + "a": { + "$const": 1 + } + } + } + } + })", + BSON("ignoredField" << expression->serialize())); +} + +TEST(ExpressionGetFieldTest, GetFieldSerializesAndRedactsCorrectly) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto expCtx = ExpressionContextForTest{}; + VariablesParseState vps = expCtx.variablesParseState; + + BSONObj expressionBSON = BSON("$getField" << BSON("field" + << "a" + << "input" + << "$b")); + + auto expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"field":{"$getField":{"field":"HASH<a>","input":"$HASH<b>"}}})", + BSON("field" << expression->serialize(options))); + + // Test the shorthand syntax. + expressionBSON = BSON("$getField" + << "a"); + + expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"field":{"$getField":{"field":"HASH<a>","input":"$$CURRENT"}}})", + BSON("field" << expression->serialize(options))); + + // Test a field with '.' characters. + expressionBSON = BSON("$getField" + << "a.b.c"); + + expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$getField": { + "field": "HASH<a>.HASH<b>.HASH<c>", + "input": "$$CURRENT" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Test a field with a '$' character. + expressionBSON = BSON("$getField" + << "a.$b.c"); + + expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$getField": { + "field": "HASH<a>.HASH<$b>.HASH<c>", + "input": "$$CURRENT" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Test a field with a trailing '.' character (invalid FieldPath). + expressionBSON = BSON("$getField" + << "a.b.c."); + + expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$getField": { + "field": "HASH<invalidFieldPathPlaceholder>", + "input": "$$CURRENT" + } + } + })", + BSON("field" << expression->serialize(options))); +} + +TEST(ExpressionSetFieldTest, SetFieldRedactsCorrectly) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto expCtx = ExpressionContextForTest{}; + VariablesParseState vps = expCtx.variablesParseState; + + // Test that a set field redacts properly. + BSONObj expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" + << "$b" + << "value" + << "$c")); + auto expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "$HASH<b>", + "value": "$HASH<c>" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Object as input. + expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" << BSON("a" << true) << "value" << 10)); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "?object", + "value": "?number" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Nested object as input. + expressionBSON = + BSON("$setField" << BSON("field" + << "a" + << "input" << BSON("a" << BSON("b" << 5)) << "value" << 10)); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "?object", + "value": "?number" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Object with field path in input. + expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" + << BSON("a" + << "$field") + << "value" << 10)); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": { + "HASH<a>": "$HASH<field>" + }, + "value": "?number" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Object with field path in value. + expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" + << BSON("a" + << "b") + << "value" + << BSON("c" + << "$d"))); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "?object", + "value": { + "HASH<c>": "$HASH<d>" + } + } + } + })", + BSON("field" << expression->serialize(options))); + + // Array as input. + expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" << BSON("a" << BSON_ARRAY(3 << 4 << 5)) + << "value" << 10)); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "?object", + "value": "?number" + } + } + })", + BSON("field" << expression->serialize(options))); } TEST(ExpressionSetFieldTest, SetFieldSerializesCorrectly) { @@ -3673,7 +3906,20 @@ TEST(ExpressionSetFieldTest, SetFieldSerializesCorrectly) { << "foo") << "input" << BSON("a" << BSON("$const" << 1)) << "value" << BSON("$const" << 24)))), - BSON("ignoredField" << expression->serialize(false))); + BSON("ignoredField" << expression->serialize())); +} + +TEST(ExpressionSetFieldTest, SetFieldRejectsNullCharInFieldArgument) { + auto expCtx = ExpressionContextForTest{}; + VariablesParseState vps = expCtx.variablesParseState; + auto fieldExpr = make_intrusive<ExpressionConstant>(&expCtx, Value("ab\0c"_sd)); + auto inputExpr = make_intrusive<ExpressionConstant>(&expCtx, Value(BSON("a" << 1))); + auto valueExpr = make_intrusive<ExpressionConstant>(&expCtx, Value(true)); + ASSERT_THROWS_CODE( + make_intrusive<ExpressionSetField>( + &expCtx, std::move(fieldExpr), std::move(inputExpr), std::move(valueExpr)), + AssertionException, + 9534700); } TEST(ExpressionIfNullTest, OptimizedExpressionIfNullShouldRemoveNullConstant) { @@ -3693,7 +3939,7 @@ TEST(ExpressionIfNullTest, auto expr = fromjson("{$ifNull: [null, \"$a\"]}"); auto exprIfNull = ExpressionIfNull::parse(&expCtx, expr.firstElement(), vps); auto optimizedNullRemoved = exprIfNull->optimize(); - ASSERT_VALUE_EQ(optimizedNullRemoved->serialize(false), Value("$a"_sd)); + ASSERT_VALUE_EQ(optimizedNullRemoved->serialize(), Value("$a"_sd)); } TEST(ExpressionIfNullTest, OptimizedExpressionIfNullShouldRemoveAllNullConstantsButLast) { @@ -4025,7 +4271,7 @@ TEST(ExpressionFLETest, TestBinData_RoundTrip) { ASSERT_VALUE_EQ(exprFle->evaluate({}, &expCtx.variables), Value(true)); // Verify it round trips - auto value = exprFle->serialize(false); + auto value = exprFle->serialize(); auto roundTripExpr = fromjson(R"({$_internalFleEq: { field: { @@ -4050,7 +4296,7 @@ TEST(ExpressionFLETest, TestBinData_RoundTrip) { "subType": "6" } } - } })"); + } })"); ASSERT_BSONOBJ_EQ(value.getDocument().toBson(), roundTripExpr); diff --git a/src/mongo/db/pipeline/expression_test_api_version.cpp b/src/mongo/db/pipeline/expression_test_api_version.cpp index 893ddf301be..08938f1afa8 100644 --- a/src/mongo/db/pipeline/expression_test_api_version.cpp +++ b/src/mongo/db/pipeline/expression_test_api_version.cpp @@ -89,7 +89,7 @@ boost::intrusive_ptr<Expression> ExpressionTestApiVersion::parse(ExpressionConte return new ExpressionTestApiVersion(expCtx, unstableField, deprecatedField); } -Value ExpressionTestApiVersion::serialize(bool explain) const { +Value ExpressionTestApiVersion::serialize(const SerializationOptions& options) const { return Value(Document{{"$_testApiVersion", Document{{"unstable", _unstable ? Value(_unstable) : Value()}, {"deprecated", _deprecated ? Value(_deprecated) : Value()}}}}); diff --git a/src/mongo/db/pipeline/expression_test_api_version.h b/src/mongo/db/pipeline/expression_test_api_version.h index cd5436d61b3..0b67556efbe 100644 --- a/src/mongo/db/pipeline/expression_test_api_version.h +++ b/src/mongo/db/pipeline/expression_test_api_version.h @@ -48,7 +48,7 @@ public: Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); diff --git a/src/mongo/db/pipeline/expression_test_api_version_test.cpp b/src/mongo/db/pipeline/expression_test_api_version_test.cpp index 05c9156ddf0..231a47fbc27 100644 --- a/src/mongo/db/pipeline/expression_test_api_version_test.cpp +++ b/src/mongo/db/pipeline/expression_test_api_version_test.cpp @@ -48,7 +48,7 @@ TEST_F(TestApiVersion, UnstableAcceptsBooleanValue) { expCtx->variablesParseState); ASSERT_VALUE_EQ(Value(DOC("$_testApiVersion" << DOC("unstable" << true))), - expression->serialize(false)); + expression->serialize()); } TEST_F(TestApiVersion, UnstableDoesNotAcceptNumericValue) { @@ -70,7 +70,7 @@ TEST_F(TestApiVersion, DeprecatedAcceptsBooleanValue) { expCtx->variablesParseState); ASSERT_VALUE_EQ(Value(DOC("$_testApiVersion" << DOC("deprecated" << true))), - expression->serialize(false)); + expression->serialize()); } TEST_F(TestApiVersion, DeprecatedDoesNotAcceptNumericValue) { diff --git a/src/mongo/db/pipeline/expression_trim_test.cpp b/src/mongo/db/pipeline/expression_trim_test.cpp index 620b371c5f7..bf5a5d63302 100644 --- a/src/mongo/db/pipeline/expression_trim_test.cpp +++ b/src/mongo/db/pipeline/expression_trim_test.cpp @@ -722,14 +722,16 @@ TEST(ExpressionTrimTest, DoesSerializeCorrectly) { BSON("$trim" << BSON("input" << " abc ")), expCtx.variablesParseState); - ASSERT_VALUE_EQ(trim->serialize(false), trim->serialize(true)); + ASSERT_VALUE_EQ(trim->serialize(), + trim->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})); ASSERT_VALUE_EQ( - trim->serialize(false), + trim->serialize(), Value(Document{{"$trim", Document{{"input", Document{{"$const", " abc "_sd}}}}}})); // Make sure we can re-parse it and evaluate it. auto reparsedTrim = Expression::parseExpression( - &expCtx, trim->serialize(false).getDocument().toBson(), expCtx.variablesParseState); + &expCtx, trim->serialize().getDocument().toBson(), expCtx.variablesParseState); ASSERT_VALUE_EQ(reparsedTrim->evaluate({}, &expCtx.variables), Value("abc"_sd)); // Use $ltrim, and specify the 'chars' option. @@ -740,12 +742,12 @@ TEST(ExpressionTrimTest, DoesSerializeCorrectly) { << "$$CURRENT.a")), expCtx.variablesParseState); ASSERT_VALUE_EQ( - trim->serialize(false), + trim->serialize(), Value(Document{{"$ltrim", Document{{"input", "$inputField"_sd}, {"chars", "$a"_sd}}}})); // Make sure we can re-parse it and evaluate it. reparsedTrim = Expression::parseExpression( - &expCtx, trim->serialize(false).getDocument().toBson(), expCtx.variablesParseState); + &expCtx, trim->serialize().getDocument().toBson(), expCtx.variablesParseState); ASSERT_VALUE_EQ(reparsedTrim->evaluate(Document{{"inputField", " , 4"_sd}, {"a", " ,"_sd}}, &expCtx.variables), Value("4"_sd)); diff --git a/src/mongo/db/pipeline/expression_walker_test.cpp b/src/mongo/db/pipeline/expression_walker_test.cpp index 36fbe505cc8..b03673c8df5 100644 --- a/src/mongo/db/pipeline/expression_walker_test.cpp +++ b/src/mongo/db/pipeline/expression_walker_test.cpp @@ -198,7 +198,7 @@ TEST_F(ExpressionWalkerTest, SubstitutePathOnlySubstitutesPrefix) { auto expression = parseExpression("{$concat: ['$a', '$b', '$a.a', '$b.a', '$$NOW']}"); walk<Expression>(expression.get(), &substituteWalker); ASSERT_BSONOBJ_EQ(fromjson("{$concat: ['$b', '$b', '$b.a', '$b.a', '$$NOW']}"), - expression->serialize(false).getDocument().toBson()); + expression->serialize().getDocument().toBson()); } TEST_F(ExpressionWalkerTest, SubstitutePathSubstitutesWhenThereAreDottedFields) { @@ -207,7 +207,7 @@ TEST_F(ExpressionWalkerTest, SubstitutePathSubstitutesWhenThereAreDottedFields) auto expression = parseExpression("{$concat: ['$a.b', '$a.b.c', '$c', '$d.e.f']}"); walk<Expression>(expression.get(), &substituteWalker); ASSERT_BSONOBJ_EQ(fromjson("{$concat: ['$a.b', '$x', '$q.r', '$y.f']}"), - expression->serialize(false).getDocument().toBson()); + expression->serialize().getDocument().toBson()); } TEST_F(ExpressionWalkerTest, SubstitutePathSubstitutesWhenExpressionIsNested) { @@ -217,7 +217,7 @@ TEST_F(ExpressionWalkerTest, SubstitutePathSubstitutesWhenExpressionIsNested) { parseExpression("{$multiply: [{$add: ['$a.b', '$c']}, {$ifNull: ['$a.b.c', '$d']}]}"); walk<Expression>(expression.get(), &substituteWalker); ASSERT_BSONOBJ_EQ(fromjson("{$multiply: [{$add: ['$x', '$y']}, {$ifNull: ['$x.c', '$d']}]}"), - expression->serialize(false).getDocument().toBson()); + expression->serialize().getDocument().toBson()); } TEST_F(ExpressionWalkerTest, SubstitutePathDoesNotSubstitutesWhenExpressionHasNoFieldPaths) { @@ -226,7 +226,7 @@ TEST_F(ExpressionWalkerTest, SubstitutePathDoesNotSubstitutesWhenExpressionHasNo auto expression = parseExpression("{$multiply: [1, 2, 3, 4]}"); walk<Expression>(expression.get(), &substituteWalker); ASSERT_BSONOBJ_EQ(fromjson("{$multiply: [{$const: 1}, {$const: 2}, {$const: 3}, {$const: 4}]}"), - expression->serialize(false).getDocument().toBson()); + expression->serialize().getDocument().toBson()); } } // namespace diff --git a/src/mongo/db/pipeline/field_path.cpp b/src/mongo/db/pipeline/field_path.cpp index 8839b906daf..28cd63e2c05 100644 --- a/src/mongo/db/pipeline/field_path.cpp +++ b/src/mongo/db/pipeline/field_path.cpp @@ -74,7 +74,7 @@ string FieldPath::getFullyQualifiedPath(StringData prefix, StringData suffix) { return str::stream() << prefix << "." << suffix; } -FieldPath::FieldPath(std::string inputPath) +FieldPath::FieldPath(std::string inputPath, bool validateFieldNames) : _fieldPath(std::move(inputPath)), _fieldPathDotPosition{string::npos}, _fieldHash{kHashUninitialized} { @@ -98,7 +98,10 @@ FieldPath::FieldPath(std::string inputPath) "FieldPath is too long", pathLength <= BSONDepth::getMaxAllowableDepth()); for (size_t i = 0; i < pathLength; ++i) { - uassertValidFieldName(getFieldName(i)); + const auto& fieldName = getFieldName(i); + if (validateFieldNames) { + uassertValidFieldName(fieldName); + } } } diff --git a/src/mongo/db/pipeline/field_path.h b/src/mongo/db/pipeline/field_path.h index d2ee93734e7..b8370ba7dac 100644 --- a/src/mongo/db/pipeline/field_path.h +++ b/src/mongo/db/pipeline/field_path.h @@ -69,9 +69,11 @@ public: * * Field names are validated using uassertValidFieldName(). */ - /* implicit */ FieldPath(std::string inputPath); - /* implicit */ FieldPath(StringData inputPath) : FieldPath(inputPath.toString()) {} - /* implicit */ FieldPath(const char* inputPath) : FieldPath(std::string(inputPath)) {} + /* implicit */ FieldPath(std::string inputPath, bool validateFieldNames = true); + /* implicit */ FieldPath(StringData inputPath, bool validateFieldNames = true) + : FieldPath(inputPath.toString(), validateFieldNames) {} + /* implicit */ FieldPath(const char* inputPath, bool validateFieldNames = true) + : FieldPath(std::string(inputPath), validateFieldNames) {} /** * Returns the number of path elements in the field path. @@ -134,11 +136,20 @@ public: } /** + * Returns the full path as StringData instead of const std::string&, not including the prefix + * 'FieldPath::prefix'. + */ + StringData fullPathStringData() const { + return _fieldPath; + } + + /** * Returns the full path, including the prefix 'FieldPath::prefix'. */ std::string fullPathWithPrefix() const { return prefix + _fieldPath; } + /** * A FieldPath like this but missing the first element (useful for recursion). * Precondition getPathLength() > 1. @@ -157,6 +168,14 @@ public: FieldPath concat(const FieldPath& tail) const; + bool isPrefixOf(const FieldPath& rhsPath) const { + auto lhsStr = fullPathStringData(); + auto rhsStr = rhsPath.fullPathStringData(); + return lhsStr.size() < rhsStr.size() + ? rhsStr.startsWith(lhsStr) && rhsStr[lhsStr.size()] == '.' + : lhsStr == rhsStr; + } + private: FieldPath(std::string string, std::vector<size_t> dots, std::vector<size_t> hashes) : _fieldPath(std::move(string)), diff --git a/src/mongo/db/pipeline/group_from_first_document_transformation.cpp b/src/mongo/db/pipeline/group_from_first_document_transformation.cpp index e1c711f30f3..c766bcb44d0 100644 --- a/src/mongo/db/pipeline/group_from_first_document_transformation.cpp +++ b/src/mongo/db/pipeline/group_from_first_document_transformation.cpp @@ -48,11 +48,11 @@ void GroupFromFirstDocumentTransformation::optimize() { } Document GroupFromFirstDocumentTransformation::serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const { - + boost::optional<ExplainOptions::Verbosity> explain, const SerializationOptions& options) const { MutableDocument newRoot(_accumulatorExprs.size()); + for (auto&& expr : _accumulatorExprs) { - newRoot.addField(expr.first, expr.second->serialize(static_cast<bool>(explain))); + newRoot.addField(expr.first, expr.second->serialize(options)); } return {{"newRoot", newRoot.freezeToValue()}}; diff --git a/src/mongo/db/pipeline/group_from_first_document_transformation.h b/src/mongo/db/pipeline/group_from_first_document_transformation.h index 541cd87967c..318a18cdb89 100644 --- a/src/mongo/db/pipeline/group_from_first_document_transformation.h +++ b/src/mongo/db/pipeline/group_from_first_document_transformation.h @@ -70,8 +70,8 @@ public: void optimize() final; - Document serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const final; + Document serializeTransformation(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options = {}) const final; DepsTracker::State addDependencies(DepsTracker* deps) const final; diff --git a/src/mongo/db/pipeline/javascript_execution.cpp b/src/mongo/db/pipeline/javascript_execution.cpp index f9ce429ba76..8b6d3bc6f9b 100644 --- a/src/mongo/db/pipeline/javascript_execution.cpp +++ b/src/mongo/db/pipeline/javascript_execution.cpp @@ -47,22 +47,37 @@ JsExecution* JsExecution::get(OperationContext* opCtx, StringData database, bool loadStoredProcedures, boost::optional<int> jsHeapLimitMB) { + // If a JsExecution object has already been created, return it. + JsExecution* jsExec = getCached(opCtx, loadStoredProcedures); + if (jsExec) { + return jsExec; + } + + // There is no cached JsExecution object, so create and cache one now. auto& exec = getExec(opCtx); - if (!exec) { - exec = std::make_unique<JsExecution>(opCtx, scope, jsHeapLimitMB); - exec->getScope()->setLocalDB(database); - if (loadStoredProcedures) { - exec->getScope()->loadStored(opCtx, true); - } - exec->_storedProceduresLoaded = loadStoredProcedures; - } else { - uassert(31438, - "A single operation cannot use both JavaScript aggregation expressions and $where.", - loadStoredProcedures == exec->_storedProceduresLoaded); + exec = std::make_unique<JsExecution>(opCtx, scope, jsHeapLimitMB); + exec->getScope()->setLocalDB(database); + if (loadStoredProcedures) { + exec->getScope()->loadStored(opCtx, true); } + exec->_storedProceduresLoaded = loadStoredProcedures; + return exec.get(); } +JsExecution* JsExecution::getCached(OperationContext* opCtx, bool loadStoredProcedures) { + auto& exec = getExec(opCtx); + if (exec) { + if (loadStoredProcedures == exec->_storedProceduresLoaded) { + return exec.get(); + } + tasserted( + 9136200, + "A single operation cannot use both JavaScript aggregation expressions and $where."); + } + return nullptr; +} + Value JsExecution::callFunction(ScriptingFunction func, const BSONObj& params, const BSONObj& thisObj) { diff --git a/src/mongo/db/pipeline/javascript_execution.h b/src/mongo/db/pipeline/javascript_execution.h index d95e90f6eea..a66dee6c6ca 100644 --- a/src/mongo/db/pipeline/javascript_execution.h +++ b/src/mongo/db/pipeline/javascript_execution.h @@ -58,6 +58,15 @@ public: StringData database, bool loadStoredProcedures, boost::optional<int> jsHeapLimitMB); + + /** + * Gets a pointer to the cached JsExecution instance if it exists, else returns nullptr. This + * allows the caller to skip passing all the arguments to the more general JsExecution::get() + * method, which avoids having to potentially construct and create a heap copy of a large scope + * object that will not be used because a cached JsExecution object already exists. + */ + static JsExecution* getCached(OperationContext* opCtx, bool loadStoredProcedures); + /** * Construct with a thread-local scope and initialize with the given scope variables. */ diff --git a/src/mongo/db/pipeline/lite_parsed_pipeline.h b/src/mongo/db/pipeline/lite_parsed_pipeline.h index 2c424a84ee3..279135ece43 100644 --- a/src/mongo/db/pipeline/lite_parsed_pipeline.h +++ b/src/mongo/db/pipeline/lite_parsed_pipeline.h @@ -37,6 +37,7 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/pipeline/aggregate_command_gen.h" #include "mongo/db/pipeline/lite_parsed_document_source.h" +#include "mongo/db/query/util/deferred.h" #include "mongo/db/read_concern_support_result.h" namespace mongo { @@ -66,14 +67,8 @@ public: /** * Returns all foreign namespaces referenced by stages within this pipeline, if any. */ - stdx::unordered_set<NamespaceString> getInvolvedNamespaces() const { - stdx::unordered_set<NamespaceString> involvedNamespaces; - for (auto&& spec : _stageSpecs) { - auto stagesInvolvedNamespaces = spec->getInvolvedNamespaces(); - involvedNamespaces.insert(stagesInvolvedNamespaces.begin(), - stagesInvolvedNamespaces.end()); - } - return involvedNamespaces; + const stdx::unordered_set<NamespaceString>& getInvolvedNamespaces() const { + return _involvedNamespaces.get(_stageSpecs); } /** @@ -133,9 +128,7 @@ public: * Returns true if the pipeline has a $changeStream stage. */ bool hasChangeStream() const { - return std::any_of(_stageSpecs.begin(), _stageSpecs.end(), [](auto&& spec) { - return spec->isChangeStream(); - }); + return _hasChangeStream.get(_stageSpecs); } /** @@ -226,7 +219,25 @@ public: void validate(const OperationContext* opCtx, bool performApiVersionChecks = true) const; private: + // This is logically const - any changes to _stageSpecs will invalidate cached copies of + // "_hasChangeStream" and "_involvedNamespaces" below. std::vector<std::unique_ptr<LiteParsedDocumentSource>> _stageSpecs; + Deferred<bool, const decltype(_stageSpecs)&> _hasChangeStream{[](const auto& stageSpecs) { + return std::any_of(stageSpecs.begin(), stageSpecs.end(), [](auto&& spec) { + return spec->isChangeStream(); + }); + }}; + + Deferred<stdx::unordered_set<NamespaceString>, const decltype(_stageSpecs)&> + _involvedNamespaces{[](const auto& stageSpecs) -> stdx::unordered_set<NamespaceString> { + stdx::unordered_set<NamespaceString> involvedNamespaces; + for (const auto& spec : stageSpecs) { + auto stagesInvolvedNamespaces = spec->getInvolvedNamespaces(); + involvedNamespaces.insert(stagesInvolvedNamespaces.begin(), + stagesInvolvedNamespaces.end()); + } + return involvedNamespaces; + }}; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp index b916c59728f..30dbcba6290 100644 --- a/src/mongo/db/pipeline/pipeline.cpp +++ b/src/mongo/db/pipeline/pipeline.cpp @@ -305,6 +305,11 @@ void Pipeline::validateCommon(bool alreadyOptimized) const { str::stream() << stage->getSourceName() << " can only be used once in the pipeline", !(constraints.canAppearOnlyOnceInPipeline && !singleUseStages.insert(stage->getSourceName()).second)); + + tassert(7355707, + "If a stage is broadcast to all shard servers then it must be a data source.", + constraints.hostRequirement != HostTypeRequirement::kAllShardServers || + !constraints.requiresInputDocSource); } } @@ -313,21 +318,29 @@ void Pipeline::optimizePipeline() { if (MONGO_unlikely(disablePipelineOptimization.shouldFail())) { return; } - optimizeContainer(&_sources); + optimizeEachStage(&_sources); } void Pipeline::optimizeContainer(SourceContainer* container) { - SourceContainer optimizedSources; - SourceContainer::iterator itr = container->begin(); try { while (itr != container->end()) { invariant((*itr).get()); itr = (*itr).get()->optimizeAt(itr, container); } + } catch (DBException& ex) { + ex.addContext("Failed to optimize pipeline"); + throw; + } - // Once we have reached our final number of stages, optimize each individually. + stitch(container); +} + +void Pipeline::optimizeEachStage(SourceContainer* container) { + SourceContainer optimizedSources; + try { + // We should have our final number of stages. Optimize each individually. for (auto&& source : *container) { if (auto out = source->optimize()) { optimizedSources.push_back(out); @@ -448,11 +461,19 @@ bool Pipeline::needsMongosMerger() const { }); } +bool Pipeline::needsAllShardServers() const { + return std::any_of(_sources.begin(), _sources.end(), [&](const auto& stage) { + return stage->constraints().resolvedHostTypeRequirement(pCtx) == + HostTypeRequirement::kAllShardServers; + }); +} + bool Pipeline::needsShard() const { return std::any_of(_sources.begin(), _sources.end(), [&](const auto& stage) { auto hostType = stage->constraints().resolvedHostTypeRequirement(pCtx); return (hostType == HostTypeRequirement::kAnyShard || - hostType == HostTypeRequirement::kPrimaryShard); + hostType == HostTypeRequirement::kPrimaryShard || + hostType == HostTypeRequirement::kAllShardServers); }); } @@ -498,20 +519,20 @@ stdx::unordered_set<NamespaceString> Pipeline::getInvolvedCollections() const { } vector<Value> Pipeline::serializeContainer(const SourceContainer& container, - boost::optional<ExplainOptions::Verbosity> explain) { + boost::optional<const SerializationOptions&> opts) { vector<Value> serializedSources; for (auto&& source : container) { - source->serializeToArray(serializedSources, explain); + source->serializeToArray(serializedSources, opts ? opts.get() : SerializationOptions()); } return serializedSources; } -vector<Value> Pipeline::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return serializeContainer(_sources, explain); + +vector<Value> Pipeline::serialize(boost::optional<const SerializationOptions&> opts) const { + return serializeContainer(_sources, opts); } -vector<BSONObj> Pipeline::serializeToBson( - boost::optional<ExplainOptions::Verbosity> explain) const { - const auto serialized = serialize(explain); +vector<BSONObj> Pipeline::serializeToBson(boost::optional<const SerializationOptions&> opts) const { + const auto serialized = serialize(opts); std::vector<BSONObj> asBson; asBson.reserve(serialized.size()); for (auto&& stage : serialized) { @@ -552,16 +573,16 @@ boost::optional<Document> Pipeline::getNext() { : boost::optional<Document>{nextResult.releaseDocument()}; } -vector<Value> Pipeline::writeExplainOps(ExplainOptions::Verbosity verbosity) const { +vector<Value> Pipeline::writeExplainOps(const SerializationOptions& opts) const { vector<Value> array; for (auto&& stage : _sources) { auto beforeSize = array.size(); - stage->serializeToArray(array, verbosity); + stage->serializeToArray(array, opts); auto afterSize = array.size(); // Append execution stats to the serialized stage if the specified verbosity is // 'executionStats' or 'allPlansExecution'. invariant(afterSize - beforeSize == 1u); - if (verbosity >= ExplainOptions::Verbosity::kExecStats) { + if (*opts.verbosity >= ExplainOptions::Verbosity::kExecStats) { auto serializedStage = array.back(); array.back() = appendCommonExecStats(serializedStage, stage->getCommonStats()); } @@ -652,7 +673,8 @@ Status Pipeline::_pipelineCanRunOnMongoS() const { auto hostRequirement = constraints.resolvedHostTypeRequirement(pCtx); const bool needsShard = (hostRequirement == HostTypeRequirement::kAnyShard || - hostRequirement == HostTypeRequirement::kPrimaryShard); + hostRequirement == HostTypeRequirement::kPrimaryShard || + hostRequirement == HostTypeRequirement::kAllShardServers); const bool mustWriteToDisk = (constraints.diskRequirement == DiskUseRequirement::kWritesPersistentData); @@ -739,17 +761,31 @@ boost::intrusive_ptr<DocumentSource> Pipeline::popFrontWithNameAndCriteria( return popFront(); } +void Pipeline::appendPipeline(std::unique_ptr<Pipeline, PipelineDeleter> otherPipeline) { + auto& otherPipelineSources = otherPipeline->getSources(); + while (!otherPipelineSources.empty()) { + _sources.push_back(std::move(otherPipelineSources.front())); + otherPipelineSources.pop_front(); + } + constexpr bool alreadyOptimized = false; + validateCommon(alreadyOptimized); + stitch(); +} + + std::unique_ptr<Pipeline, PipelineDeleter> Pipeline::makePipeline( const std::vector<BSONObj>& rawPipeline, const boost::intrusive_ptr<ExpressionContext>& expCtx, const MakePipelineOptions opts) { auto pipeline = Pipeline::parse(rawPipeline, expCtx, opts.validator); + bool alreadyOptimized = opts.alreadyOptimized; + if (opts.optimize) { pipeline->optimizePipeline(); + alreadyOptimized = true; } - constexpr bool alreadyOptimized = true; pipeline->validateCommon(alreadyOptimized); if (opts.attachCursorSource) { @@ -766,30 +802,13 @@ Pipeline::SourceContainer::iterator Pipeline::optimizeEndOfPipeline( // optimize, since otherwise calls to optimizeAt() will overrun these limits. auto endOfPipeline = Pipeline::SourceContainer(std::next(itr), container->end()); Pipeline::optimizeContainer(&endOfPipeline); + Pipeline::optimizeEachStage(&endOfPipeline); container->erase(std::next(itr), container->end()); container->splice(std::next(itr), endOfPipeline); return std::next(itr); } -Pipeline::SourceContainer::iterator Pipeline::optimizeAtEndOfPipeline( - Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { - if (itr == container->end()) { - return itr; - } - itr = std::next(itr); - try { - while (itr != container->end()) { - invariant((*itr).get()); - itr = (*itr).get()->optimizeAt(itr, container); - } - } catch (DBException& ex) { - ex.addContext("Failed to optimize pipeline"); - throw; - } - return itr; -} - std::unique_ptr<Pipeline, PipelineDeleter> Pipeline::makePipelineFromViewDefinition( const boost::intrusive_ptr<ExpressionContext>& subPipelineExpCtx, ExpressionContext::ResolvedNamespace resolvedNs, diff --git a/src/mongo/db/pipeline/pipeline.h b/src/mongo/db/pipeline/pipeline.h index a3108be8fa1..367b4382941 100644 --- a/src/mongo/db/pipeline/pipeline.h +++ b/src/mongo/db/pipeline/pipeline.h @@ -45,7 +45,6 @@ #include "mongo/db/query/explain_options.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/executor/task_executor.h" -#include "mongo/s/query/async_results_merger_params_gen.h" #include "mongo/util/intrusive_counter.h" #include "mongo/util/timer.h" @@ -70,6 +69,10 @@ using PipelineValidatorCallback = std::function<void(const Pipeline&)>; struct MakePipelineOptions { bool optimize = true; + // It is assumed that the pipeline has already been optimized when we create the + // MakePipelineOptions. If this is not the case, the caller is responsible for setting + // alreadyOptimized to false. + bool alreadyOptimized = true; bool attachCursorSource = true; ShardTargetingPolicy shardTargetingPolicy = ShardTargetingPolicy::kAllowed; PipelineValidatorCallback validator = nullptr; @@ -171,16 +174,6 @@ public: static Pipeline::SourceContainer::iterator optimizeEndOfPipeline( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container); - /** - * Applies optimizeAt() to all stages in the given pipeline after the stage that 'itr' points - * to. - * - * Returns a valid iterator that points to the new "end of the pipeline": i.e., the stage that - * comes after 'itr' in the newly optimized pipeline. - */ - static Pipeline::SourceContainer::iterator optimizeAtEndOfPipeline( - Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container); - static std::unique_ptr<Pipeline, PipelineDeleter> makePipelineFromViewDefinition( const boost::intrusive_ptr<ExpressionContext>& subPipelineExpCtx, ExpressionContext::ResolvedNamespace resolvedNs, @@ -269,6 +262,11 @@ public: bool needsShard() const; /** + * Returns 'true' if any stage in the pipeline requires being run on all shards. + */ + bool needsAllShardServers() const; + + /** * Returns true if the pipeline can run on mongoS, but is not obliged to; that is, it can run * either on mongoS or on a shard. */ @@ -286,7 +284,13 @@ public: void optimizePipeline(); /** - * Modifies the container, optimizing it by combining and swapping stages. + * Modifies the container, optimizes each stage individually. + */ + static void optimizeEachStage(SourceContainer* container); + + /** + * Modifies the container, optimizing it by combining, swapping, dropping and/or inserting + * stages. */ static void optimizeContainer(SourceContainer* container); @@ -301,11 +305,12 @@ public: * Helpers to serialize a pipeline. */ std::vector<Value> serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const; + boost::optional<const SerializationOptions&> opts = boost::none) const; std::vector<BSONObj> serializeToBson( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const; + boost::optional<const SerializationOptions&> opts = boost::none) const; static std::vector<Value> serializeContainer( - const SourceContainer& container, boost::optional<ExplainOptions::Verbosity> = boost::none); + const SourceContainer& container, + boost::optional<const SerializationOptions&> opts = boost::none); /** * Serializes the pipeline into BSON for explain/debug logging purposes. @@ -326,7 +331,8 @@ public: * Write the pipeline's operators to a std::vector<Value>, providing the level of detail * specified by 'verbosity'. */ - std::vector<Value> writeExplainOps(ExplainOptions::Verbosity verbosity) const; + std::vector<Value> writeExplainOps( + const SerializationOptions& opts = SerializationOptions{}) const; /** * Returns the dependencies needed by this pipeline. 'unavailableMetadata' should reflect what @@ -398,6 +404,12 @@ public: StringData targetStageName, std::function<bool(const DocumentSource* const)> predicate); /** + * Appends another pipeline to the existing pipeline. + * NOTE: The other pipeline will be destroyed. + */ + void appendPipeline(std::unique_ptr<Pipeline, PipelineDeleter> otherPipeline); + + /** * Performs common validation for top-level or facet pipelines. Throws if the pipeline is * invalid. * diff --git a/src/mongo/db/pipeline/pipeline_metadata_tree_test.cpp b/src/mongo/db/pipeline/pipeline_metadata_tree_test.cpp index 8b8f6597596..b34a1cfc2c3 100644 --- a/src/mongo/db/pipeline/pipeline_metadata_tree_test.cpp +++ b/src/mongo/db/pipeline/pipeline_metadata_tree_test.cpp @@ -58,16 +58,6 @@ #include "mongo/unittest/temp_dir.h" #include "mongo/unittest/unittest.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - ::mongo::str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString() \ - << " Exception occured in: " << #EXPRESSION; \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace mongo { namespace { diff --git a/src/mongo/db/pipeline/pipeline_test.cpp b/src/mongo/db/pipeline/pipeline_test.cpp index 288769a3961..d36681531fe 100644 --- a/src/mongo/db/pipeline/pipeline_test.cpp +++ b/src/mongo/db/pipeline/pipeline_test.cpp @@ -42,6 +42,8 @@ #include "mongo/db/pipeline/document_source_change_stream.h" #include "mongo/db/pipeline/document_source_change_stream_add_post_image.h" #include "mongo/db/pipeline/document_source_change_stream_add_pre_image.h" +#include "mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h" +#include "mongo/db/pipeline/document_source_change_stream_gen.h" #include "mongo/db/pipeline/document_source_change_stream_handle_topology_change.h" #include "mongo/db/pipeline/document_source_facet.h" #include "mongo/db/pipeline/document_source_graph_lookup.h" @@ -78,6 +80,8 @@ using std::vector; const NamespaceString kTestNss = NamespaceString("a.collection"); const NamespaceString kAdminCollectionlessNss = NamespaceString("admin.$cmd.aggregate"); +const auto kExplain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; constexpr size_t getChangeStreamStageSize() { return 6; @@ -89,6 +93,25 @@ void setMockReplicationCoordinatorOnOpCtx(OperationContext* opCtx) { std::make_unique<repl::ReplicationCoordinatorMock>(opCtx->getServiceContext())); } +DocumentSource* getStageAtPos(const Pipeline::SourceContainer& stages, int pos) { + if (pos >= 0) { + auto it = stages.begin(); + std::advance(it, pos); + return (*it).get(); + } else { + auto it = stages.rbegin(); + std::advance( + it, + -pos - 1); // Subtract 1 because rbegin() points to the element before the last one. + return (*it).get(); + } +} + +template <typename T> +void assertStageAtPos(const Pipeline::SourceContainer& stages, int pos) { + ASSERT(dynamic_cast<T*>(getStageAtPos(stages, pos))); +} + namespace Optimizations { namespace Local { @@ -102,7 +125,8 @@ class StubExplainInterface : public StubMongoProcessInterface { std::unique_ptr<Pipeline, PipelineDeleter> pipeline( ownedPipeline, PipelineDeleter(ownedPipeline->getContext()->opCtx)); BSONArrayBuilder bab; - auto pipelineVec = pipeline->writeExplainOps(verbosity); + auto opts = SerializationOptions{boost::make_optional(verbosity)}; + auto pipelineVec = pipeline->writeExplainOps(opts); for (auto&& stage : pipelineVec) { bab << stage; } @@ -149,7 +173,8 @@ void assertPipelineOptimizesAndSerializesTo(std::string inputPipeJson, auto outputPipe = Pipeline::parse(request.getPipeline(), ctx); outputPipe->optimizePipeline(); - ASSERT_VALUE_EQ(Value(outputPipe->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner)), + ASSERT_VALUE_EQ(Value(outputPipe->writeExplainOps(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})), Value(outputPipeExpected["pipeline"])); ASSERT_VALUE_EQ(Value(outputPipe->serialize()), Value(serializePipeExpected["pipeline"])); } @@ -497,6 +522,22 @@ TEST(PipelineOptimizationTest, SortDoesNotSwapBeforeUnwindBecauseUnwindPathEqual assertPipelineOptimizesAndSerializesTo(inputPipe, outputPipe, serializedPipe); } +TEST(PipelineOptimizationTest, SortDoesNotSwapBeforeUnwindBecauseArrayIndexField) { + std::string inputPipe = + "[{$unwind : {path: '$a', includeArrayIndex: 'i'}}" + ",{$sort : {i: 1}}" + "]"; + std::string outputPipe = + "[{$unwind : {path: '$a', includeArrayIndex: 'i'}}" + ",{$sort : {sortKey: {i: 1}}}" + "]"; + std::string serializedPipe = + "[{$unwind : {path: '$a', includeArrayIndex: 'i'}}" + ",{$sort : {i: 1}}" + "]"; + assertPipelineOptimizesAndSerializesTo(inputPipe, outputPipe, serializedPipe); +} + TEST(PipelineOptimizationTest, LookupShouldCoalesceWithUnwindOnAsSortDoesNotInterfere) { string inputPipe = "[{$lookup: {from : 'lookupColl', as : 'same', localField: 'left', foreignField: " @@ -2643,155 +2684,164 @@ TEST(PipelineOptimizationTest, MatchOnFmodShouldSwapWithAdjacentStage) { assertPipelineOptimizesTo(inputPipe, outputPipe); } -TEST(PipelineOptimizationTest, ChangeStreamLookupSwapsWithIndependentMatch) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); +class ChangeStreamPipelineOptimizationTest : public ServiceContextTest { +public: + ChangeStreamPipelineOptimizationTest() + : ChangeStreamPipelineOptimizationTest(false /* inMongos */) {} - intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); + ChangeStreamPipelineOptimizationTest(bool inMongos) { + _opCtx = _testServiceContext.makeOperationContext(); + _expCtx = make_intrusive<ExpressionContextForTest>(_opCtx.get(), kTestNss); + _expCtx->opCtx = _opCtx.get(); + _expCtx->uuid = UUID::gen(); + _expCtx->inMongos = inMongos; + setMockReplicationCoordinatorOnOpCtx(_expCtx->opCtx); + } + BSONObj changestreamStage(const std::string& stageStr) { + return fromjson("{$changeStream: " + stageStr + "}"); + } + BSONObj matchStage(const std::string& stageStr) { + return fromjson("{$match: " + stageStr + "}"); + } + BSONObj redactStage(const std::string& stageStr) { + return fromjson("{$redact: " + stageStr + "}"); + } + std::unique_ptr<Pipeline, PipelineDeleter> makePipeline( + const std::vector<BSONObj>& rawPipeline) { + auto pipeline = Pipeline::parse(rawPipeline, _expCtx); + return pipeline; + } - // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which - // filters out newly added events. - auto spec = BSON("$changeStream" << BSON( - "fullDocument" - << "updateLookup" - << DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName << true)); - auto stages = DocumentSourceChangeStream::createFromBson(spec.firstElement(), expCtx); - ASSERT_EQ(stages.size(), getChangeStreamStageSize()); - // Make sure the change lookup is at the end. - ASSERT(dynamic_cast<DocumentSourceChangeStreamAddPostImage*>(stages.back().get())); + static std::string generateEventResumeToken() { + ResumeTokenData resumeTokenDataIn{Timestamp{1001, 3}, + ResumeTokenData::kDefaultTokenVersion, + 0, + UUID::gen(), + Value(Document{{"operationType", "drop"_sd}})}; + return ResumeToken(resumeTokenDataIn).toBSON().toString(); + } - auto matchPredicate = BSON("extra" - << "predicate"); - stages.push_back(DocumentSourceMatch::create(matchPredicate, expCtx)); - auto pipeline = Pipeline::create(stages, expCtx); - pipeline->optimizePipeline(); +private: + QueryTestServiceContext _testServiceContext; + ServiceContext::UniqueOperationContext _opCtx; + boost::intrusive_ptr<ExpressionContextForTest> _expCtx; +}; - // Make sure the $match stage has swapped before the change look up. - ASSERT( - dynamic_cast<DocumentSourceChangeStreamAddPostImage*>(pipeline->getSources().back().get())); +TEST_F(ChangeStreamPipelineOptimizationTest, ChangeStreamLookUpSize) { + auto pipeline = makePipeline( + {changestreamStage("{fullDocument: 'updateLookup', showExpandedEvents: true}")}); + ASSERT_EQ(pipeline->getSources().size(), getChangeStreamStageSize()); + // Make sure the change lookup is at the end. + assertStageAtPos<DocumentSourceChangeStreamAddPostImage>(pipeline->getSources(), -1 /* pos */); } -TEST(PipelineOptimizationTest, ChangeStreamLookupDoesNotSwapWithMatchOnPostImage) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); - - intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); - +TEST_F(ChangeStreamPipelineOptimizationTest, ChangeStreamLookupSwapsWithIndependentMatch) { // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which // filters out newly added events. - auto spec = BSON("$changeStream" << BSON( - "fullDocument" - << "updateLookup" - << DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName << true)); - auto stages = DocumentSourceChangeStream::createFromBson(spec.firstElement(), expCtx); - ASSERT_EQ(stages.size(), getChangeStreamStageSize()); - // Make sure the change lookup is at the end. - ASSERT(dynamic_cast<DocumentSourceChangeStreamAddPostImage*>(stages.back().get())); - - stages.push_back(DocumentSourceMatch::create( - BSON(DocumentSourceChangeStreamAddPostImage::kFullDocumentFieldName << BSONNULL), expCtx)); - auto pipeline = Pipeline::create(stages, expCtx); + auto pipeline = + makePipeline({changestreamStage("{fullDocument: 'updateLookup', showExpandedEvents: true}"), + matchStage("{extra: 'predicate'}")}); pipeline->optimizePipeline(); + // Make sure the $match stage has swapped before the change look up. + assertStageAtPos<DocumentSourceChangeStreamAddPostImage>(pipeline->getSources(), -1 /* pos */); +} +TEST_F(ChangeStreamPipelineOptimizationTest, ChangeStreamLookupDoesNotSwapWithMatchOnPostImage) { + // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which + // filters out newly added eve + auto pipeline = + makePipeline({changestreamStage("{fullDocument: 'updateLookup', showExpandedEvents: true}"), + matchStage("{fullDocument: null}")}); + pipeline->optimizePipeline(); // Make sure the $match stage stays at the end. - ASSERT(dynamic_cast<DocumentSourceMatch*>(pipeline->getSources().back().get())); + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -1 /* pos */); } -TEST(PipelineOptimizationTest, FullDocumentBeforeChangeLookupSwapsWithIndependentMatch) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); - - intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); - +TEST_F(ChangeStreamPipelineOptimizationTest, FullDocumentBeforeChangeLookupSize) { // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which // filters out newly added events. - auto spec = BSON("$changeStream" << BSON( - "fullDocumentBeforeChange" - << "required" - << DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName << true)); - auto stages = DocumentSourceChangeStream::createFromBson(spec.firstElement(), expCtx); - ASSERT_EQ(stages.size(), getChangeStreamStageSize()); + auto pipeline = makePipeline( + {changestreamStage("{fullDocumentBeforeChange: 'required', showExpandedEvents: true}")}); + ASSERT_EQ(pipeline->getSources().size(), getChangeStreamStageSize()); // Make sure the pre-image lookup is at the end. - ASSERT(dynamic_cast<DocumentSourceChangeStreamAddPreImage*>(stages.back().get())); + assertStageAtPos<DocumentSourceChangeStreamAddPreImage>(pipeline->getSources(), -1 /* pos */); +} - auto matchPredicate = BSON("extra" - << "predicate"); - stages.push_back(DocumentSourceMatch::create(matchPredicate, expCtx)); - auto pipeline = Pipeline::create(stages, expCtx); +TEST_F(ChangeStreamPipelineOptimizationTest, + FullDocumentBeforeChangeLookupSwapsWithIndependentMatch) { + // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which + // filters out newly added events. + auto pipeline = makePipeline( + {changestreamStage("{fullDocumentBeforeChange: 'required', showExpandedEvents: true}"), + matchStage("{extra: 'predicate'}")}); pipeline->optimizePipeline(); - // Make sure the $match stage has swapped before the change look up. - ASSERT( - dynamic_cast<DocumentSourceChangeStreamAddPreImage*>(pipeline->getSources().back().get())); + assertStageAtPos<DocumentSourceChangeStreamAddPreImage>(pipeline->getSources(), -1 /* pos */); } -TEST(PipelineOptimizationTest, FullDocumentBeforeChangeDoesNotSwapWithMatchOnPreImage) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); - - intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); - +TEST_F(ChangeStreamPipelineOptimizationTest, + FullDocumentBeforeChangeDoesNotSwapWithMatchOnPreImage) { // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which // filters out newly added events. - auto spec = BSON("$changeStream" << BSON( - "fullDocumentBeforeChange" - << "required" - << DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName << true)); - auto stages = DocumentSourceChangeStream::createFromBson(spec.firstElement(), expCtx); - ASSERT_EQ(stages.size(), getChangeStreamStageSize()); - // Make sure the pre-image lookup is at the end. - ASSERT(dynamic_cast<DocumentSourceChangeStreamAddPreImage*>(stages.back().get())); - - stages.push_back(DocumentSourceMatch::create( - BSON(DocumentSourceChangeStreamAddPreImage::kFullDocumentBeforeChangeFieldName << BSONNULL), - expCtx)); - auto pipeline = Pipeline::create(stages, expCtx); + auto pipeline = makePipeline( + {changestreamStage("{fullDocumentBeforeChange: 'required', showExpandedEvents: true}"), + matchStage("{fullDocumentBeforeChange: null}")}); pipeline->optimizePipeline(); - // Make sure the $match stage stays at the end. - ASSERT(dynamic_cast<DocumentSourceMatch*>(pipeline->getSources().back().get())); + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -1 /* pos */); } -TEST(PipelineOptimizationTest, ChangeStreamHandleTopologyChangeSwapsWithRedact) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); +TEST_F(ChangeStreamPipelineOptimizationTest, + ChangeStreamEnsureResumeTokenSwapsWithJsonSchemaMatch) { + auto pipeline = makePipeline( + {changestreamStage("{resumeAfter: " + generateEventResumeToken() + "}"), + matchStage( + "{$jsonSchema: {properties: {documentKey: {properties: {_id: {enum: [1, 2]}}}}}}")}); + + // Assert $match is the last stage before optimization. + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -1); + + pipeline->optimizePipeline(); + + // Assert that $match swaps with $_internalChangeStreamHandleTopologyChange after optimization. + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -2); + assertStageAtPos<DocumentSourceChangeStreamEnsureResumeTokenPresent>(pipeline->getSources(), + -1); +} - boost::intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - expCtx->inMongos = true; // To enforce the $_internalChangeStreamHandleTopologyChange stage. - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); +// To enforce the $_internalChangeStreamHandleTopologyChange stage. +class ChangeStreamPipelineOptimizationTestWithMongoS : public ChangeStreamPipelineOptimizationTest { +public: + ChangeStreamPipelineOptimizationTestWithMongoS() + : ChangeStreamPipelineOptimizationTest(true /* inMongos */) {} +}; - auto stages = DocumentSourceChangeStream::createFromBson( - fromjson("{$changeStream: {showExpandedEvents: true}}").firstElement(), expCtx); +TEST_F(ChangeStreamPipelineOptimizationTestWithMongoS, + ChangeStreamHandleTopologyChangeSwapsWithRedact) { + auto pipeline = + makePipeline({changestreamStage("{showExpandedEvents: true}"), redactStage("'$$PRUNE'")}); + pipeline->optimizePipeline(); + // Assert that $redact swaps with $_internalChangeStreamHandleTopologyChange after optimization. + assertStageAtPos<DocumentSourceRedact>(pipeline->getSources(), -2 /* pos */); + assertStageAtPos<DocumentSourceChangeStreamHandleTopologyChange>(pipeline->getSources(), + -1 /* pos */); +} - // Assert that the last stage is $_internalChangeStreamHandleTopologyChange. - ASSERT(dynamic_cast<DocumentSourceChangeStreamHandleTopologyChange*>(stages.back().get())); +TEST_F(ChangeStreamPipelineOptimizationTestWithMongoS, + ChangeStreamHandleTopologyChangeSwapsWithJsonSchemaMatch) { + auto pipeline = makePipeline( + {changestreamStage("{}"), + matchStage( + "{$jsonSchema: {properties: {documentKey: {properties: {_id: {enum: [1, 2]}}}}}}")}); - // Add $redact as the last stage. - stages.push_back(DocumentSourceRedact::createFromBson( - fromjson("{$redact: '$$PRUNE'}").firstElement(), expCtx)); + // Assert $match is the last stage before optimization. + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -1); - auto pipeline = Pipeline::create(stages, expCtx); pipeline->optimizePipeline(); - // Assert that $redact swaps with $_internalChangeStreamHandleTopologyChange after optimization. - ASSERT(dynamic_cast<DocumentSourceRedact*>( - std::prev(std::prev(pipeline->getSources().end()))->get())); - ASSERT(dynamic_cast<DocumentSourceChangeStreamHandleTopologyChange*>( - pipeline->getSources().back().get())); + // Assert that $match swaps with $_internalChangeStreamHandleTopologyChange after optimization. + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -2); + assertStageAtPos<DocumentSourceChangeStreamHandleTopologyChange>(pipeline->getSources(), -1); } TEST(PipelineOptimizationTest, SortLimProjLimBecomesTopKSortProj) { @@ -3194,7 +3244,8 @@ void assertTwoPipelinesOptimizeAndMergeTo(const std::string inputPipe1, } pipeline1->optimizePipeline(); - ASSERT_VALUE_EQ(Value(pipeline1->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner)), + ASSERT_VALUE_EQ(Value(pipeline1->writeExplainOps(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})), Value(outputBson["pipeline"])); } @@ -3312,12 +3363,11 @@ public: mergePipe->optimizePipeline(); auto splitPipeline = sharded_agg_helpers::splitPipeline(std::move(mergePipe)); - - ASSERT_VALUE_EQ(Value(splitPipeline.shardsPipeline->writeExplainOps( - ExplainOptions::Verbosity::kQueryPlanner)), + const auto explain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; + ASSERT_VALUE_EQ(Value(splitPipeline.shardsPipeline->writeExplainOps(explain)), Value(shardPipeExpected["pipeline"])); - ASSERT_VALUE_EQ(Value(splitPipeline.mergePipeline->writeExplainOps( - ExplainOptions::Verbosity::kQueryPlanner)), + ASSERT_VALUE_EQ(Value(splitPipeline.mergePipeline->writeExplainOps(explain)), Value(mergePipeExpected["pipeline"])); shardPipe = std::move(splitPipeline.shardsPipeline); @@ -3860,7 +3910,7 @@ class Out : public ShardMergerBase { return "[]"; } string mergePipeJson() { - return "[{$out: {db: 'a', coll: 'outColl'}}]"; + return "[{$out: {coll: 'outColl', db: 'a'}}]"; } }; diff --git a/src/mongo/db/pipeline/plan_executor_pipeline.cpp b/src/mongo/db/pipeline/plan_executor_pipeline.cpp index 958c8a653af..7e39571b589 100644 --- a/src/mongo/db/pipeline/plan_executor_pipeline.cpp +++ b/src/mongo/db/pipeline/plan_executor_pipeline.cpp @@ -37,6 +37,7 @@ #include "mongo/db/pipeline/plan_explainer_pipeline.h" #include "mongo/db/pipeline/resume_token.h" #include "mongo/db/repl/speculative_majority_read_info.h" +#include "mongo/util/duration.h" namespace mongo { namespace { diff --git a/src/mongo/db/pipeline/plan_executor_pipeline.h b/src/mongo/db/pipeline/plan_executor_pipeline.h index 52139d1b34d..577a2d3fe3f 100644 --- a/src/mongo/db/pipeline/plan_executor_pipeline.h +++ b/src/mongo/db/pipeline/plan_executor_pipeline.h @@ -29,12 +29,14 @@ #pragma once +#include "mongo/util/duration.h" #include <queue> #include "mongo/db/exec/document_value/document.h" #include "mongo/db/pipeline/pipeline.h" #include "mongo/db/pipeline/plan_explainer_pipeline.h" #include "mongo/db/query/plan_executor.h" +#include "mongo/db/query/query_shape/serialization_options.h" namespace mongo { @@ -155,7 +157,9 @@ public: * providing the level of detail specified by 'verbosity'. */ std::vector<Value> writeExplainOps(ExplainOptions::Verbosity verbosity) const { - return _pipeline->writeExplainOps(verbosity); + auto opts = SerializationOptions{}; + opts.verbosity = verbosity; + return _pipeline->writeExplainOps(opts); } void enableSaveRecoveryUnitAcrossCommandsIfSupported() override {} diff --git a/src/mongo/db/pipeline/plan_explainer_pipeline.h b/src/mongo/db/pipeline/plan_explainer_pipeline.h index ae2dca8239f..e6c1a689a16 100644 --- a/src/mongo/db/pipeline/plan_explainer_pipeline.h +++ b/src/mongo/db/pipeline/plan_explainer_pipeline.h @@ -31,6 +31,7 @@ #include "mongo/db/pipeline/pipeline.h" #include "mongo/db/query/plan_explainer.h" +#include "mongo/util/duration.h" namespace mongo { /** diff --git a/src/mongo/db/pipeline/process_interface/mongo_process_interface.h b/src/mongo/db/pipeline/process_interface/mongo_process_interface.h index f4fe2f54c4a..9a980594737 100644 --- a/src/mongo/db/pipeline/process_interface/mongo_process_interface.h +++ b/src/mongo/db/pipeline/process_interface/mongo_process_interface.h @@ -84,7 +84,7 @@ public: * 2. write_ops::UpdateModification - either the new document we want to upsert or insert into * the collection (i.e. a 'classic' replacement update), or the pipeline to run to compute * the new document. - * 3. boost::optional<BSONObj> - for pipeline-style updated, specifies variables that can be + * 3. boost::optional<BSONObj> - for pipeline-style updates, specifies variables that can be * referred to in the pipeline performing the custom update. */ using BatchObject = @@ -172,6 +172,20 @@ public: virtual bool isSharded(OperationContext* opCtx, const NamespaceString& ns) = 0; /** + * TODO SERVER-79508 validate callers of this function remain correct. + * + * Returns false if the current request only handles parsing and validating queries. In other + * words, we are not executing queries. Examples include query analysis for queryable + * encryption, executing pipeline-style operations in the Update system, and creating a Query + * Shape. This function only returns false when the process interface is of type + * 'StubMongoProcessInterface'. + * + */ + virtual bool isExpectedToExecuteQueries() { + return true; + } + + /** * Advances the proxied write time associated with the client in ReplClientInfo to * be at least as high as the one tracked by the OperationTimeTracker associated with the * given operation context. diff --git a/src/mongo/db/pipeline/process_interface/non_shardsvr_process_interface.cpp b/src/mongo/db/pipeline/process_interface/non_shardsvr_process_interface.cpp index 63fbfe7c7ec..2a1ce64792a 100644 --- a/src/mongo/db/pipeline/process_interface/non_shardsvr_process_interface.cpp +++ b/src/mongo/db/pipeline/process_interface/non_shardsvr_process_interface.cpp @@ -205,6 +205,7 @@ BSONObj NonShardServerProcessInterface::preparePipelineAndExplain( Pipeline* ownedPipeline, ExplainOptions::Verbosity verbosity) { std::vector<Value> pipelineVec; auto firstStage = ownedPipeline->peekFront(); + auto opts = SerializationOptions{verbosity}; // If the pipeline already has a cursor explain with that one, otherwise attach a new one like // we would for a normal execution and explain that. if (firstStage && typeid(*firstStage) == typeid(DocumentSourceCursor)) { @@ -212,7 +213,7 @@ BSONObj NonShardServerProcessInterface::preparePipelineAndExplain( // extracted the necessary information and won't need it again. std::unique_ptr<Pipeline, PipelineDeleter> managedPipeline( ownedPipeline, PipelineDeleter(ownedPipeline->getContext()->opCtx)); - pipelineVec = managedPipeline->writeExplainOps(verbosity); + pipelineVec = managedPipeline->writeExplainOps(opts); ownedPipeline = nullptr; } else { auto pipelineWithCursor = attachCursorSourceToPipelineForLocalRead(ownedPipeline); @@ -221,7 +222,7 @@ BSONObj NonShardServerProcessInterface::preparePipelineAndExplain( while (pipelineWithCursor->getNext()) { } } - pipelineVec = pipelineWithCursor->writeExplainOps(verbosity); + pipelineVec = pipelineWithCursor->writeExplainOps(opts); } BSONArrayBuilder bab; for (auto&& stage : pipelineVec) { diff --git a/src/mongo/db/pipeline/process_interface/shardsvr_process_interface.cpp b/src/mongo/db/pipeline/process_interface/shardsvr_process_interface.cpp index 9755fcc7dbd..aa6914f88c3 100644 --- a/src/mongo/db/pipeline/process_interface/shardsvr_process_interface.cpp +++ b/src/mongo/db/pipeline/process_interface/shardsvr_process_interface.cpp @@ -110,7 +110,9 @@ Status ShardServerProcessInterface::insert( BatchedCommandRequest batchInsertCommand(std::move(insertCommand)); - batchInsertCommand.setWriteConcern(wc.toBSON()); + const auto originalWC = expCtx->opCtx->getWriteConcern(); + ScopeGuard resetWCGuard([&] { expCtx->opCtx->setWriteConcern(originalWC); }); + expCtx->opCtx->setWriteConcern(wc); cluster::write(expCtx->opCtx, batchInsertCommand, &stats, &response, targetEpoch); @@ -129,7 +131,10 @@ StatusWith<MongoProcessInterface::UpdateResult> ShardServerProcessInterface::upd BatchWriteExecStats stats; BatchedCommandRequest batchUpdateCommand(std::move(updateCommand)); - batchUpdateCommand.setWriteConcern(wc.toBSON()); + + const auto originalWC = expCtx->opCtx->getWriteConcern(); + ScopeGuard resetWCGuard([&] { expCtx->opCtx->setWriteConcern(originalWC); }); + expCtx->opCtx->setWriteConcern(wc); cluster::write(expCtx->opCtx, batchUpdateCommand, &stats, &response, targetEpoch); diff --git a/src/mongo/db/pipeline/process_interface/shardsvr_process_interface_test.cpp b/src/mongo/db/pipeline/process_interface/shardsvr_process_interface_test.cpp index 5c3f7eebf97..d60af845ccc 100644 --- a/src/mongo/db/pipeline/process_interface/shardsvr_process_interface_test.cpp +++ b/src/mongo/db/pipeline/process_interface/shardsvr_process_interface_test.cpp @@ -28,9 +28,11 @@ */ #include "mongo/db/concurrency/lock_state.h" +#include "mongo/db/cursor_id.h" #include "mongo/db/pipeline/document_source_out.h" #include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/process_interface/shardsvr_process_interface.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/s/query/sharded_agg_test_fixture.h" #include "mongo/unittest/unittest.h" diff --git a/src/mongo/db/pipeline/process_interface/stub_mongo_process_interface.h b/src/mongo/db/pipeline/process_interface/stub_mongo_process_interface.h index 0e984abb1b7..d69d5af7809 100644 --- a/src/mongo/db/pipeline/process_interface/stub_mongo_process_interface.h +++ b/src/mongo/db/pipeline/process_interface/stub_mongo_process_interface.h @@ -79,6 +79,10 @@ public: return std::make_unique<StubWriteSizeEstimator>(); } + bool isExpectedToExecuteQueries() override { + return false; + } + bool isSharded(OperationContext* opCtx, const NamespaceString& ns) override { return false; } diff --git a/src/mongo/db/pipeline/resume_token.cpp b/src/mongo/db/pipeline/resume_token.cpp index 8cc4fde72a0..23553fc5881 100644 --- a/src/mongo/db/pipeline/resume_token.cpp +++ b/src/mongo/db/pipeline/resume_token.cpp @@ -309,8 +309,26 @@ ResumeTokenData ResumeToken::getData() const { return result; } -Document ResumeToken::toDocument() const { - return Document{{kDataFieldName, _hexKeyString}, {kTypeBitsFieldName, _typeBits}}; +Document ResumeToken::toDocument(const SerializationOptions& options) const { + // This is our default resume token for the representative query shape. + static const auto kDefaultTokenQueryStats = makeHighWaterMarkToken(Timestamp(), 1); + + return Document{ + {kDataFieldName, + options.serializeLiteral(_hexKeyString, Value(kDefaultTokenQueryStats._hexKeyString))}, + + // When serializing with 'kToDebugTypeString' 'serializeLiteral' will return an + // incorrect result. Therefore, we prefer to always exclude '_typeBits' when serializing + // the debug string by passing an empty value, since '_typeBits' is rarely set and will + // always be either missing or of type BinData. + {kTypeBitsFieldName, + options.literalPolicy == LiteralSerializationPolicy::kToDebugTypeString + ? Value() + : options.serializeLiteral(_typeBits, kDefaultTokenQueryStats._typeBits)}}; +} + +BSONObj ResumeToken::toBSON(const SerializationOptions& options) const { + return toDocument(options).toBson(); } ResumeToken ResumeToken::parse(const Document& resumeDoc) { diff --git a/src/mongo/db/pipeline/resume_token.h b/src/mongo/db/pipeline/resume_token.h index c4e8b5d06fd..74b2dd9ddef 100644 --- a/src/mongo/db/pipeline/resume_token.h +++ b/src/mongo/db/pipeline/resume_token.h @@ -36,6 +36,7 @@ #include "mongo/bson/timestamp.h" #include "mongo/db/exec/document_value/document.h" #include "mongo/db/exec/document_value/value.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/util/uuid.h" namespace mongo { @@ -165,11 +166,18 @@ public: */ explicit ResumeToken(const ResumeTokenData& resumeValue); - Document toDocument() const; + /** + * Convenience method to represent the ResumeToken as a Document. + * Provides support for specifying SerializationOptions, as this method is used to service the + * toBSON(). + */ + Document toDocument(const SerializationOptions& options = {}) const; - BSONObj toBSON() const { - return toDocument().toBson(); - } + /** + * Serialization to BSONObj. Provides support for specifying SerializationOptions, + * as ResumeToken requires a "query_shape: custom" specification in its IDL uses. + */ + BSONObj toBSON(const SerializationOptions& options = {}) const; ResumeTokenData getData() const; diff --git a/src/mongo/db/pipeline/resume_token.idl b/src/mongo/db/pipeline/resume_token.idl new file mode 100644 index 00000000000..ee20d401dd0 --- /dev/null +++ b/src/mongo/db/pipeline/resume_token.idl @@ -0,0 +1,46 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo" + cpp_includes: + - "mongo/db/pipeline/resume_token.h" + +imports: + - "mongo/idl/basic_types.idl" + +types: + # A resume token is an opaque document we return to the user that contains all the information + # needed to resume a stream where they left off. It also provides the ordering of streams + # from multiple shards. + resumeToken: + bson_serialization_type: object + description: An object representing a resume token for a change stream + cpp_type: ResumeToken + serializer: ResumeToken::toBSON + deserializer: ResumeToken::parse diff --git a/src/mongo/db/pipeline/sharded_agg_helpers.cpp b/src/mongo/db/pipeline/sharded_agg_helpers.cpp index b2a0392ef15..2fe2307b0a6 100644 --- a/src/mongo/db/pipeline/sharded_agg_helpers.cpp +++ b/src/mongo/db/pipeline/sharded_agg_helpers.cpp @@ -67,6 +67,9 @@ #include "mongo/util/fail_point.h" #include "mongo/util/visit_helper.h" +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + + namespace mongo { namespace sharded_agg_helpers { namespace { @@ -193,11 +196,12 @@ BSONObj genericTransformForShards(MutableDocument&& cmdForShards, std::vector<RemoteCursor> establishShardCursors(OperationContext* opCtx, std::shared_ptr<executor::TaskExecutor> executor, const NamespaceString& nss, - bool mustRunOnAll, + bool mustRunOnAllShards, boost::optional<ChunkManager>& cm, const std::set<ShardId>& shardIds, const BSONObj& cmdObj, - const ReadPreferenceSetting& readPref) { + const ReadPreferenceSetting& readPref, + bool targetEveryShardServer) { LOGV2_DEBUG(20904, 1, "Dispatching command {cmdObj} to establish cursors on shards", @@ -206,9 +210,27 @@ std::vector<RemoteCursor> establishShardCursors(OperationContext* opCtx, std::vector<std::pair<ShardId, BSONObj>> requests; // If we don't need to run on all shards, then we should always have a valid routing table. - invariant(cm || mustRunOnAll); + invariant(cm || mustRunOnAllShards); + + if (targetEveryShardServer) { + uassert(7355703, + "Cannot target all hosts if the pipeline is not run on all shards.", + mustRunOnAllShards); + if (MONGO_unlikely(shardedAggregateHangBeforeEstablishingShardCursors.shouldFail())) { + LOGV2( + 7355704, + "shardedAggregateHangBeforeEstablishingShardCursors fail point enabled. Blocking " + "until fail point is disabled."); + while ( + MONGO_unlikely(shardedAggregateHangBeforeEstablishingShardCursors.shouldFail())) { + sleepsecs(1); + } + } + return establishCursorsOnAllHosts( + opCtx, std::move(executor), nss, shardIds, cmdObj, false, getDesiredRetryPolicy(opCtx)); + } - if (mustRunOnAll) { + if (mustRunOnAllShards) { // The pipeline contains a stage which must be run on all shards. Skip versioning and // enqueue the raw command objects. for (const auto& shardId : shardIds) { @@ -1057,15 +1079,17 @@ DispatchShardPipelineResults dispatchShardPipeline( : expCtx->getCollatorBSON(); // Determine whether we can run the entire aggregation on a single shard. - const bool mustRunOnAll = mustRunOnAllShards(expCtx->ns, hasChangeStream, startsWithDocuments); + const bool mustRunOnAllShards = + checkIfMustRunOnAllShards(expCtx->ns, hasChangeStream, startsWithDocuments); std::set<ShardId> shardIds = getTargetedShards( - expCtx, mustRunOnAll, executionNsRoutingInfo, shardQuery, shardTargetingCollation); + expCtx, mustRunOnAllShards, executionNsRoutingInfo, shardQuery, shardTargetingCollation); + bool targetEveryShardServer = pipeline->needsAllShardServers(); // Don't need to split the pipeline if we are only targeting a single shard, unless: // - There is a stage that needs to be run on the primary shard and the single target shard // is not the primary. // - The pipeline contains one or more stages which must always merge on mongoS. - const bool needsSplit = (shardIds.size() > 1u || needsMongosMerge || + const bool needsSplit = (shardIds.size() > 1u || needsMongosMerge || targetEveryShardServer || (needsPrimaryShardMerge && executionNsRoutingInfo && *(shardIds.begin()) != executionNsRoutingInfo->dbPrimary())); @@ -1134,8 +1158,11 @@ DispatchShardPipelineResults dispatchShardPipeline( if (hasChangeStream) { Grid::get(opCtx)->shardRegistry()->reload(opCtx); // Rebuild the set of shards as the shard registry might have changed. - shardIds = getTargetedShards( - expCtx, mustRunOnAll, executionNsRoutingInfo, shardQuery, shardTargetingCollation); + shardIds = getTargetedShards(expCtx, + mustRunOnAllShards, + executionNsRoutingInfo, + shardQuery, + shardTargetingCollation); } // If there were no shards when we began execution, we wouldn't have run this aggregation in the @@ -1147,7 +1174,7 @@ DispatchShardPipelineResults dispatchShardPipeline( // Explain does not produce a cursor, so instead we scatter-gather commands to the shards. if (explain) { - if (mustRunOnAll) { + if (mustRunOnAllShards) { // Some stages (such as $currentOp) need to be broadcast to all shards, and // should not participate in the shard version protocol. shardResults = @@ -1176,11 +1203,12 @@ DispatchShardPipelineResults dispatchShardPipeline( cursors = establishShardCursors(opCtx, expCtx->mongoProcessInterface->taskExecutor, expCtx->ns, - mustRunOnAll, + mustRunOnAllShards, executionNsRoutingInfo, shardIds, targetedCommand, - ReadPreferenceSetting::get(opCtx)); + ReadPreferenceSetting::get(opCtx), + targetEveryShardServer); } catch (const ExceptionFor<ErrorCodes::StaleConfig>& e) { // Check to see if the command failed because of a stale shard version or something @@ -1407,9 +1435,11 @@ Status appendExplainResults(DispatchShardPipelineResults&& dispatchResults, MutableDocument pipelinesDoc; // We specify "queryPlanner" verbosity when building the output for "shardsPart" because // execution stats are reported by each shard individually. - pipelinesDoc.addField("shardsPart", - Value(dispatchResults.splitPipeline->shardsPipeline->writeExplainOps( - ExplainOptions::Verbosity::kQueryPlanner))); + auto opts = SerializationOptions{}; + opts.verbosity = boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner); + pipelinesDoc.addField( + "shardsPart", + Value(dispatchResults.splitPipeline->shardsPipeline->writeExplainOps(opts))); if (dispatchResults.exchangeSpec) { BSONObjBuilder bob; dispatchResults.exchangeSpec->exchangeSpec.serialize(&bob); @@ -1418,7 +1448,7 @@ Status appendExplainResults(DispatchShardPipelineResults&& dispatchResults, } // We specify "queryPlanner" verbosity because execution stats are not currently // supported when building the output for "mergerPart". - auto explainOps = mergePipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto explainOps = mergePipeline->writeExplainOps(opts); // No cursors to remote shards are established for an explain, and the $mergeCursors // aggregation stage which is normally built in addMergeCursorsSource() requires vectors of @@ -1538,9 +1568,9 @@ Shard::RetryPolicy getDesiredRetryPolicy(OperationContext* opCtx) { return Shard::RetryPolicy::kIdempotent; } -bool mustRunOnAllShards(const NamespaceString& nss, - bool hasChangeStream, - bool startsWithDocuments) { +bool checkIfMustRunOnAllShards(const NamespaceString& nss, + bool hasChangeStream, + bool startsWithDocuments) { // The following aggregations must be routed to all shards: // - Any collectionless aggregation, such as non-localOps $currentOp. // - Any aggregation which begins with a $changeStream stage. diff --git a/src/mongo/db/pipeline/sharded_agg_helpers.h b/src/mongo/db/pipeline/sharded_agg_helpers.h index 12945397b59..c23ad646bbd 100644 --- a/src/mongo/db/pipeline/sharded_agg_helpers.h +++ b/src/mongo/db/pipeline/sharded_agg_helpers.h @@ -195,7 +195,9 @@ StatusWith<ChunkManager> getExecutionNsRoutingInfo(OperationContext* opCtx, /** * Returns true if an aggregation over 'nss' must run on all shards. */ -bool mustRunOnAllShards(const NamespaceString& nss, bool hasChangeStream, bool startsWithDocuments); +bool checkIfMustRunOnAllShards(const NamespaceString& nss, + bool hasChangeStream, + bool startsWithDocuments); /** * Retrieves the desired retry policy based on whether the default writeConcern is set on 'opCtx'. diff --git a/src/mongo/db/pipeline/sharded_union_test.cpp b/src/mongo/db/pipeline/sharded_union_test.cpp index 79863fc7f14..71336e4dfb9 100644 --- a/src/mongo/db/pipeline/sharded_union_test.cpp +++ b/src/mongo/db/pipeline/sharded_union_test.cpp @@ -35,6 +35,7 @@ #include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/document_source_union_with.h" #include "mongo/db/pipeline/process_interface/shardsvr_process_interface.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/db/repl/read_concern_args.h" #include "mongo/db/views/resolved_view.h" #include "mongo/s/query/sharded_agg_test_fixture.h" diff --git a/src/mongo/db/pipeline/stage_constraints.h b/src/mongo/db/pipeline/stage_constraints.h index 74bd32fbaa8..456f3cfc65f 100644 --- a/src/mongo/db/pipeline/stage_constraints.h +++ b/src/mongo/db/pipeline/stage_constraints.h @@ -77,6 +77,10 @@ struct StageConstraints { kAnyShard, // Indicates that the stage can only run on mongoS. kMongoS, + // Indicates that the stage should run on all data-bearing nodes, primary and seconday, for + // the participating shards. This is useful for stages like $currentOp which generate + // node-specific metadata. + kAllShardServers, }; /** @@ -190,7 +194,8 @@ struct StageConstraints { // shard, since it needs to be able to run on mongoS in a cluster. invariant(!(changeStreamRequirement == ChangeStreamRequirement::kAllowlist && (hostRequirement == HostTypeRequirement::kAnyShard || - hostRequirement == HostTypeRequirement::kPrimaryShard))); + hostRequirement == HostTypeRequirement::kPrimaryShard || + hostRequirement == HostTypeRequirement::kAllShardServers))); // A stage which is allowlisted for $changeStream cannot have a position requirement. invariant(!(changeStreamRequirement == ChangeStreamRequirement::kAllowlist && @@ -209,6 +214,13 @@ struct StageConstraints { if (diskRequirement == DiskUseRequirement::kWritesPersistentData) { invariant(!isAllowedInTransaction()); } + + tassert( + 7355706, + "Stage can only broadcast to all shard servers if it must be the first stage in the " + "pipeline.", + hostRequirement != HostTypeRequirement::kAllShardServers || + (requiredPosition == PositionRequirement::kFirst)); } /** diff --git a/src/mongo/db/pipeline/storage_stats_spec.idl b/src/mongo/db/pipeline/storage_stats_spec.idl index 8052f09de51..c0cc0a4ce6a 100644 --- a/src/mongo/db/pipeline/storage_stats_spec.idl +++ b/src/mongo/db/pipeline/storage_stats_spec.idl @@ -36,18 +36,23 @@ structs: StorageStatsSpec: description: Represents the 'storageStats' argument to the $collStats stage. strict: false + query_shape_component: true fields: scale: description: A number to use as a scaling factor applied to reported metrics. type: safeInt optional: true validator: { gte: 1 } + query_shape: literal verbose: type: optionalBool default: false + query_shape: parameter waitForLock: type: optionalBool default: true + query_shape: parameter numericOnly: type: optionalBool default: false + query_shape: parameter diff --git a/src/mongo/db/pipeline/transformer_interface.h b/src/mongo/db/pipeline/transformer_interface.h index 6be98af2570..6057a2e328c 100644 --- a/src/mongo/db/pipeline/transformer_interface.h +++ b/src/mongo/db/pipeline/transformer_interface.h @@ -65,8 +65,8 @@ public: * Returns a document describing this transformation. For example, this function will return * {_id: 0, x: 1} for the stage parsed from {$project: {_id: 0, x: 1}}. */ - virtual Document serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const = 0; + virtual Document serializeTransformation(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options = {}) const = 0; /** * Method used by inclusion and add fields projecton executors to extract computed projections diff --git a/src/mongo/db/pipeline/variables.cpp b/src/mongo/db/pipeline/variables.cpp index 26793962efd..279b8630ce6 100644 --- a/src/mongo/db/pipeline/variables.cpp +++ b/src/mongo/db/pipeline/variables.cpp @@ -40,6 +40,21 @@ namespace mongo { +namespace { + +// We need to be careful when serializing values, e.g. to populate the 'let' parameter of a command +// to be sent over the wire. First, missing values should be serialied as $$REMOVE, otherwise they +// might be incorrectly omitted or serialized as empty objects ({}). Also, we should wrap values in +// $literal to avoid a scenario like the following: suppose we had a user-defined 'let' specified as +// {let: {a: {$literal: "$notAFieldName"}}}. On mongos, this will be evaluated to the string +// "$notAFieldName". When we serialize it again for the shard commands, it must appear as {$literal: +// "$notAFieldName"}, not simply "$notAFieldName", since the latter will be treated as a field name +// by mongods. +Value serializeValue(Value val) { + return val.missing() ? Value("$$REMOVE"_sd) : Value(DOC("$literal" << val)); +} +} // namespace + using namespace std::string_literals; constexpr Variables::Id Variables::kRootId; @@ -330,7 +345,8 @@ LegacyRuntimeConstants Variables::transitionalExtractRuntimeConstants() const { } Variables::Id VariablesParseState::defineVariable(StringData name) { - // Caller should have validated before hand by using variableValidationvalidateNameForUserWrite. + // Caller should have validated before hand by using + // variableValidation::validateNameForUserWrite. massert(17275, "Can't redefine a non-user-writable variable", Variables::kBuiltinVarNameToId.find(name) == Variables::kBuiltinVarNameToId.end()); @@ -374,8 +390,9 @@ std::set<Variables::Id> VariablesParseState::getDefinedVariableIDs() const { BSONObj VariablesParseState::serialize(const Variables& vars) const { auto bob = BSONObjBuilder{}; for (auto&& [var_name, id] : _variables) - if (vars.hasValue(id)) - bob << var_name << Value(DOC("$literal" << vars.getValue(id))); + if (vars.hasValue(id)) { + bob << var_name << serializeValue(vars.getValue(id)); + } // System variables have to be added separately since the variable IDs are reserved and not // allocated like normal variables, and so not present in '_variables'. @@ -387,8 +404,9 @@ std::pair<LegacyRuntimeConstants, BSONObj> VariablesParseState::transitionalComp const Variables& vars) const { auto bob = BSONObjBuilder{}; for (auto&& [var_name, id] : _variables) - if (vars.hasValue(id)) - bob << var_name << Value(DOC("$literal" << vars.getValue(id))); + if (vars.hasValue(id)) { + bob << var_name << serializeValue(vars.getValue(id)); + } return {vars.transitionalExtractRuntimeConstants(), bob.obj()}; } diff --git a/src/mongo/db/pipeline/variables.h b/src/mongo/db/pipeline/variables.h index d7a79c523b9..d69d9d46831 100644 --- a/src/mongo/db/pipeline/variables.h +++ b/src/mongo/db/pipeline/variables.h @@ -190,6 +190,13 @@ public: MONGO_UNREACHABLE_TASSERT(5858104); } + /** + * Return true if the passed-in variable ID belongs to a builtin variable. + */ + static auto isBuiltin(Variables::Id variable) { + return kIdToBuiltinVarName.find(variable) != kIdToBuiltinVarName.end(); + } + private: struct ValueAndState { ValueAndState() = default; diff --git a/src/mongo/db/pipeline/visitors/document_source_visitor.h b/src/mongo/db/pipeline/visitors/document_source_visitor.h index a0158147e38..9358b6267f9 100644 --- a/src/mongo/db/pipeline/visitors/document_source_visitor.h +++ b/src/mongo/db/pipeline/visitors/document_source_visitor.h @@ -67,6 +67,7 @@ class DocumentSourceSingleDocumentTransformation; class DocumentSourceSkip; class DocumentSourceSort; class DocumentSourceTeeConsumer; +class DocumentSourceQueryStats; class DocumentSourceUnionWith; class DocumentSourceUnwind; @@ -125,6 +126,7 @@ public: virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceSkip> source) = 0; virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceSort> source) = 0; virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceTeeConsumer> source) = 0; + virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceQueryStats> source) = 0; virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceUnionWith> source) = 0; virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceUnwind> source) = 0; }; diff --git a/src/mongo/db/pipeline/visitors/document_source_walker.cpp b/src/mongo/db/pipeline/visitors/document_source_walker.cpp index b0ea004cae9..5f271814e1c 100644 --- a/src/mongo/db/pipeline/visitors/document_source_walker.cpp +++ b/src/mongo/db/pipeline/visitors/document_source_walker.cpp @@ -54,6 +54,7 @@ #include "mongo/db/pipeline/document_source_operation_metrics.h" #include "mongo/db/pipeline/document_source_out.h" #include "mongo/db/pipeline/document_source_plan_cache_stats.h" +#include "mongo/db/pipeline/document_source_query_stats.h" #include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/document_source_redact.h" #include "mongo/db/pipeline/document_source_sample.h" @@ -126,6 +127,7 @@ void DocumentSourceWalker::walk(const Pipeline& pipeline) { visitHelper<DocumentSourceSingleDocumentTransformation>(ds) || visitHelper<DocumentSourceSkip>(ds) || visitHelper<DocumentSourceSort>(ds) || visitHelper<DocumentSourceTeeConsumer>(ds) || + visitHelper<DocumentSourceQueryStats>(ds) || visitHelper<DocumentSourceUnionWith>(ds) || visitHelper<DocumentSourceUnwind>(ds) // TODO: uncomment after fixing dependency //&& visitHelper<DocumentSourceUpdateOnAddShard>(ds) diff --git a/src/mongo/db/pipeline/window_function/partition_iterator.cpp b/src/mongo/db/pipeline/window_function/partition_iterator.cpp index 83925b862b3..6188845d516 100644 --- a/src/mongo/db/pipeline/window_function/partition_iterator.cpp +++ b/src/mongo/db/pipeline/window_function/partition_iterator.cpp @@ -205,15 +205,6 @@ optional<int> numericBound(WindowBounds::Bound<int> bound) { bound); } -// Assumes both arguments are numeric, and performs Decimal128 addition on them. -Value decimalAdd(const Value& left, const Value& right) { - // Widening to Decimal128 is a convenient way to avoid having many cases for different numeric - // types. The 'threshold' values we compute are only used to choose a set of documents; the - // user can't observe the type. - return Value(left.coerceToDecimal().add(right.coerceToDecimal())); -} - - } // namespace optional<std::pair<int, int>> PartitionIterator::getEndpointsRangeBased( @@ -247,7 +238,18 @@ optional<std::pair<int, int>> PartitionIterator::getEndpointsRangeBased( dateAdd(base.coerceToDate(), *range.unit, delta.coerceToInt(), TimeZone())}; } else { tassert(5429406, "Range-based bounds are specified as a number", delta.numeric()); - return decimalAdd(base, delta); + if (base.getType() == BSONType::NumberDouble) { + // When we compare a double and a Decimal128, we convert the Decimal128 to double + // and compare two double values. Since converting a double to Decimal128 is + // expensive and since during the comparison we will convert the Decimal128 to + // double, we compute the threshold as double from the beginning when the base + // value is already a double. + return Value(base.getDouble() + delta.coerceToDouble()); + } + // Widening to Decimal128 is a convenient way to avoid having many cases for different + // numeric types. The 'threshold' values we compute are only used to choose a set of + // documents; the user can't observe the type. + return Value(base.coerceToDecimal().add(delta.coerceToDecimal())); } }; auto hasExpectedType = [&](const Value& v) -> bool { diff --git a/src/mongo/db/pipeline/window_function/window_bounds.cpp b/src/mongo/db/pipeline/window_function/window_bounds.cpp index dd082135e02..7ede49d0065 100644 --- a/src/mongo/db/pipeline/window_function/window_bounds.cpp +++ b/src/mongo/db/pipeline/window_function/window_bounds.cpp @@ -63,12 +63,19 @@ WindowBounds::Bound<T> parseBound(ExpressionContext* expCtx, } template <class T> -Value serializeBound(const WindowBounds::Bound<T>& bound) { +Value serializeBound(const WindowBounds::Bound<T>& bound, + const SerializationOptions& opts, + const Value& representativeValue) { return stdx::visit( visit_helper::Overloaded{ - [](const WindowBounds::Unbounded&) { return Value(WindowBounds::kValUnbounded); }, - [](const WindowBounds::Current&) { return Value(WindowBounds::kValCurrent); }, - [](const T& n) { return Value(n); }, + [&](const WindowBounds::Unbounded&) { return Value(WindowBounds::kValUnbounded); }, + [&](const WindowBounds::Current&) { return Value(WindowBounds::kValCurrent); }, + [&](const T& n) { + // If not "unbounded" or "current", n must be a literal constant + // The upper bound must be greater than the lower bound. We override the + // representative value to meet this constraint. + return opts.serializeLiteral(n, representativeValue); + }, }, bound); } @@ -215,22 +222,31 @@ WindowBounds WindowBounds::parse(BSONObj args, uassert(5339902, "Range-based bounds require sortBy a single field", sortBy && sortBy->size() == 1); + const SortPattern::SortPatternPart& part = *sortBy->begin(); + uassert(8947400, + "Range-based bounds require a non-expression sortBy", + part.fieldPath && !part.expression); + uassert(8947401, "Range-based bounds require an ascending sortBy", part.isAscending); return bounds; } } -void WindowBounds::serialize(MutableDocument& args) const { +void WindowBounds::serialize(MutableDocument& args, const SerializationOptions& opts) const { stdx::visit( visit_helper::Overloaded{ [&](const DocumentBased& docBounds) { args[kArgDocuments] = Value{std::vector<Value>{ - serializeBound(docBounds.lower), - serializeBound(docBounds.upper), + serializeBound( + docBounds.lower, opts, /* representative value, if needed */ Value(0LL)), + serializeBound( + docBounds.upper, opts, /* representative value, if needed */ Value(1LL)), }}; }, [&](const RangeBased& rangeBounds) { args[kArgRange] = Value{std::vector<Value>{ - serializeBound(rangeBounds.lower), - serializeBound(rangeBounds.upper), + serializeBound( + rangeBounds.lower, opts, /* representative value, if needed */ Value(0LL)), + serializeBound( + rangeBounds.upper, opts, /* representative value, if needed */ Value(1LL)), }}; if (rangeBounds.unit) { args[kArgUnit] = Value{serializeTimeUnit(*rangeBounds.unit)}; diff --git a/src/mongo/db/pipeline/window_function/window_bounds.h b/src/mongo/db/pipeline/window_function/window_bounds.h index 6999f8fcdbc..90d0adf0371 100644 --- a/src/mongo/db/pipeline/window_function/window_bounds.h +++ b/src/mongo/db/pipeline/window_function/window_bounds.h @@ -121,7 +121,7 @@ struct WindowBounds { const boost::optional<SortPattern>& sortBy, ExpressionContext* expCtx); - void serialize(MutableDocument& args) const; + void serialize(MutableDocument& args, const SerializationOptions& opts) const; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/window_function/window_function_expression.cpp b/src/mongo/db/pipeline/window_function/window_function_expression.cpp index 0d43689c977..a46bac72dfa 100644 --- a/src/mongo/db/pipeline/window_function/window_function_expression.cpp +++ b/src/mongo/db/pipeline/window_function/window_function_expression.cpp @@ -310,12 +310,12 @@ boost::intrusive_ptr<Expression> ExpressionFirstLast::parse( template <typename WindowFunctionN, typename AccumulatorNType> Value ExpressionN<WindowFunctionN, AccumulatorNType>::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { + const SerializationOptions& opts) const { auto acc = buildAccumulatorOnly(); - MutableDocument result(acc->serialize(nExpr, _input, static_cast<bool>(explain))); + MutableDocument result(acc->serialize(nExpr, _input, opts)); MutableDocument windowField; - _bounds.serialize(windowField); + _bounds.serialize(windowField, opts); result[kWindowArg] = windowField.freezeToValue(); return result.freezeToValue(); } diff --git a/src/mongo/db/pipeline/window_function/window_function_expression.h b/src/mongo/db/pipeline/window_function/window_function_expression.h index 261d4212fe5..b14a81d1ae3 100644 --- a/src/mongo/db/pipeline/window_function/window_function_expression.h +++ b/src/mongo/db/pipeline/window_function/window_function_expression.h @@ -189,17 +189,16 @@ public: } }; - virtual Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const { + virtual Value serialize(const SerializationOptions& opts) const { MutableDocument args; - args[_accumulatorName] = _input->serialize(static_cast<bool>(explain)); + args[_accumulatorName] = _input->serialize(opts); MutableDocument windowField; - _bounds.serialize(windowField); + _bounds.serialize(windowField, opts); args[kWindowArg] = windowField.freezeToValue(); return args.freezeToValue(); } - protected: ExpressionContext* _expCtx; std::string _accumulatorName; @@ -326,9 +325,9 @@ public: << " is not supported as a removable window function"); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument args; - args.addField(_accumulatorName, Value(_input->serialize(static_cast<bool>(explain)))); + args.addField(_accumulatorName, Value(_input->serialize(opts))); return args.freezeToValue(); } }; @@ -444,7 +443,7 @@ public: << " is not supported with a removable window"); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument args; args.addField(_accumulatorName, Value(Document())); return args.freezeToValue(); @@ -493,15 +492,17 @@ public: << " is not supported with a removable window"); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument subObj; tassert(5433604, "ExpMovingAvg neither N nor alpha was set", _N || _alpha); if (_N) { - subObj[kNArg] = Value(_N.get()); + subObj[kNArg] = opts.serializeLiteral(_N.get()); } else { - subObj[kAlphaArg] = Value(_alpha.get()); + // Alpha must be between zero and one (exclusive), so choose a legal representative + // value if applicable. + subObj[kAlphaArg] = opts.serializeLiteral(_alpha.get(), Value(0.1)); } - subObj[kInputArg] = _input->serialize(static_cast<bool>(explain)); + subObj[kInputArg] = _input->serialize(opts); MutableDocument outerObj; outerObj[kAccName] = subObj.freezeToValue(); return outerObj.freezeToValue(); @@ -528,15 +529,15 @@ public: return _unit; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument result; - result[_accumulatorName][kArgInput] = _input->serialize(static_cast<bool>(explain)); + result[_accumulatorName][kArgInput] = _input->serialize(opts); if (_unit) { result[_accumulatorName][kArgUnit] = Value(serializeTimeUnit(*_unit)); } MutableDocument windowField; - _bounds.serialize(windowField); + _bounds.serialize(windowField, opts); result[kWindowArg] = windowField.freezeToValue(); return result.freezeToValue(); } @@ -810,9 +811,9 @@ public: MONGO_UNREACHABLE_TASSERT(5490705); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument args; - args.addField(_accumulatorName, Value(_input->serialize(static_cast<bool>(explain)))); + args.addField(_accumulatorName, Value(_input->serialize(opts))); return args.freezeToValue(); } }; @@ -904,7 +905,7 @@ public: nExpr(std::move(nExpr)), sortPattern(std::move(sortPattern)) {} - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts) const final; boost::intrusive_ptr<AccumulatorState> buildAccumulatorOnly() const final; diff --git a/src/mongo/db/pipeline/window_function/window_function_shift.cpp b/src/mongo/db/pipeline/window_function/window_function_shift.cpp index c74424e74e7..0ae15c4e3f0 100644 --- a/src/mongo/db/pipeline/window_function/window_function_shift.cpp +++ b/src/mongo/db/pipeline/window_function/window_function_shift.cpp @@ -119,12 +119,12 @@ boost::intrusive_ptr<Expression> ExpressionShift::parse(BSONObj obj, return shiftExpr; } -Value ExpressionShift::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value ExpressionShift::serialize(const SerializationOptions& opts) const { MutableDocument args; - args.addField(kByArg, Value(_offset)); - args.addField(kOutputArg, _input->serialize(static_cast<bool>(explain))); - args.addField(kDefaultArg, _defaultVal.get_value_or(mongo::Value(BSONNULL))); - + args.addField(kByArg, opts.serializeLiteral(_offset)); + args.addField(kOutputArg, _input->serialize(opts)); + args.addField(kDefaultArg, + opts.serializeLiteral(_defaultVal.get_value_or(mongo::Value(BSONNULL)))); MutableDocument windowFun; windowFun.addField(_accumulatorName, args.freezeToValue()); return windowFun.freezeToValue(); diff --git a/src/mongo/db/pipeline/window_function/window_function_shift.h b/src/mongo/db/pipeline/window_function/window_function_shift.h index 99a45dd3f84..649e90ce7e4 100644 --- a/src/mongo/db/pipeline/window_function/window_function_shift.h +++ b/src/mongo/db/pipeline/window_function/window_function_shift.h @@ -68,7 +68,7 @@ public: MONGO_UNREACHABLE_TASSERT(5424302); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts) const final; private: static boost::intrusive_ptr<Expression> parseShiftArgs(BSONObj obj, diff --git a/src/mongo/db/process_health/fault_state_machine_test.cpp b/src/mongo/db/process_health/fault_state_machine_test.cpp index 913fb168d69..065ac057136 100644 --- a/src/mongo/db/process_health/fault_state_machine_test.cpp +++ b/src/mongo/db/process_health/fault_state_machine_test.cpp @@ -362,7 +362,7 @@ TEST_F(FaultManagerTest, HealthCheckWithOffFacetCreatesNoFaultInOk) { TEST_F(FaultManagerTest, DNSHealthCheckWithBadHostNameFailsAndGoodHostNameSuccess) { RAIIServerParameterControllerForTest _controller{"featureFlagHealthMonitoring", true}; - RAIIServerParameterControllerForTest serverParamController{"activeFaultDurationSecs", 10}; + RAIIServerParameterControllerForTest serverParamController{"activeFaultDurationSecs", 30}; const auto faultFacetType = FaultFacetType::kDns; auto config = std::make_unique<FaultManagerConfig>(); config->setIntensityForType(faultFacetType, HealthObserverIntensityEnum::kCritical); diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript index d23c5871aa7..d6da5af72e4 100644 --- a/src/mongo/db/query/SConscript +++ b/src/mongo/db/query/SConscript @@ -1,6 +1,9 @@ # -*- mode: python -*- -Import("env") +Import([ + "env", + "get_option", +]) env = env.Clone() @@ -10,6 +13,8 @@ env.SConscript( "collation", "datetime", 'optimizer', + 'query_stats', + 'query_shape', ], exports=[ 'env' @@ -19,8 +24,11 @@ env.SConscript( env.Library( target='canonical_query', source=[ - "canonical_query.cpp", - "canonical_query_encoder.cpp", + 'canonical_query.cpp', + 'canonical_query_encoder.cpp', + 'parsed_find_command.cpp', + 'query_shape/find_cmd_shape.cpp', + 'query_stats/find_key.cpp', ], LIBDEPS=[ "$BUILD_DIR/mongo/crypto/encrypted_field_config", @@ -53,6 +61,7 @@ env.Library( "query_planner.cpp", "query_settings.cpp", "query_solution.cpp", + "record_id_range.cpp", "stage_types.cpp", ], LIBDEPS=[ @@ -90,6 +99,17 @@ env.Library( ) env.Library( + target='memory_util', + source=[ + 'util/memory_util.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/util/processinfo', + '$BUILD_DIR/mongo/util/regex_util', + ], +) + +env.Library( target="query_plan_cache", source=[ "classic_plan_cache.cpp", @@ -101,6 +121,7 @@ env.Library( "$BUILD_DIR/mongo/base", "$BUILD_DIR/mongo/db/exec/sbe/query_sbe", "canonical_query", + "memory_util", ] ) @@ -261,17 +282,18 @@ env.Library( env.Library( target="query_knobs", source=[ - 'plan_cache_size_parameter.cpp', 'query_feature_flags.idl', 'query_knobs.idl', + 'query_stats/query_stats_on_parameter_change.cpp', 'sbe_plan_cache_on_parameter_change.cpp', ], LIBDEPS_PRIVATE=[ - "$BUILD_DIR/mongo/db/service_context", + '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/idl/feature_flag', '$BUILD_DIR/mongo/idl/server_parameter', '$BUILD_DIR/third_party/shim_pcrecpp', - ] + 'memory_util', + ], ) env.Library( @@ -348,7 +370,7 @@ env.Library( '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/storage/recovery_unit_base', ], - ) +) env.CppUnitTest( target="db_query_test", @@ -375,10 +397,8 @@ env.CppUnitTest( "interval_test.cpp", "killcursors_request_test.cpp", "lru_key_value_test.cpp", - 'map_reduce_output_format_test.cpp', "parsed_distinct_test.cpp", "plan_cache_indexability_test.cpp", - "plan_cache_size_parameter_test.cpp", "plan_cache_key_info_test.cpp", "plan_cache_test.cpp", "plan_ranker_test.cpp", @@ -392,41 +412,52 @@ env.CppUnitTest( "query_planner_collation_test.cpp", "query_planner_columnar_test.cpp", "query_planner_geo_test.cpp", - "query_planner_pipeline_pushdown_test.cpp", "query_planner_hashed_index_test.cpp", - "query_planner_partialidx_test.cpp", "query_planner_index_test.cpp", "query_planner_operator_test.cpp", "query_planner_options_test.cpp", - "query_planner_tree_test.cpp", + "query_planner_partialidx_test.cpp", + "query_planner_pipeline_pushdown_test.cpp", "query_planner_text_test.cpp", + "query_planner_tree_test.cpp", "query_planner_wildcard_index_test.cpp", "query_request_test.cpp", "query_settings_test.cpp", + 'query_shape/agg_cmd_shape_test.cpp', + 'query_shape/cmd_with_let_shape_test.cpp', + "query_shape/find_cmd_shape_test.cpp", "query_solution_test.cpp", + "record_id_range_test.cpp", "sbe_and_hash_test.cpp", "sbe_and_sorted_test.cpp", + "sbe_shard_filter_test.cpp", "sbe_stage_builder_accumulator_test.cpp", "sbe_stage_builder_lookup_test.cpp", - "sbe_stage_builder_test_fixture.cpp", "sbe_stage_builder_test.cpp", - "sbe_shard_filter_test.cpp", + "sbe_stage_builder_test_fixture.cpp", "shard_filterer_factory_mock.cpp", + "sort_pattern_test.cpp", + "util/deferred_test.cpp", + "util/memory_util_test.cpp", "view_response_formatter_test.cpp", + 'map_reduce_output_format_test.cpp', ], LIBDEPS=[ "$BUILD_DIR/mongo/db/auth/authmocks", "$BUILD_DIR/mongo/db/concurrency/lock_manager", + "$BUILD_DIR/mongo/db/exec/document_value/document_value_test_util", "$BUILD_DIR/mongo/db/exec/sbe/sbe_plan_stage_test", "$BUILD_DIR/mongo/db/multitenancy", "$BUILD_DIR/mongo/db/pipeline/aggregation_request_helper", "$BUILD_DIR/mongo/db/pipeline/document_source_mock", "$BUILD_DIR/mongo/db/query_exec", + '$BUILD_DIR/mongo/db/record_id_helpers', "$BUILD_DIR/mongo/db/repl/replmocks", "$BUILD_DIR/mongo/db/repl/storage_interface_impl", "$BUILD_DIR/mongo/db/service_context_d_test_fixture", "$BUILD_DIR/mongo/db/service_context_test_fixture", "$BUILD_DIR/mongo/dbtests/mocklib", + "$BUILD_DIR/mongo/idl/idl_parser", "$BUILD_DIR/mongo/rpc/rpc", "$BUILD_DIR/mongo/util/clock_source_mock", "collation/collator_factory_mock", @@ -438,6 +469,7 @@ env.CppUnitTest( "query_planner", "query_planner_test_fixture", "query_request", + "query_shape/query_shape", "query_test_service_context", ], ) diff --git a/src/mongo/db/query/canonical_query.cpp b/src/mongo/db/query/canonical_query.cpp index d203a6197bf..32ebde51cfb 100644 --- a/src/mongo/db/query/canonical_query.cpp +++ b/src/mongo/db/query/canonical_query.cpp @@ -36,7 +36,6 @@ #include "mongo/crypto/encryption_fields_gen.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/commands/test_commands_enabled.h" -#include "mongo/db/cst/cst_parser.h" #include "mongo/db/jsobj.h" #include "mongo/db/matcher/expression_array.h" #include "mongo/db/namespace_string.h" @@ -45,111 +44,67 @@ #include "mongo/db/query/collation/collator_factory_interface.h" #include "mongo/db/query/fle/server_rewrite.h" #include "mongo/db/query/indexability.h" +#include "mongo/db/query/parsed_find_command.h" #include "mongo/db/query/projection_parser.h" #include "mongo/db/query/query_planner_common.h" #include "mongo/logv2/log.h" namespace mongo { -namespace { - -bool parsingCanProduceNoopMatchNodes(const ExtensionsCallback& extensionsCallback, - MatchExpressionParser::AllowedFeatureSet allowedFeatures) { - return extensionsCallback.hasNoopExtensions() && - (allowedFeatures & MatchExpressionParser::AllowedFeatures::kText || - allowedFeatures & MatchExpressionParser::AllowedFeatures::kJavascript); -} - -} // namespace // static StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize( OperationContext* opCtx, std::unique_ptr<FindCommandRequest> findCommand, bool explain, - const boost::intrusive_ptr<ExpressionContext>& expCtx, + const boost::intrusive_ptr<ExpressionContext>& givenExpCtx, const ExtensionsCallback& extensionsCallback, MatchExpressionParser::AllowedFeatureSet allowedFeatures, const ProjectionPolicies& projectionPolicies, std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) { - tassert(5746107, - "ntoreturn should not be set on the findCommand", - findCommand->getNtoreturn() == boost::none); - auto status = query_request_helper::validateFindCommandRequest(*findCommand); - if (!status.isOK()) { - return status; - } - - std::unique_ptr<CollatorInterface> collator; - if (!findCommand->getCollation().isEmpty()) { - auto statusWithCollator = CollatorFactoryInterface::get(opCtx->getServiceContext()) - ->makeFromBSON(findCommand->getCollation()); - if (!statusWithCollator.isOK()) { - return statusWithCollator.getStatus(); + if (givenExpCtx) { + // Caller provided an ExpressionContext, let's go ahead and use that. + auto swParsedFind = parsed_find_command::parse(givenExpCtx, + std::move(findCommand), + extensionsCallback, + allowedFeatures, + projectionPolicies); + if (!swParsedFind.isOK()) { + return swParsedFind.getStatus(); } - collator = std::move(statusWithCollator.getValue()); - } - - // Make MatchExpression. - boost::intrusive_ptr<ExpressionContext> newExpCtx; - if (!expCtx.get()) { - invariant(findCommand->getNamespaceOrUUID().nss()); - newExpCtx = make_intrusive<ExpressionContext>(opCtx, - std::move(collator), - *findCommand->getNamespaceOrUUID().nss(), - findCommand->getLegacyRuntimeConstants(), - findCommand->getLet()); + return canonicalize(std::move(givenExpCtx), + std::move(swParsedFind.getValue()), + explain, + std::move(pipeline)); } else { - newExpCtx = expCtx; - // A collator can enter through both the FindCommandRequest and ExpressionContext arguments. - // This invariant ensures that both collators are the same because downstream we - // pull the collator from only one of the ExpressionContext carrier. - if (collator.get() && expCtx->getCollator()) { - invariant(CollatorInterface::collatorsMatch(collator.get(), expCtx->getCollator())); + // No ExpressionContext provided, let's call the override that makes one for us. + auto swResults = parsed_find_command::parse( + opCtx, std::move(findCommand), extensionsCallback, allowedFeatures, projectionPolicies); + if (!swResults.isOK()) { + return swResults.getStatus(); } + auto&& [expCtx, parsedFind] = std::move(swResults.getValue()); + return canonicalize(std::move(expCtx), std::move(parsedFind), explain, std::move(pipeline)); } +} + +// static +StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize( + boost::intrusive_ptr<ExpressionContext> expCtx, + std::unique_ptr<ParsedFindCommand> parsedFind, + bool explain, + std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) { // Make the CQ we'll hopefully return. - std::unique_ptr<CanonicalQuery> cq(new CanonicalQuery()); + auto cq = std::make_unique<CanonicalQuery>(); cq->setExplain(explain); - - StatusWithMatchExpression statusWithMatcher = [&]() -> StatusWithMatchExpression { - if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) { - try { - return cst::parseToMatchExpression( - findCommand->getFilter(), newExpCtx, extensionsCallback); - } catch (const DBException& ex) { - return ex.toStatus(); - } - } else { - return MatchExpressionParser::parse( - findCommand->getFilter(), newExpCtx, extensionsCallback, allowedFeatures); - } - }(); - if (!statusWithMatcher.isOK()) { - return statusWithMatcher.getStatus(); - } - - // Stop counting expressions after they have been parsed to exclude expressions created - // during optimization and other processing steps. - newExpCtx->stopExpressionCounters(); - - std::unique_ptr<MatchExpression> me = std::move(statusWithMatcher.getValue()); - - Status initStatus = - cq->init(opCtx, - std::move(newExpCtx), - std::move(findCommand), - parsingCanProduceNoopMatchNodes(extensionsCallback, allowedFeatures), - std::move(me), - projectionPolicies, - std::move(pipeline), - true /*optimizeMatchExpression*/ - ); - - if (!initStatus.isOK()) { + if (auto initStatus = cq->init(std::move(expCtx), + std::move(parsedFind), + std::move(pipeline), + true /*optimizeMatchExpression*/); + !initStatus.isOK()) { return initStatus; } - return std::move(cq); + return {std::move(cq)}; } // static @@ -163,67 +118,84 @@ StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::makeForSubplanner( baseQuery.root()->numChildren() > i); auto root = baseQuery.root()->getChild(i); auto findCommand = std::make_unique<FindCommandRequest>(baseQuery.nss()); - BSONObjBuilder builder; - root->serialize(&builder, true); - findCommand->setFilter(builder.obj()); + findCommand->setFilter(root->serialize()); findCommand->setProjection(baseQuery.getFindCommandRequest().getProjection().getOwned()); findCommand->setSort(baseQuery.getFindCommandRequest().getSort().getOwned()); findCommand->setCollation(baseQuery.getFindCommandRequest().getCollation().getOwned()); - auto status = query_request_helper::validateFindCommandRequest(*findCommand); - if (!status.isOK()) { - return status; - } // Make the CQ we'll hopefully return. - std::unique_ptr<CanonicalQuery> cq(new CanonicalQuery()); + auto cq = std::make_unique<CanonicalQuery>(); cq->setExplain(baseQuery.getExplain()); - + auto swParsedFind = ParsedFindCommand::withExistingFilter( + baseQuery.getExpCtx(), + baseQuery.getCollator() ? baseQuery.getCollator()->clone() : nullptr, + root->shallowClone(), + std::move(findCommand)); + if (!swParsedFind.isOK()) { + return swParsedFind.getStatus(); + } // Note: we do not optimize the MatchExpression representing the branch of the top-level $or // that we are currently examining. This is because repeated invocations of // MatchExpression::optimize() may change the order of predicates in the MatchExpression, due to // new rewrites being unlocked by previous ones. We need to preserve the order of predicates to // allow index tagging to work properly. See SERVER-84013 for more details. - Status initStatus = cq->init(opCtx, - baseQuery.getExpCtx(), - std::move(findCommand), - baseQuery._canHaveNoopMatchNodes, - root->shallowClone(), - ProjectionPolicies::findProjectionPolicies(), + Status initStatus = cq->init(baseQuery.getExpCtx(), + std::move(swParsedFind.getValue()), {} /* an empty pipeline */, false /*optimizeMatchExpression*/); - if (!initStatus.isOK()) { - return initStatus; - } - return std::move(cq); + invariant(initStatus.isOK()); + return {std::move(cq)}; } -Status CanonicalQuery::init(OperationContext* opCtx, - boost::intrusive_ptr<ExpressionContext> expCtx, - std::unique_ptr<FindCommandRequest> findCommand, - bool canHaveNoopMatchNodes, - std::unique_ptr<MatchExpression> root, - const ProjectionPolicies& projectionPolicies, +Status CanonicalQuery::init(boost::intrusive_ptr<ExpressionContext> expCtx, + std::unique_ptr<ParsedFindCommand> parsedFind, std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline, bool optimizeMatchExpression) { _expCtx = expCtx; - _findCommand = std::move(findCommand); + _findCommand = std::move(parsedFind->findCommandRequest); + _canHaveNoopMatchNodes = parsedFind->canHaveNoopMatchNodes; - _canHaveNoopMatchNodes = canHaveNoopMatchNodes; _forceClassicEngine = internalQueryForceClassicEngine.load(); - auto validStatus = isValid(root.get(), *_findCommand); - if (!validStatus.isOK()) { - return validStatus.getStatus(); - } - auto unavailableMetadata = validStatus.getValue(); - if (optimizeMatchExpression) { - _root = MatchExpression::normalize(std::move(root)); + _root = MatchExpression::normalize(std::move(parsedFind->filter)); } else { - _root = std::move(root); + _root = std::move(parsedFind->filter); + } + + if (parsedFind->proj) { + // The projection will be optimized only if the query is not compatible with SBE or there's + // no user-specified "let" variable. This is to prevent the user-defined variable being + // optimized out. We will optimize the projection later after we are certain that the query + // is ineligible for SBE. + bool shouldOptimizeProj = !expCtx->sbeCompatible || !_findCommand->getLet(); + if (parsedFind->proj->requiresMatchDetails()) { + // Sadly, in some cases the match details cannot be generated from the unoptimized + // MatchExpression. For example, a rooted-$or of equalities won't work to produce the + // details, but if you optimize that query to an $in, it will work. If we were starting + // from scratch, we may disallow this. But it has already been released as working so we + // will keep it so, and here have to re-parse the projection using the new, normalized + // MatchExpression, before we save this projection for later execution. + _proj.emplace(projection_ast::parseAndAnalyze(expCtx, + _findCommand->getProjection(), + _root.get(), + _findCommand->getFilter(), + *parsedFind->savedProjectionPolicies, + shouldOptimizeProj)); + } else { + _proj.emplace(std::move(*parsedFind->proj)); + if (shouldOptimizeProj) { + _proj->optimize(); + } + } } + if (parsedFind->sort) { + _sortPattern = std::move(parsedFind->sort); + } + _pipeline = std::move(pipeline); + // Perform auto-parameterization only if the query is SBE-compatible and caching is enabled. if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) { const bool hasNoTextNodes = !QueryPlannerCommon::hasNode(_root.get(), MatchExpression::TEXT); @@ -240,89 +212,45 @@ Status CanonicalQuery::init(OperationContext* opCtx, } } // The tree must always be valid after normalization. - dassert(isValid(_root.get(), *_findCommand).isOK()); + dassert(parsed_find_command::isValid(_root.get(), *_findCommand).isOK()); if (auto status = isValidNormalized(_root.get()); !status.isOK()) { return status; } - // Validate the projection if there is one. - if (!_findCommand->getProjection().isEmpty()) { - try { - _proj.emplace(projection_ast::parseAndAnalyze(expCtx, - _findCommand->getProjection(), - _root.get(), - _findCommand->getFilter(), - projectionPolicies, - true /* Should optimize? */)); + if (_proj) { + _metadataDeps = _proj->metadataDeps(); - // Fail if any of the projection's dependencies are unavailable. - DepsTracker{unavailableMetadata}.requestMetadata(_proj->metadataDeps()); - } catch (const DBException& e) { - return e.toStatus(); + if (_proj->metadataDeps()[DocumentMetadataFields::kSortKey] && + _findCommand->getSort().isEmpty()) { + return {ErrorCodes::BadValue, "cannot use sortKey $meta projection without a sort"}; } - - _metadataDeps = _proj->metadataDeps(); } - _pipeline = std::move(pipeline); + if (_sortPattern) { + // Be sure to track and add any metadata dependencies from the sort (e.g. text score). + _metadataDeps |= _sortPattern->metadataDeps(parsedFind->unavailableMetadata); - if (_proj && _proj->metadataDeps()[DocumentMetadataFields::kSortKey] && - _findCommand->getSort().isEmpty()) { - return Status(ErrorCodes::BadValue, "cannot use sortKey $meta projection without a sort"); - } - - // If there is a sort, parse it and add any metadata dependencies it induces. - try { - initSortPattern(unavailableMetadata); - } catch (const DBException& ex) { - return ex.toStatus(); + // If the results of this query might have to be merged on a remote node, then that node + // might need the sort key metadata. Request that the plan generates this metadata. + if (_expCtx->needsMerge) { + _metadataDeps.set(DocumentMetadataFields::kSortKey); + } } // If the 'returnKey' option is set, then the plan should produce index key metadata. if (_findCommand->getReturnKey()) { _metadataDeps.set(DocumentMetadataFields::kIndexKey); } - return Status::OK(); } -void CanonicalQuery::initSortPattern(QueryMetadataBitSet unavailableMetadata) { - if (_findCommand->getSort().isEmpty()) { - return; - } - - // A $natural sort is really a hint, and should be handled as such. Furthermore, the downstream - // sort handling code may not expect a $natural sort. - // - // We have already validated that if there is a $natural sort and a hint, that the hint - // also specifies $natural with the same direction. Therefore, it is safe to clear the $natural - // sort and rewrite it as a $natural hint. - if (_findCommand->getSort()[query_request_helper::kNaturalSortField]) { - _findCommand->setHint(_findCommand->getSort().getOwned()); - _findCommand->setSort(BSONObj{}); - } - - if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) { - _sortPattern = cst::parseToSortPattern(_findCommand->getSort(), _expCtx); - } else { - _sortPattern = SortPattern{_findCommand->getSort(), _expCtx}; - } - _metadataDeps |= _sortPattern->metadataDeps(unavailableMetadata); - - // If the results of this query might have to be merged on a remote node, then that node might - // need the sort key metadata. Request that the plan generates this metadata. - if (_expCtx->needsMerge) { - _metadataDeps.set(DocumentMetadataFields::kSortKey); - } -} - void CanonicalQuery::setCollator(std::unique_ptr<CollatorInterface> collator) { auto collatorRaw = collator.get(); // We must give the ExpressionContext the same collator. _expCtx->setCollator(std::move(collator)); - // The collator associated with the match expression tree is now invalid, since we have reset - // the collator owned by the ExpressionContext. + // The collator associated with the match expression tree is now invalid, since we have + // reset the collator owned by the ExpressionContext. _root->setCollator(collatorRaw); } @@ -356,138 +284,9 @@ bool CanonicalQuery::isSimpleIdQuery(const BSONObj& query) { return hasID; } -size_t CanonicalQuery::countNodes(const MatchExpression* root, MatchExpression::MatchType type) { - size_t sum = 0; - if (type == root->matchType()) { - sum = 1; - } - for (size_t i = 0; i < root->numChildren(); ++i) { - sum += countNodes(root->getChild(i), type); - } - return sum; -} - -/** - * Does 'root' have a subtree of type 'subtreeType' with a node of type 'childType' inside? - */ -bool hasNodeInSubtree(const MatchExpression* root, - MatchExpression::MatchType childType, - MatchExpression::MatchType subtreeType) { - if (subtreeType == root->matchType()) { - return QueryPlannerCommon::hasNode(root, childType); - } - for (size_t i = 0; i < root->numChildren(); ++i) { - if (hasNodeInSubtree(root->getChild(i), childType, subtreeType)) { - return true; - } - } - return false; -} - -StatusWith<QueryMetadataBitSet> CanonicalQuery::isValid(const MatchExpression* root, - const FindCommandRequest& findCommand) { - QueryMetadataBitSet unavailableMetadata{}; - - // There can only be one TEXT. If there is a TEXT, it cannot appear inside a NOR. - // - // Note that the query grammar (as enforced by the MatchExpression parser) forbids TEXT - // inside of value-expression clauses like NOT, so we don't check those here. - size_t numText = countNodes(root, MatchExpression::TEXT); - if (numText > 1) { - return Status(ErrorCodes::BadValue, "Too many text expressions"); - } else if (1 == numText) { - if (hasNodeInSubtree(root, MatchExpression::TEXT, MatchExpression::NOR)) { - return Status(ErrorCodes::BadValue, "text expression not allowed in nor"); - } - } else { - // Text metadata is not available. - unavailableMetadata.set(DocumentMetadataFields::kTextScore); - } - - // There can only be one NEAR. If there is a NEAR, it must be either the root or the root - // must be an AND and its child must be a NEAR. - size_t numGeoNear = countNodes(root, MatchExpression::GEO_NEAR); - if (numGeoNear > 1) { - return Status(ErrorCodes::BadValue, "Too many geoNear expressions"); - } else if (1 == numGeoNear) { - // Do nothing, we will perform extra checks in CanonicalQuery::isValidNormalized. - } else { - // Geo distance and geo point metadata are unavailable. - unavailableMetadata |= DepsTracker::kAllGeoNearData; - } - - const BSONObj& sortObj = findCommand.getSort(); - BSONElement sortNaturalElt = sortObj["$natural"]; - const BSONObj& hintObj = findCommand.getHint(); - BSONElement hintNaturalElt = hintObj["$natural"]; - - if (sortNaturalElt && sortObj.nFields() != 1) { - return Status(ErrorCodes::BadValue, - str::stream() << "Cannot include '$natural' in compound sort: " << sortObj); - } - - if (hintNaturalElt && hintObj.nFields() != 1) { - return Status(ErrorCodes::BadValue, - str::stream() << "Cannot include '$natural' in compound hint: " << hintObj); - } - - // NEAR cannot have a $natural sort or $natural hint. - if (numGeoNear > 0) { - if (sortNaturalElt) { - return Status(ErrorCodes::BadValue, - "geoNear expression not allowed with $natural sort order"); - } - - if (hintNaturalElt) { - return Status(ErrorCodes::BadValue, - "geoNear expression not allowed with $natural hint"); - } - } - - // TEXT and NEAR cannot both be in the query. - if (numText > 0 && numGeoNear > 0) { - return Status(ErrorCodes::BadValue, "text and geoNear not allowed in same query"); - } - - // TEXT and {$natural: ...} sort order cannot both be in the query. - if (numText > 0 && sortNaturalElt) { - return Status(ErrorCodes::BadValue, "text expression not allowed with $natural sort order"); - } - - // TEXT and hint cannot both be in the query. - if (numText > 0 && !hintObj.isEmpty()) { - return Status(ErrorCodes::BadValue, "text and hint not allowed in same query"); - } - - // TEXT and tailable are incompatible. - if (numText > 0 && findCommand.getTailable()) { - return Status(ErrorCodes::BadValue, "text and tailable cursor not allowed in same query"); - } - - // NEAR and tailable are incompatible. - if (numGeoNear > 0 && findCommand.getTailable()) { - return Status(ErrorCodes::BadValue, - "Tailable cursors and geo $near cannot be used together"); - } - - // $natural sort order must agree with hint. - if (sortNaturalElt) { - if (!hintObj.isEmpty() && !hintNaturalElt) { - return Status(ErrorCodes::BadValue, "index hint not allowed with $natural sort order"); - } - if (hintNaturalElt) { - if (hintNaturalElt.numberInt() != sortNaturalElt.numberInt()) { - return Status(ErrorCodes::BadValue, - "$natural hint must be in the same direction as $natural sort order"); - } - } - } - - return unavailableMetadata; -} - Status CanonicalQuery::isValidNormalized(const MatchExpression* root) { - if (auto numGeoNear = countNodes(root, MatchExpression::GEO_NEAR); numGeoNear > 0) { + if (auto numGeoNear = QueryPlannerCommon::countNodes(root, MatchExpression::GEO_NEAR); + numGeoNear > 0) { tassert(5705300, "Only one geo $near expression is expected", numGeoNear == 1); auto topLevel = false; diff --git a/src/mongo/db/query/canonical_query.h b/src/mongo/db/query/canonical_query.h index 79b49c08463..570b71b6fae 100644 --- a/src/mongo/db/query/canonical_query.h +++ b/src/mongo/db/query/canonical_query.h @@ -37,6 +37,7 @@ #include "mongo/db/matcher/extensions_callback_noop.h" #include "mongo/db/pipeline/inner_pipeline_stage_interface.h" #include "mongo/db/query/collation/collator_interface.h" +#include "mongo/db/query/parsed_find_command.h" #include "mongo/db/query/projection.h" #include "mongo/db/query/projection_policies.h" #include "mongo/db/query/query_request_helper.h" @@ -76,6 +77,15 @@ public: std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline = {}); /** + * Creates a CanonicalQuery from a ParsedFindCommand. Uses 'expCtx->opCtx', which must be valid. + */ + static StatusWith<std::unique_ptr<CanonicalQuery>> canonicalize( + boost::intrusive_ptr<ExpressionContext> expCtx, + std::unique_ptr<ParsedFindCommand> parsedFind, + bool explain = false, + std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline = {}); + + /** * For testing or for internal clients to use. */ @@ -93,32 +103,22 @@ public: static bool isSimpleIdQuery(const BSONObj& query); /** - * Validates the match expression 'root' as well as the query specified by 'request', checking - * for illegal combinations of operators. Returns a non-OK status if any such illegal - * combination is found. - * - * This method can be called both on normalized and non-normalized 'root'. However, some checks - * can only be performed once the match expressions is normalized. To perform these checks one - * can call 'isValidNormalized()'. - * - * On success, returns a bitset indicating which types of metadata are *unavailable*. For - * example, if 'root' does not contain a $text predicate, then the returned metadata bitset will - * indicate that text score metadata is unavailable. This means that if subsequent - * $meta:"textScore" expressions are found during analysis of the query, we should raise in an - * error. + * Perform validation checks on the normalized 'root' which could not be checked before + * normalization - those should happen in parsed_find_command::isValid(). */ - static StatusWith<QueryMetadataBitSet> isValid(const MatchExpression* root, - const FindCommandRequest& findCommand); + static Status isValidNormalized(const MatchExpression* root); /** - * Perform additional validation checks on the normalized 'root'. + * For internal use only - but public for accessibility for make_unique(). You must go through + * canonicalize to create a CanonicalQuery. */ - static Status isValidNormalized(const MatchExpression* root); + CanonicalQuery() {} const NamespaceString nss() const { invariant(_findCommand->getNamespaceOrUUID().nss()); return *_findCommand->getNamespaceOrUUID().nss(); } + const std::string ns() const { return nss().ns(); } @@ -197,11 +197,6 @@ public: std::string toStringShort() const; /** - * Returns a count of 'type' nodes in expression tree. - */ - static size_t countNodes(const MatchExpression* root, MatchExpression::MatchType type); - - /** * Returns true if this canonical query may have converted extensions such as $where and $text * into no-ops during parsing. This will be the case if it allowed $where and $text in parsing, * but parsed using an ExtensionsCallbackNoop. This does not guarantee that a $where or $text @@ -262,25 +257,18 @@ public: return _pipeline; } -private: - // You must go through canonicalize to create a CanonicalQuery. - CanonicalQuery() {} + void optimizeProjection() { + if (_proj) { + _proj->optimize(); + } + } - Status init(OperationContext* opCtx, - boost::intrusive_ptr<ExpressionContext> expCtx, - std::unique_ptr<FindCommandRequest> findCommand, - bool canHaveNoopMatchNodes, - std::unique_ptr<MatchExpression> root, - const ProjectionPolicies& projectionPolicies, +private: + Status init(boost::intrusive_ptr<ExpressionContext> expCtx, + std::unique_ptr<ParsedFindCommand> parsedFind, std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline, bool optimizeMatchExpression); - // Initializes '_sortPattern', adding any metadata dependencies implied by the sort. - // - // Throws a UserException if the sort is illegal, or if any metadata type in - // 'unavailableMetadata' is required. - void initSortPattern(QueryMetadataBitSet unavailableMetadata); - boost::intrusive_ptr<ExpressionContext> _expCtx; std::unique_ptr<FindCommandRequest> _findCommand; diff --git a/src/mongo/db/query/canonical_query_encoder.cpp b/src/mongo/db/query/canonical_query_encoder.cpp index c8a8cd29d48..61ee86dc2c8 100644 --- a/src/mongo/db/query/canonical_query_encoder.cpp +++ b/src/mongo/db/query/canonical_query_encoder.cpp @@ -989,7 +989,7 @@ public: void preVisit(const MatchExpression* expr) { // Encode the type of the node as well as the path (if there is a non-empty path). - _builder->appendStr(encodeMatchType(expr->matchType())); + _builder->appendCStr(encodeMatchType(expr->matchType())); encodeUserString(expr->path(), _builder); // The node encodes itself, and then its children. @@ -1053,7 +1053,7 @@ std::string encodeSBE(const CanonicalQuery& cq) { encodeKeyForAutoParameterizedMatchSBE(cq.root(), &bufBuilder); bufBuilder.appendBuf(proj.objdata(), proj.objsize()); - bufBuilder.appendStr(strBuilderEncoded, false /* includeEndingNull */); + bufBuilder.appendStrBytes(strBuilderEncoded); encodeFindCommandRequest(cq.getFindCommandRequest(), &bufBuilder); diff --git a/src/mongo/db/query/canonical_query_test.cpp b/src/mongo/db/query/canonical_query_test.cpp index 497d2671270..5cba0bcc3f4 100644 --- a/src/mongo/db/query/canonical_query_test.cpp +++ b/src/mongo/db/query/canonical_query_test.cpp @@ -281,9 +281,50 @@ TEST(CanonicalQueryTest, CanonicalizeFromBaseQuery) { MatchExpression* firstClauseExpr = baseCq->root()->getChild(0); auto childCq = assertGet(CanonicalQuery::makeForSubplanner(opCtx.get(), *baseCq, 0)); - BSONObjBuilder expectedFilter; - firstClauseExpr->serialize(&expectedFilter); - ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getFilter(), expectedFilter.obj()); + ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getFilter(), firstClauseExpr->serialize()); + + ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getProjection(), + baseCq->getFindCommandRequest().getProjection()); + ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getSort(), + baseCq->getFindCommandRequest().getSort()); + ASSERT_TRUE(childCq->getExplain()); +} + +TEST(CanonicalQueryTest, CanonicalizeFromBaseQueryWithSpecialFeature) { + // Like the above test, but use $text which is a 'special feature' not always allowed. This is + // meant to reproduce SERVER-XYZ. + QueryTestServiceContext serviceContext; + auto opCtx = serviceContext.makeOperationContext(); + + const bool isExplain = true; + const std::string cmdStr = R"({ + find:'bogusns', + filter: { + $or:[ + {a: 'foo'}, + {$text: {$search: 'bar'}} + ] + }, + projection: {a:1}, + sort: {b:1}, + $db: 'test' + })"; + auto findCommand = query_request_helper::makeFromFindCommandForTests(fromjson(cmdStr)); + auto baseCq = + assertGet(CanonicalQuery::canonicalize(opCtx.get(), + std::move(findCommand), + isExplain, + nullptr, + ExtensionsCallbackNoop(), + MatchExpressionParser::kAllowAllSpecialFeatures)); + + // Note: be sure to use the second child to get $text, since we 'normalize' and sort the + // MatchExpression tree as part of canonicalization. This will put the text search clause + // second. + MatchExpression* secondClauseExpr = baseCq->root()->getChild(1); + auto childCq = assertGet(CanonicalQuery::makeForSubplanner(opCtx.get(), *baseCq, 1)); + + ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getFilter(), secondClauseExpr->serialize()); ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getProjection(), baseCq->getFindCommandRequest().getProjection()); diff --git a/src/mongo/db/query/classic_plan_cache.h b/src/mongo/db/query/classic_plan_cache.h index 8d2e6889bfb..e510b48dd6d 100644 --- a/src/mongo/db/query/classic_plan_cache.h +++ b/src/mongo/db/query/classic_plan_cache.h @@ -235,7 +235,12 @@ using PlanCacheEntry = PlanCacheEntryBase<SolutionCacheData, plan_cache_debug_in using CachedSolution = CachedPlanHolder<SolutionCacheData, plan_cache_debug_info::DebugInfo>; struct BudgetEstimator { - size_t operator()(const std::shared_ptr<const PlanCacheEntry>&) { + /** + * This estimator function is called when an entry is added or removed to LRU cache in order to + * make sure the total plan cache size does not exceed the maximum size. + */ + + size_t operator()(const PlanCacheKey&, const std::shared_ptr<const PlanCacheEntry>&) { return 1; } }; diff --git a/src/mongo/db/query/explain.cpp b/src/mongo/db/query/explain.cpp index 414badb8332..796875bca68 100644 --- a/src/mongo/db/query/explain.cpp +++ b/src/mongo/db/query/explain.cpp @@ -124,7 +124,7 @@ void generatePlannerInfo(PlanExecutor* exec, auto query = exec->getCanonicalQuery(); if (nullptr != query) { BSONObjBuilder parsedQueryBob(plannerBob.subobjStart("parsedQuery")); - query->root()->serialize(&parsedQueryBob); + query->root()->serialize(&parsedQueryBob, {}); parsedQueryBob.doneFast(); if (query->getCollator()) { diff --git a/src/mongo/db/query/find.cpp b/src/mongo/db/query/find.cpp index d2f36155e13..cd8e21bf6c7 100644 --- a/src/mongo/db/query/find.cpp +++ b/src/mongo/db/query/find.cpp @@ -107,19 +107,28 @@ void endQueryOp(OperationContext* opCtx, const CollectionPtr& collection, const PlanExecutor& exec, long long numResults, - CursorId cursorId) { + boost::optional<ClientCursorPin&> cursor, + const BSONObj& cmdObj) { auto curOp = CurOp::get(opCtx); - // Fill out basic CurOp query exec properties. - curOp->debug().nreturned = numResults; - curOp->debug().cursorid = (0 == cursorId ? -1 : cursorId); - curOp->debug().cursorExhausted = (0 == cursorId); + // Fill out basic CurOp query exec properties. More metrics (nreturned and executionTime) + // are collected within collectQueryStatsMongod. + curOp->debug().cursorid = (cursor.has_value() ? cursor->getCursor()->cursorid() : -1); + curOp->debug().cursorExhausted = !cursor.has_value(); + curOp->debug().additiveMetrics.nBatches = 1; // Fill out CurOp based on explain summary statistics. PlanSummaryStats summaryStats; auto&& explainer = exec.getPlanExplainer(); explainer.getSummaryStats(&summaryStats); curOp->debug().setPlanSummaryMetrics(summaryStats); + curOp->setEndOfOpMetrics(numResults); + + if (cursor) { + collectQueryStatsMongod(opCtx, *cursor); + } else { + collectQueryStatsMongod(opCtx, std::move(curOp->debug().queryStatsInfo.key)); + } if (collection) { CollectionQueryInfo::get(collection).notifyOfQuery(opCtx, collection, summaryStats); diff --git a/src/mongo/db/query/find.h b/src/mongo/db/query/find.h index 2b45efcd602..235ebf91ed8 100644 --- a/src/mongo/db/query/find.h +++ b/src/mongo/db/query/find.h @@ -73,6 +73,7 @@ void endQueryOp(OperationContext* opCtx, const CollectionPtr& collection, const PlanExecutor& exec, long long numResults, - CursorId cursorId); + boost::optional<ClientCursorPin&> cursor, + const BSONObj& cmdObj); } // namespace mongo diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index 5e1d413745e..602be15dbc2 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -29,10 +29,13 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery +#include "mongo/db/curop.h" #include "mongo/platform/basic.h" #include "mongo/db/query/get_executor.h" +#include "mongo/util/duration.h" +#include "mongo/util/tick_source.h" #include <boost/optional.hpp> #include <limits> #include <memory> @@ -107,6 +110,7 @@ #include "mongo/logv2/log.h" #include "mongo/scripting/engine.h" #include "mongo/util/str.h" +#include "mongo/util/timer.h" namespace mongo { MONGO_FAIL_POINT_DEFINE(includeFakeColumnarIndex); @@ -603,6 +607,8 @@ public: StatusWith<std::unique_ptr<ResultType>> prepare() { const auto& mainColl = getMainCollection(); + + ON_BLOCK_EXIT([&] { CurOp::get(_opCtx)->stopQueryPlanningTimer(); }); if (!mainColl) { LOGV2_DEBUG(20921, 2, @@ -702,10 +708,8 @@ public: "Only one plan is available", "query"_attr = redact(_cq->toStringShort()), "planSummary"_attr = result->getPlanSummary()); - return std::move(result); } - return buildMultiPlan(std::move(solutions)); } @@ -1317,16 +1321,16 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getSlotBasedExe auto&& [roots, solutions] = planningResult->extractResultData(); // In some circumstances (e.g. when have multiple candidate plans or using a cached one), we // might need to execute the plan(s) to pick the best one or to confirm the choice. - if (auto planner = makeRuntimePlannerIfNeeded(opCtx, - collections, - cq.get(), - solutions.size(), - planningResult->decisionWorks(), - planningResult->needsSubplanning(), - yieldPolicy.get(), - plannerParams.options)) { + if (auto runTimePlanner = makeRuntimePlannerIfNeeded(opCtx, + collections, + cq.get(), + solutions.size(), + planningResult->decisionWorks(), + planningResult->needsSubplanning(), + yieldPolicy.get(), + plannerParams.options)) { // Do the runtime planning and pick the best candidate plan. - auto candidates = planner->plan(std::move(solutions), std::move(roots)); + auto candidates = runTimePlanner->plan(std::move(solutions), std::move(roots)); return plan_executor_factory::make(opCtx, std::move(cq), @@ -1397,6 +1401,11 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutor( } } + // There's a special case of the projection optimization being skipped when a query has any + // user-defined "let" variable and the query may be run with SBE. Here we make sure the + // projection is optimized for the classic engine. + canonicalQuery->optimizeProjection(); + return getClassicExecutor( opCtx, mainColl, std::move(canonicalQuery), yieldPolicy, plannerParams); } @@ -1409,6 +1418,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutor( PlanYieldPolicy::YieldPolicy yieldPolicy, size_t plannerOptions) { MultipleCollectionAccessor multi{collection}; + return getExecutor(opCtx, multi, std::move(canonicalQuery), @@ -1450,6 +1460,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorFind std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages, bool permitYield, size_t plannerOptions) { + MultipleCollectionAccessor multi{*coll}; return getExecutorFind(opCtx, multi, @@ -1665,6 +1676,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorDele ClassicPrepareExecutionHelper helper{ opCtx, collection, ws.get(), cq.get(), nullptr, defaultPlannerOptions}; auto executionResult = helper.prepare(); + if (!executionResult.isOK()) { return executionResult.getStatus(); } @@ -1852,6 +1864,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorUpda ClassicPrepareExecutionHelper helper{ opCtx, collection, ws.get(), cq.get(), nullptr, defaultPlannerOptions}; auto executionResult = helper.prepare(); + if (!executionResult.isOK()) { return executionResult.getStatus(); } @@ -2125,8 +2138,8 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorCoun OperationContext* opCtx = expCtx->opCtx; std::unique_ptr<WorkingSet> ws = std::make_unique<WorkingSet>(); - auto findCommand = std::make_unique<FindCommandRequest>(nss); + findCommand->setFilter(request.getQuery()); auto collation = request.getCollation().value_or(BSONObj()); findCommand->setCollation(collation); @@ -2200,6 +2213,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorCoun if (!executionResult.isOK()) { return executionResult.getStatus(); } + auto [root, querySolution] = executionResult.getValue()->extractResultData(); invariant(root); @@ -2208,6 +2222,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorCoun expCtx.get(), collection, limit, skip, ws.get(), root.release()); // We must have a tree of stages in order to have a valid plan executor, but the query // solution may be NULL. Takes ownership of all args other than 'collection' and 'opCtx' + return plan_executor_factory::make(std::move(cq), std::move(ws), std::move(root), diff --git a/src/mongo/db/query/indexability.h b/src/mongo/db/query/indexability.h index 48b9e0d91b4..1736d92a89b 100644 --- a/src/mongo/db/query/indexability.h +++ b/src/mongo/db/query/indexability.h @@ -136,6 +136,20 @@ public: } /** + * Returns true if 'me' is ELEM_MATCH_OBJECT and has non-empty path component. + * + * Note: we skip empty path components since they are not allowed in index key patterns. + * Therefore, $elemMatch with an empty path component can never use an index. + * + * Example: {"": {$elemMatch: {a: "hi", b: "bye"}}. + * In this case the predicate cannot use any indexes since the $elemMatch is with an empty path + * component. + */ + static bool isBoundsGeneratingElemMatchObject(const MatchExpression* me) { + return arrayUsesIndexOnChildren(me) && !me->path().empty(); + } + + /** * Returns true if 'me' is a NOT, and the child of the NOT can use * an index on its own field. */ diff --git a/src/mongo/db/query/lru_key_value.h b/src/mongo/db/query/lru_key_value.h index 88186c70923..2786e1c40ab 100644 --- a/src/mongo/db/query/lru_key_value.h +++ b/src/mongo/db/query/lru_key_value.h @@ -28,7 +28,6 @@ */ #pragma once - #include <fmt/format.h> #include <list> #include <memory> @@ -40,30 +39,56 @@ namespace mongo { /** + * 'InsertionEvictionListener' class to use with 'LRUBudgetTracker' that will always noop. + */ +class NoopInsertionEvictionListener { +public: + // Called when a key-value pair is being inserted. Parameters are the key-value pair and its + // estimated size. + template <class K, class V> + void onInsert(const K&, const V&, size_t) {} + + // Called when a key-value pair is being evicted. Parameters are the key-value pair and its + // estimated size. + template <class K, class V> + void onEvict(const K&, const V&, size_t) {} + + // Called when the cache is being cleared. Parameter is the estimated size of the key-value + // pairs in the cache before it was cleared. + void onClear(size_t) {} +}; + +/** * This class tracks a size of entries in 'LRUKeyValue'. * The size can be understood as a number of the entries, an amount of memory they occupied, * or any other value defined by the template parameter 'Estimator'. * The 'Estimator' must be deterministic and always return the same value for the same entry. + * The 'InsertionEvictionListener' will be called on every insertion and eviction as well as when + * the cache is cleared. */ -template <typename V, typename Estimator> +template <class K, class V, typename Estimator, typename InsertionEvictionListener> class LRUBudgetTracker { public: LRUBudgetTracker(size_t maxBudget) : _max(maxBudget), _current(0) {} - void onAdd(const V& v) { - _current += _estimator(v); + void onAdd(const K& k, const V& v) { + size_t budget = _estimator(k, v); + _current += budget; + _listener.onInsert(k, v, budget); } - void onRemove(const V& v) { + void onRemove(const K& k, const V& v) { using namespace fmt::literals; - size_t budget = _estimator(v); + size_t budget = _estimator(k, v); tassert(5968300, "LRU budget underflow: current={}, budget={} "_format(_current, budget), _current >= budget); _current -= budget; + _listener.onEvict(k, v, budget); } void onClear() { + _listener.onClear(_current); _current = 0; } @@ -84,6 +109,7 @@ private: size_t _max; size_t _current; Estimator _estimator; + InsertionEvictionListener _listener; }; /** @@ -91,6 +117,9 @@ private: * policy. The size allowed in the kv-store is controlled by 'LRUBudgetTracker' * set in the constructor. * + * An 'InsertionEvictionListener' may optionally be specified to track the insertion and eviction of + * each key-value pair. + * * Caveat: * This kv-store is NOT thread safe! The client to this utility is responsible * for protecting concurrent access to the LRU store if used in a threaded @@ -102,7 +131,12 @@ private: * TODO: We could move this into the util/ directory and do any cleanup necessary to make it * fully general. */ -template <class K, class V, class BudgetEstimator, class KeyHasher = std::hash<K>> +template <class K, + class V, + class KeyValueBudgetEstimator, + class InsertionEvictionListener = NoopInsertionEvictionListener, + class KeyHasher = std::hash<K>, + class Eq = std::equal_to<K>> class LRUKeyValue { public: LRUKeyValue(size_t maxSize) : _budgetTracker{maxSize} {} @@ -111,13 +145,13 @@ public: clear(); } - typedef std::pair<K, V> KVListEntry; + typedef std::pair<const K*, V> KVListEntry; typedef std::list<KVListEntry> KVList; typedef typename KVList::iterator KVListIt; typedef typename KVList::const_iterator KVListConstIt; - typedef stdx::unordered_map<K, KVListIt, KeyHasher> KVMap; + typedef stdx::unordered_map<K, KVListIt, KeyHasher, Eq> KVMap; typedef typename KVMap::const_iterator KVMapConstIt; // These type declarations are required by the 'Partitioned' utility. @@ -136,14 +170,15 @@ public: KVMapConstIt i = _kvMap.find(key); if (i != _kvMap.end()) { KVListIt found = i->second; - _budgetTracker.onRemove(found->second); + _budgetTracker.onRemove(key, found->second); _kvMap.erase(i); _kvList.erase(found); } - _budgetTracker.onAdd(entry); - _kvList.push_front(std::make_pair(key, std::move(entry))); + _budgetTracker.onAdd(key, entry); + _kvList.push_front(std::make_pair(nullptr, std::move(entry))); _kvMap[key] = _kvList.begin(); + _kvList.begin()->first = &(_kvMap.find(key)->first); return evict(); } @@ -161,10 +196,11 @@ public: KVListIt found = i->second; // Promote the kv-store entry to the front of the list. It is now the most recently used. - _kvList.push_front(std::make_pair(key, std::move(found->second))); + _kvList.push_front(std::make_pair(nullptr, std::move(found->second))); _kvMap.erase(i); _kvList.erase(found); _kvMap[key] = _kvList.begin(); + _kvList.begin()->first = &(_kvMap.find(key)->first); return _kvList.begin(); } @@ -179,7 +215,7 @@ public: return false; } KVListIt found = i->second; - _budgetTracker.onRemove(found->second); + _budgetTracker.onRemove(key, found->second); _kvMap.erase(i); _kvList.erase(found); return true; @@ -193,9 +229,9 @@ public: size_t removeIf(KeyValuePredicate predicate) { size_t removed = 0; for (auto it = _kvList.begin(); it != _kvList.end();) { - if (predicate(it->first, *it->second)) { - _budgetTracker.onRemove(it->second); - _kvMap.erase(it->first); + if (predicate(*it->first, *it->second)) { + _budgetTracker.onRemove(*it->first, it->second); + _kvMap.erase(*it->first); it = _kvList.erase(it); ++removed; } else { @@ -209,9 +245,9 @@ public: * Deletes all entries in the kv-store. */ void clear() { - _budgetTracker.onClear(); _kvList.clear(); _kvMap.clear(); + _budgetTracker.onClear(); } /** @@ -258,8 +294,8 @@ private: while (_budgetTracker.isOverBudget()) { invariant(!_kvList.empty()); - _budgetTracker.onRemove(_kvList.back().second); - _kvMap.erase(_kvList.back().first); + _budgetTracker.onRemove(*_kvList.back().first, _kvList.back().second); + _kvMap.erase(*_kvList.back().first); _kvList.pop_back(); ++nEvicted; @@ -268,13 +304,14 @@ private: return nEvicted; } - LRUBudgetTracker<V, BudgetEstimator> _budgetTracker; + LRUBudgetTracker<K, V, KeyValueBudgetEstimator, InsertionEvictionListener> _budgetTracker; // (K, V) pairs are stored in this std::list. They are sorted in order of use, where the front // is the most recently used and the back is the least recently used. mutable KVList _kvList; // Maps from a key to the corresponding std::list entry. + // TODO: SERVER-73659 LRUKeyValue should track and include the size of _kvMap in overall budget. mutable KVMap _kvMap; }; diff --git a/src/mongo/db/query/lru_key_value_test.cpp b/src/mongo/db/query/lru_key_value_test.cpp index 3ebf47267fb..6dcdfc5a4ea 100644 --- a/src/mongo/db/query/lru_key_value_test.cpp +++ b/src/mongo/db/query/lru_key_value_test.cpp @@ -64,15 +64,15 @@ struct ValueType { struct TrivialBudgetEstimator { static constexpr size_t kSize = 1; - size_t operator()(const ValueType&) { + size_t operator()(const int&, const ValueType&) { return kSize; } - size_t operator()(const std::shared_ptr<int>&) { + size_t operator()(const int&, const std::unique_ptr<int>&) { return kSize; } - size_t operator()(const std::unique_ptr<int>&) { + size_t operator()(const int&, const std::shared_ptr<int>) { return kSize; } }; @@ -87,7 +87,7 @@ struct NonTrivialEntry { }; struct NonTrivialBudgetEstimator { - size_t operator()(const std::shared_ptr<NonTrivialEntry>& value) { + size_t operator()(const int& key, const std::shared_ptr<NonTrivialEntry> value) { return value->budgetSize; } }; @@ -95,8 +95,40 @@ struct NonTrivialBudgetEstimator { using NonTrivialTestSharedPtrValue = LRUKeyValue<size_t, std::shared_ptr<NonTrivialEntry>, NonTrivialBudgetEstimator>; -template <typename Key, typename Value, typename Estimator> -void assertInKVStore(LRUKeyValue<Key, Value, Estimator>& cache, Key key, Value value) { +class NonTrivialInsertionEvictionListener { +public: + NonTrivialInsertionEvictionListener() { + keyTotal = 0; + valueTotal = 0; + budgetTotal = 0; + } + + void onInsert(const int& k, const ValueType& v, size_t budget) { + keyTotal += k; + valueTotal += v.val; + budgetTotal += budget; + } + + void onEvict(const int& k, const ValueType& v, size_t budget) { + keyTotal -= k; + valueTotal -= v.val; + budgetTotal -= budget; + } + + void onClear(size_t budget) { + budgetTotal -= budget; + } + + static size_t keyTotal; + static size_t valueTotal; + static size_t budgetTotal; +}; +size_t NonTrivialInsertionEvictionListener::keyTotal; +size_t NonTrivialInsertionEvictionListener::valueTotal; +size_t NonTrivialInsertionEvictionListener::budgetTotal; + +template <typename Key, typename Value, typename Estimator, typename Listener> +void assertInKVStore(LRUKeyValue<Key, Value, Estimator, Listener>& cache, Key key, Value value) { ASSERT_TRUE(cache.hasKey(key)); auto s = cache.get(key); ASSERT(s.isOK()); @@ -105,8 +137,8 @@ void assertInKVStore(LRUKeyValue<Key, Value, Estimator>& cache, Key key, Value v ASSERT_EQUALS(*(kvItr->second), *value); } -template <typename Key, typename Value, typename Estimator> -void assertNotInKVStore(LRUKeyValue<Key, Value, Estimator>& cache, Key key) { +template <typename Key, typename Value, typename Estimator, typename Listener> +void assertNotInKVStore(LRUKeyValue<Key, Value, Estimator, Listener>& cache, Key key) { ASSERT_FALSE(cache.hasKey(key)); auto s = cache.get(key); ASSERT(!s.isOK()); @@ -295,10 +327,10 @@ TEST(LRUKeyValueTest, IterationTest) { cache.add(2, std::make_shared<int>(2)); auto i = cache.begin(); - ASSERT_EQUALS(i->first, 2); + ASSERT_EQUALS(*i->first, 2); ASSERT_EQUALS(*i->second, 2); ++i; - ASSERT_EQUALS(i->first, 1); + ASSERT_EQUALS(*i->first, 1); ASSERT_EQUALS(*i->second, 1); ++i; ASSERT(i == cache.end()); @@ -356,7 +388,8 @@ TEST(LRUKeyValueTest, UniquePtrKeyValue) { assertNotInKVStore(cacheForEviction, 1); // The entry with key '1' has been Evicted. } -using TestScalarValue = LRUKeyValue<int, ValueType, TrivialBudgetEstimator>; +using TestScalarValue = + LRUKeyValue<int, ValueType, TrivialBudgetEstimator, NonTrivialInsertionEvictionListener>; void assertValueInKVStore(TestScalarValue& cache, int key, ValueType value) { ASSERT_TRUE(cache.hasKey(key)); @@ -373,9 +406,17 @@ TEST(LRUKeyValueTest, ScalarKeyValue) { assertValueInKVStore(cache, 1, ValueType{2}); assertNotInKVStore(cache, 3); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::keyTotal, 1); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::valueTotal, 2); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 1); + cache.add(1, ValueType{3}); assertValueInKVStore(cache, 1, ValueType{3}); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::keyTotal, 1); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::valueTotal, 3); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 1); + // Test eviction. TestScalarValue cacheForEviction{2}; cacheForEviction.add(1, ValueType{1}); @@ -384,6 +425,18 @@ TEST(LRUKeyValueTest, ScalarKeyValue) { ASSERT_EQUALS(cacheForEviction.size(), static_cast<size_t>(2)); assertNotInKVStore(cacheForEviction, 1); // The entry with key '1' has been Evicted. + + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::keyTotal, 5); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::valueTotal, 5); + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 2); + + // Clear the remaining values. + cacheForEviction.clear(); + + assertNotInKVStore(cacheForEviction, 2); // The entry with key '2' has been Evicted. + assertNotInKVStore(cacheForEviction, 3); // The entry with key '3' has been Evicted. + + ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 0); } } // namespace diff --git a/src/mongo/db/query/parsed_find_command.cpp b/src/mongo/db/query/parsed_find_command.cpp new file mode 100644 index 00000000000..2ef2e955c06 --- /dev/null +++ b/src/mongo/db/query/parsed_find_command.cpp @@ -0,0 +1,381 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/parsed_find_command.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + +#include "mongo/db/cst/cst_parser.h" +#include "mongo/db/query/collation/collator_factory_interface.h" +#include "mongo/db/query/projection_ast_util.h" +#include "mongo/db/query/projection_parser.h" +#include "mongo/db/query/query_planner_common.h" +#include "mongo/db/query/query_request_helper.h" +#include "mongo/logv2/log.h" + +namespace mongo { + +namespace { +/** + * Does 'root' have a subtree of type 'subtreeType' with a node of type 'childType' inside? + */ +bool hasNodeInSubtree(const MatchExpression* root, + MatchExpression::MatchType childType, + MatchExpression::MatchType subtreeType) { + if (subtreeType == root->matchType()) { + return QueryPlannerCommon::hasNode(root, childType); + } + for (size_t i = 0; i < root->numChildren(); ++i) { + if (hasNodeInSubtree(root->getChild(i), childType, subtreeType)) { + return true; + } + } + return false; +} + +bool parsingCanProduceNoopMatchNodes(const ExtensionsCallback& extensionsCallback, + MatchExpressionParser::AllowedFeatureSet allowedFeatures) { + return extensionsCallback.hasNoopExtensions() && + (allowedFeatures & MatchExpressionParser::AllowedFeatures::kText || + allowedFeatures & MatchExpressionParser::AllowedFeatures::kJavascript); +} + +} // namespace + +std::unique_ptr<CollatorInterface> resolveCollator( + OperationContext* opCtx, const std::unique_ptr<FindCommandRequest>& findCommand) { + if (!findCommand->getCollation().isEmpty()) { + return uassertStatusOKWithContext(CollatorFactoryInterface::get(opCtx->getServiceContext()) + ->makeFromBSON(findCommand->getCollation()), + "unable to parse collation"); + } + return nullptr; +} + +/** + * Helper for building 'out.' If there is a projection, parse it and add any metadata dependencies + * it induces. + * + * Throws exceptions if there is an error parsing the projection. + */ +void setProjection(ParsedFindCommand* out, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const std::unique_ptr<FindCommandRequest>& findCommand, + const ProjectionPolicies& policies) { + if (!findCommand->getProjection().isEmpty()) { + out->savedProjectionPolicies.emplace(policies); + out->proj.emplace(projection_ast::parseAndAnalyze(expCtx, + findCommand->getProjection(), + out->filter.get(), + findCommand->getFilter(), + policies)); + + // This will throw if any of the projection's dependencies are unavailable. + DepsTracker{out->unavailableMetadata}.requestMetadata(out->proj->metadataDeps()); + } +} + +/** + * Helper for building 'out.' If there is a sort, parse it and add any metadata dependencies it + * induces. + * + * Throws exceptions if there is an error parsing the sort pattern. + */ +void setSort(ParsedFindCommand* out, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const std::unique_ptr<FindCommandRequest>& findCommand) { + if (!findCommand->getSort().isEmpty()) { + // A $natural sort is really a hint, and should be handled as such. Furthermore, the + // downstream sort handling code may not expect a $natural sort. + // + // We have already validated that if there is a $natural sort and a hint, that the hint + // also specifies $natural with the same direction. Therefore, it is safe to clear the + // $natural sort and rewrite it as a $natural hint. + if (findCommand->getSort()[query_request_helper::kNaturalSortField]) { + findCommand->setHint(findCommand->getSort().getOwned()); + findCommand->setSort(BSONObj{}); + } + if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) { + out->sort = cst::parseToSortPattern(findCommand->getSort(), expCtx); + } else { + out->sort.emplace(findCommand->getSort(), expCtx); + } + } +} + +/** + * Helper for building 'out.' If there is a sort, parse it and add any metadata dependencies it + * induces. + */ +Status setSortAndProjection(ParsedFindCommand* out, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const std::unique_ptr<FindCommandRequest>& findCommand, + const ProjectionPolicies& policies) { + try { + setProjection(out, expCtx, findCommand, policies); + setSort(out, expCtx, findCommand); + } catch (const DBException& ex) { + return ex.toStatus(); + } + + return Status::OK(); +} + +/** + * Helper for building 'out.' Sets 'out->filter' and validates that it is well formed. In the + * process, also populates 'out->unavailableMetadata.' + */ +Status setFilter(ParsedFindCommand* out, + std::unique_ptr<MatchExpression> filter, + const std::unique_ptr<FindCommandRequest>& findCommand) { + // Verify the filter follows certain rules like there must be at most one text clause. + auto swMeta = parsed_find_command::isValid(filter.get(), *findCommand); + if (!swMeta.isOK()) { + return swMeta.getStatus(); + } + out->unavailableMetadata = swMeta.getValue(); + out->filter = std::move(filter); + return Status::OK(); +} + + +StatusWith<std::unique_ptr<ParsedFindCommand>> parseWithValidatedCollator( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + std::unique_ptr<FindCommandRequest> findCommand, + const ExtensionsCallback& extensionsCallback, + MatchExpressionParser::AllowedFeatureSet allowedFeatures, + const ProjectionPolicies& projectionPolicies) { + auto out = std::make_unique<ParsedFindCommand>(); + + tassert(5746107, + "ntoreturn should not be set on the findCommand", + findCommand->getNtoreturn() == boost::none); + + if (auto status = query_request_helper::validateFindCommandRequest(*findCommand); + !status.isOK()) { + return status; + } + + // Parse the MatchExpression. + StatusWithMatchExpression statusWithMatcher = [&]() -> StatusWithMatchExpression { + if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) { + try { + return cst::parseToMatchExpression( + findCommand->getFilter(), expCtx, extensionsCallback); + } catch (const DBException& ex) { + return ex.toStatus(); + } + } else { + return MatchExpressionParser::parse( + findCommand->getFilter(), expCtx, extensionsCallback, allowedFeatures); + } + }(); + if (!statusWithMatcher.isOK()) { + return statusWithMatcher.getStatus(); + } + + // Stop counting expressions after they have been parsed to exclude expressions created + // during optimization and other processing steps. + expCtx->stopExpressionCounters(); + out->canHaveNoopMatchNodes = + parsingCanProduceNoopMatchNodes(extensionsCallback, allowedFeatures); + + if (auto status = setFilter(out.get(), std::move(statusWithMatcher.getValue()), findCommand); + !status.isOK()) { + return status; + } + + if (auto status = setSortAndProjection(out.get(), expCtx, findCommand, projectionPolicies); + !status.isOK()) { + return status; + } + + out->findCommandRequest = std::move(findCommand); + return {std::move(out)}; +} + +StatusWith<std::unique_ptr<ParsedFindCommand>> ParsedFindCommand::withExistingFilter( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + std::unique_ptr<CollatorInterface> collator, + std::unique_ptr<MatchExpression> filter, + std::unique_ptr<FindCommandRequest> findCommandRequest) { + auto out = std::make_unique<ParsedFindCommand>(); + out->collator = std::move(collator); + if (auto status = setFilter(out.get(), std::move(filter), findCommandRequest); !status.isOK()) { + return status; + } + if (auto status = setSortAndProjection( + out.get(), expCtx, findCommandRequest, ProjectionPolicies::findProjectionPolicies()); + !status.isOK()) { + return status; + } + out->findCommandRequest = std::move(findCommandRequest); + return std::move(out); +} + +namespace parsed_find_command { +StatusWith<QueryMetadataBitSet> isValid(const MatchExpression* root, + const FindCommandRequest& findCommand) { + QueryMetadataBitSet unavailableMetadata{}; + + // There can only be one TEXT. If there is a TEXT, it cannot appear inside a NOR. + // + // Note that the query grammar (as enforced by the MatchExpression parser) forbids TEXT + // inside of value-expression clauses like NOT, so we don't check those here. + size_t numText = QueryPlannerCommon::countNodes(root, MatchExpression::TEXT); + if (numText > 1) { + return Status(ErrorCodes::BadValue, "Too many text expressions"); + } else if (1 == numText) { + if (hasNodeInSubtree(root, MatchExpression::TEXT, MatchExpression::NOR)) { + return Status(ErrorCodes::BadValue, "text expression not allowed in nor"); + } + } else { + // Text metadata is not available. + unavailableMetadata.set(DocumentMetadataFields::kTextScore); + } + + // There can only be one NEAR. If there is a NEAR, it must be either the root or the root + // must be an AND and its child must be a NEAR. + size_t numGeoNear = QueryPlannerCommon::countNodes(root, MatchExpression::GEO_NEAR); + if (numGeoNear > 1) { + return Status(ErrorCodes::BadValue, "Too many geoNear expressions"); + } else if (1 == numGeoNear) { + // Do nothing, we will perform extra checks in CanonicalQuery::isValidNormalized. + } else { + // Geo distance and geo point metadata are unavailable. + unavailableMetadata |= DepsTracker::kAllGeoNearData; + } + + const BSONObj& sortObj = findCommand.getSort(); + BSONElement sortNaturalElt = sortObj["$natural"]; + const BSONObj& hintObj = findCommand.getHint(); + BSONElement hintNaturalElt = hintObj["$natural"]; + + if (sortNaturalElt && sortObj.nFields() != 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Cannot include '$natural' in compound sort: " << sortObj); + } + + if (hintNaturalElt && hintObj.nFields() != 1) { + return Status(ErrorCodes::BadValue, + str::stream() << "Cannot include '$natural' in compound hint: " << hintObj); + } + + // NEAR cannot have a $natural sort or $natural hint. + if (numGeoNear > 0) { + if (sortNaturalElt) { + return Status(ErrorCodes::BadValue, + "geoNear expression not allowed with $natural sort order"); + } + + if (hintNaturalElt) { + return Status(ErrorCodes::BadValue, + "geoNear expression not allowed with $natural hint"); + } + } + + // TEXT and NEAR cannot both be in the query. + if (numText > 0 && numGeoNear > 0) { + return Status(ErrorCodes::BadValue, "text and geoNear not allowed in same query"); + } + + // TEXT and {$natural: ...} sort order cannot both be in the query. + if (numText > 0 && sortNaturalElt) { + return Status(ErrorCodes::BadValue, "text expression not allowed with $natural sort order"); + } + + // TEXT and hint cannot both be in the query. + if (numText > 0 && !hintObj.isEmpty()) { + return Status(ErrorCodes::BadValue, "text and hint not allowed in same query"); + } + + // TEXT and tailable are incompatible. + if (numText > 0 && findCommand.getTailable()) { + return Status(ErrorCodes::BadValue, "text and tailable cursor not allowed in same query"); + } + + // NEAR and tailable are incompatible. + if (numGeoNear > 0 && findCommand.getTailable()) { + return Status(ErrorCodes::BadValue, + "Tailable cursors and geo $near cannot be used together"); + } + + // $natural sort order must agree with hint. + if (sortNaturalElt) { + if (!hintObj.isEmpty() && !hintNaturalElt) { + return Status(ErrorCodes::BadValue, "index hint not allowed with $natural sort order"); + } + if (hintNaturalElt) { + if (hintNaturalElt.numberInt() != sortNaturalElt.numberInt()) { + return Status(ErrorCodes::BadValue, + "$natural hint must be in the same direction as $natural sort order"); + } + } + } + + return unavailableMetadata; +} + +StatusWith<std::pair<boost::intrusive_ptr<ExpressionContext>, std::unique_ptr<ParsedFindCommand>>> +parse(OperationContext* opCtx, + std::unique_ptr<FindCommandRequest> findCommand, + const ExtensionsCallback& extensionsCallback, + MatchExpressionParser::AllowedFeatureSet allowedFeatures, + const ProjectionPolicies& projectionPolicies) { + // Make the expCtx. + invariant(findCommand->getNamespaceOrUUID().nss().has_value()); + auto expCtx = make_intrusive<ExpressionContext>( + opCtx, *findCommand, resolveCollator(opCtx, findCommand), true /* mayDbProfile */); + auto swResult = parseWithValidatedCollator( + expCtx, std::move(findCommand), extensionsCallback, allowedFeatures, projectionPolicies); + if (!swResult.isOK()) { + return swResult.getStatus(); + } + + return std::pair{std::move(expCtx), std::move(swResult.getValue())}; +} + +StatusWith<std::unique_ptr<ParsedFindCommand>> parse( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + std::unique_ptr<FindCommandRequest> findCommand, + const ExtensionsCallback& extensionsCallback, + MatchExpressionParser::AllowedFeatureSet allowedFeatures, + const ProjectionPolicies& projectionPolicies) { + // A collator can enter through both the FindCommandRequest and ExpressionContext arguments. + // This invariant ensures that both collators are the same because downstream we + // pull the collator from only one of the ExpressionContext carrier. + auto collator = resolveCollator(expCtx->opCtx, findCommand); + if (collator.get() && expCtx->getCollator()) { + invariant(CollatorInterface::collatorsMatch(collator.get(), expCtx->getCollator())); + } + return parseWithValidatedCollator( + expCtx, std::move(findCommand), extensionsCallback, allowedFeatures, projectionPolicies); +} +} // namespace parsed_find_command +} // namespace mongo diff --git a/src/mongo/db/query/parsed_find_command.h b/src/mongo/db/query/parsed_find_command.h new file mode 100644 index 00000000000..f3ec78b9204 --- /dev/null +++ b/src/mongo/db/query/parsed_find_command.h @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/matcher/expression.h" +#include "mongo/db/query/find_command_gen.h" +#include "mongo/db/query/projection.h" +#include "mongo/db/query/projection_policies.h" +#include "mongo/db/query/sort_pattern.h" + +namespace mongo { + +/** + * Represents a find command request, but with more fully parsed ASTs for some fields which are + * still raw BSONObj on the FindCommandRequest type. + */ +struct ParsedFindCommand { + ParsedFindCommand() = default; + + /** + * This API adds the ability to construct from a pre-parsed filter. The other arguments will be + * re-parsed again from BSON on the 'findCommandRequest' argument, since we don't have a good + * way of cloning them. + */ + static StatusWith<std::unique_ptr<ParsedFindCommand>> withExistingFilter( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + std::unique_ptr<CollatorInterface> collator, + std::unique_ptr<MatchExpression> filter, + std::unique_ptr<FindCommandRequest> findCommandRequest); + + std::unique_ptr<CollatorInterface> collator; + std::unique_ptr<MatchExpression> filter; + boost::optional<projection_ast::Projection> proj; + boost::optional<SortPattern> sort; + + // Based on parsing the query, which metadata will *not* be available. For example, if there is + // no $text clause, then a text score will not be available. + QueryMetadataBitSet unavailableMetadata; + + // This is saved for an edge case where we need to re-parse a projection later. Only populated + // if there is a non-empty projection. + boost::optional<ProjectionPolicies> savedProjectionPolicies; + + // True if this canonical query may have converted extensions such as $where and $text into + // no-ops during parsing. This will be the case if it allowed $where and $text in parsing, but + // parsed using an ExtensionsCallbackNoop. This does not guarantee that a $where or $text + // existed in the query. Queries with a no-op extension context are special because they can be + // parsed and planned, but they cannot be executed. + bool canHaveNoopMatchNodes; + + // All other parameters to the find command which do not have AST-like types and can be + // appropriately tracked as raw value types like ints. The fields above like 'filter' are all + // still present in their raw form on this FidnCommandRequest, but it is not expected that they + // will be useful other than to keep the original BSON values around in-memory to avoid copying + // large strings and such. + std::unique_ptr<FindCommandRequest> findCommandRequest; +}; + +namespace parsed_find_command { +/** + * Validates the match expression 'root' as well as the query specified by 'request', checking for + * illegal combinations of operators. Returns a non-OK status if any such illegal combination is + * found. + * + * This method can be called both on normalized and non-normalized 'root'. However, some checks can + * only be performed once the match expressions is normalized. To perform these checks one can call + * 'CanonicalQuery::isValidNormalized()'. + * + * On success, returns a bitset indicating which types of metadata are *unavailable*. For example, + * if 'root' does not contain a $text predicate, then the returned metadata bitset will indicate + * that text score metadata is unavailable. This means that if subsequent $meta:"textScore" + * expressions are found during analysis of the query, we should raise in an error. + */ +StatusWith<QueryMetadataBitSet> isValid(const MatchExpression* root, + const FindCommandRequest& findCommand); + +/** + * Parses each big component of the input 'findCommand.' Throws exceptions if failing to parse. + * Comes in one overload which will create an ExpressionContext for the caller, and one overload to + * be used when the caller already has an ExpressionContext. + */ +StatusWith<std::pair<boost::intrusive_ptr<ExpressionContext>, std::unique_ptr<ParsedFindCommand>>> +parse(OperationContext* opCtx, + std::unique_ptr<FindCommandRequest> findCommand, + const ExtensionsCallback& extensionsCallback = ExtensionsCallbackNoop(), + MatchExpressionParser::AllowedFeatureSet allowedFeatures = + MatchExpressionParser::kDefaultSpecialFeatures, + const ProjectionPolicies& projectionPolicies = ProjectionPolicies::findProjectionPolicies()); + +// Overload of the above for when the caller has an available ExpressionContext. +StatusWith<std::unique_ptr<ParsedFindCommand>> parse( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + std::unique_ptr<FindCommandRequest> findCommand, + const ExtensionsCallback& extensionsCallback = ExtensionsCallbackNoop(), + MatchExpressionParser::AllowedFeatureSet allowedFeatures = + MatchExpressionParser::kDefaultSpecialFeatures, + const ProjectionPolicies& projectionPolicies = ProjectionPolicies::findProjectionPolicies()); + +} // namespace parsed_find_command +} // namespace mongo diff --git a/src/mongo/db/query/partitioned_cache.h b/src/mongo/db/query/partitioned_cache.h new file mode 100644 index 00000000000..ca10f731b71 --- /dev/null +++ b/src/mongo/db/query/partitioned_cache.h @@ -0,0 +1,242 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/catalog/util/partitioned.h" +#include "mongo/db/commands/server_status_metric.h" +#include "mongo/db/query/lru_key_value.h" +#include "mongo/platform/mutex.h" +#include "mongo/util/container_size_helper.h" + +namespace mongo { + +/** + * A partitioned cache combines a size-bounded map (LRU-based entry eviction) with a partition + * function which allows reducing contention. + */ +template <class KeyType, + class ValueType, + class KeyBudgetEstimator, + class Partitioner, + class InsertionEvictionListener, + class KeyHasher = std::hash<KeyType>, + class Eq = std::equal_to<KeyType>> +class PartitionedCache { +private: + PartitionedCache(const PartitionedCache&) = delete; + PartitionedCache& operator=(const PartitionedCache&) = delete; + +public: + using Lru = LRUKeyValue<KeyType, + ValueType, + KeyBudgetEstimator, + InsertionEvictionListener, + KeyHasher, + Eq>; + using Partition = typename Partitioned<Lru, Partitioner>::OnePartition; + using PartitionId = typename Partitioned<Lru, Partitioner>::PartitionId; + + /** + * Initialize plan cache with the total cache size in bytes and number of partitions. + * + * Important edge cases to consider include: + * + * 1. Adding an entry that is larger than the max partition size to a non-empty partition. + * + * This will evict both entries. This is because entries are evicted from the partition in + * order of least recently used. Thus, the oldest, small entry will be evicted first but the + * partition will still be over budget with the new, too-large entry so it will be evicted as + * well. + * + * 2. Adding a queryStats store entry that is smaller than the overall cache size but larger + * than single partition max size. + * + * It is not possible to write entries to the cache that are larger than a single + * partition's max size, even if it is smaller than the entire cache max size. This is because + * the cache's budget is configured/regulated on the partition level (cacheSize / + * numPartitions). This makes sense as each entry is written to a specific partition, but might + * not be immediately obvious so worthy to highlight. + * + * 3. Too few partitions can cause unnecessary evictions + * + * Every class that implements the PartitionedCache template provides a partitioner() that + * returns the id of the partition to which to write the entry. In existing implementations, + * partitioner() returns the remainder after dividing the entry's key hash by numPartitions. In + * the case where we have only two partitions, every odd key hash will be written to the first + * partition (and vice versa). In this way, it can quickly be the case that one partition + * fills up completely but the partitioner() call keeps returning the already full partition and + * the cache evict old entries from it to put the new one in. At the end of all the write + * operations, the cache is below it's budget (as the second partition is only partially full) + * but we don't have all the entries we expect. It is therefore important to have sufficient + * enough number of partitions so the entries can be more equally dispersed to avoid unnecessary + * evictions. + */ + explicit PartitionedCache(size_t cacheSize, size_t numPartitions) + : _numPartitions(numPartitions) { + invariant(numPartitions > 0); + Lru lru{cacheSize / numPartitions}; + _partitionedCache = + std::make_unique<Partitioned<Lru, Partitioner>>(numPartitions, std::move(lru)); + } + + ~PartitionedCache() = default; + /** + * Inserts the provided <key, value> into the partition associated with that key. Returns the + * number of older entries evicted to fit this new one. + */ + size_t put(const KeyType& key, ValueType value) { + auto partition = _partitionedCache->lockOnePartition(key); + return partition->add(key, std::move(value)); + } + /** + * Inserts the provided <key, value> into the specified partition. Returns the number of older + * entries evicted to fit this new one. + */ + size_t put(const KeyType& key, ValueType value, Partition& partition) { + return partition->add(key, std::move(value)); + } + + StatusWith<ValueType*> lookup(const KeyType& key) const { + auto partition = _partitionedCache->lockOnePartition(key); + auto entry = partition->get(key); + if (!entry.isOK()) { + return {entry.getStatus()}; + } + + return {&entry.getValue()->second}; + } + + /** + * Lookup an entry and also return a lock over the partition. The lock is returned whether + * or not the entry is found. + */ + std::pair<StatusWith<ValueType*>, Partition> getWithPartitionLock(const KeyType& key) const { + auto partition = _partitionedCache->lockOnePartition(key); + auto entry = partition->get(key); + if (!entry.isOK()) { + return std::make_pair(entry.getStatus(), std::move(partition)); + } + + return std::make_pair(StatusWith{&entry.getValue()->second}, std::move(partition)); + } + + /** + * Remove the entry with the 'key' from the cache. If there is no entry for the given key in + * the cache, this call is a no-op. + */ + void remove(const KeyType& key) { + _partitionedCache->erase(key); + } + + /** + * Remove all the entries for keys for which the predicate returns true. Return the number of + * removed entries. + */ + template <typename UnaryPredicate> + size_t removeIf(UnaryPredicate predicate) { + size_t nRemoved = 0; + for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) { + auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId); + nRemoved += lockedPartition->removeIf(predicate); + } + return nRemoved; + } + + /** + * Remove *all* cache entries. + */ + void clear() { + _partitionedCache->clear(); + } + + /** + * Reset total cache size. If the size is set to a smaller value than before, enough entries are + * evicted in order to ensure that the cache fits within the new budget. Returns the number of + * entries evicted. + */ + size_t reset(size_t cacheSize) { + size_t numEvicted = 0; + for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) { + auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId); + numEvicted += lockedPartition->reset(cacheSize / _numPartitions); + } + + return numEvicted; + } + + /** + * Returns the size of the cache. + * Used for testing. + */ + size_t size() const { + return _partitionedCache->size(); + } + + /** + * Returns the number of partitions. + */ + size_t numPartitions() const { + return _numPartitions; + } + + /** + * Invoke `op` for each entry in the cache. Consistency across partitions is not guaranteed. + */ + void forEach(const std::function<void(const KeyType&, const ValueType&)>& op) const { + for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) { + auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId); + + for (auto&& [key, entry] : *lockedPartition) { + op(*key, entry); + } + } + } + + /** + * Allow iterating over partitions. The provided function is called for each partition. The + * argument to the function is another function which can delay acquiring the implicitly locked + * partition until it's needed. + */ + void forEachPartition(const std::function<void(const std::function<Partition()>&)>& op) const { + for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) { + op([&]() { return _partitionedCache->lockOnePartitionById(partitionId); }); + } + } + + Partition getPartition(PartitionId partitionId) { + return _partitionedCache->lockOnePartitionById(partitionId); + } + +private: + std::size_t _numPartitions; + std::unique_ptr<Partitioned<Lru, Partitioner>> _partitionedCache; +}; + +} // namespace mongo diff --git a/src/mongo/db/query/plan_cache.h b/src/mongo/db/query/plan_cache.h index dc52b10c4f4..e193309230e 100644 --- a/src/mongo/db/query/plan_cache.h +++ b/src/mongo/db/query/plan_cache.h @@ -31,6 +31,7 @@ #include "mongo/db/catalog/util/partitioned.h" #include "mongo/db/query/lru_key_value.h" +#include "mongo/db/query/partitioned_cache.h" #include "mongo/db/query/plan_cache_callbacks.h" #include "mongo/db/query/plan_cache_debug_info.h" #include "mongo/platform/mutex.h" @@ -281,21 +282,34 @@ private: */ template <class KeyType, class CachedPlanType, - class BudgetEstimator, + class KeyBudgetEstimator, class DebugInfoType, class Partitioner, class KeyHasher = std::hash<KeyType>> -class PlanCacheBase { +class PlanCacheBase + : public PartitionedCache< + KeyType, + // The 'Value' being "std::shared_ptr<const Entry>" is because we allow readers to clone + // cache entries out of the lock, therefore it is illegal to mutate the pieces of a cache + // entry that can be cloned whether you are holding a lock or not. + std::shared_ptr<const PlanCacheEntryBase<CachedPlanType, DebugInfoType>>, + KeyBudgetEstimator, + Partitioner, + NoopInsertionEvictionListener, + KeyHasher> { private: PlanCacheBase(const PlanCacheBase&) = delete; PlanCacheBase& operator=(const PlanCacheBase&) = delete; public: + using Base = + PartitionedCache<KeyType, + std::shared_ptr<const PlanCacheEntryBase<CachedPlanType, DebugInfoType>>, + KeyBudgetEstimator, + Partitioner, + NoopInsertionEvictionListener, + KeyHasher>; using Entry = PlanCacheEntryBase<CachedPlanType, DebugInfoType>; - // The 'Value' being "std::shared_ptr<const Entry>" is because we allow readers to clone cache - // entries out of the lock, therefore it is illegal to mutate the pieces of a cache entry that - // can be cloned whether you are holding a lock or not. - using Lru = LRUKeyValue<KeyType, std::shared_ptr<const Entry>, BudgetEstimator, KeyHasher>; // We have three states for a cache entry to be in. Rather than just 'present' or 'not // present', we use a notion of 'inactive entries' as a way of remembering how performant our @@ -328,11 +342,7 @@ public: * Initialize plan cache with the total cache size in bytes and number of partitions. */ explicit PlanCacheBase(size_t cacheSize, size_t numPartitions = 1) - : _numPartitions(numPartitions) { - invariant(numPartitions > 0); - Lru lru{cacheSize / numPartitions}; - _partitionedCache = std::make_unique<Partitioned<Lru, Partitioner>>(numPartitions, lru); - } + : Base(cacheSize, numPartitions) {} ~PlanCacheBase() = default; @@ -375,7 +385,11 @@ public: }}, why.stats); - auto partition = _partitionedCache->lockOnePartition(key); + auto oldEntryWithPartitionLock = this->getWithPartitionLock(key); + // Can't use reference to structured bindings in a lambda until C++20 so manually + // destructure it here. + auto partitionLock = std::move(oldEntryWithPartitionLock.second); + auto oldEntryWithStatus = std::move(oldEntryWithPartitionLock.first); auto [queryHash, planCacheKey, isNewEntryActive, shouldBeCreated, increasedWorks] = [&]() { if (internalQueryCacheDisableInactiveEntries.load()) { // All entries are always active. @@ -385,32 +399,34 @@ public: true /* shouldBeCreated */, boost::optional<size_t>(boost::none)); } else { - auto oldEntryWithStatus = partition->get(key); tassert(6007020, "LRU store must get value or NoSuchKey error code", oldEntryWithStatus.isOK() || oldEntryWithStatus.getStatus() == ErrorCodes::NoSuchKey); - auto oldEntry = - oldEntryWithStatus.isOK() ? oldEntryWithStatus.getValue()->second : nullptr; + bool hasOldEntry = oldEntryWithStatus.isOK(); const auto newState = getNewEntryState( key, - oldEntry.get(), + // Deference the pointer, then the shared_ptr, and then back to a raw pointer. + hasOldEntry ? &**oldEntryWithStatus.getValue() : nullptr, newWorks, worksGrowthCoefficient.get_value_or(internalQueryCacheWorksGrowthCoefficient), callbacks); // Avoid recomputing the hashes if we've got an old entry to grab them from. - return oldEntry ? std::make_tuple(oldEntry->queryHash, - oldEntry->planCacheKey, - newState.shouldBeActive, - newState.shouldBeCreated, - newState.increasedWorks) - : std::make_tuple(key.queryHash(), - key.planCacheKeyHash(), - newState.shouldBeActive, - newState.shouldBeCreated, - newState.increasedWorks); + auto [queryHash, planCacheKey] = [&]() { + if (hasOldEntry) { + auto&& oldEntry = &**oldEntryWithStatus.getValue(); + return std::make_pair(oldEntry->queryHash, oldEntry->planCacheKey); + } else { + return std::make_pair(key.queryHash(), key.planCacheKeyHash()); + } + }(); + return std::make_tuple(queryHash, + planCacheKey, + newState.shouldBeActive, + newState.shouldBeCreated, + newState.increasedWorks); } }(); @@ -434,7 +450,7 @@ public: increasedWorks ? *increasedWorks : newWorks, callbacks->buildDebugInfo()); - partition->add(key, std::move(newEntry)); + this->put(key, std::move(newEntry), partitionLock); return Status::OK(); } @@ -454,10 +470,7 @@ public: indexFilterKey, now, std::move(debugInfo)); - auto partition = _partitionedCache->lockOnePartition(key); - // We're not interested in the number of evicted entries if the cache store exceeds the - // budget after add(), so we just ignore the return value. - partition->add(key, std::move(entry)); + this->put(key, std::move(entry)); } /** @@ -471,8 +484,8 @@ public: return; } - auto partition = _partitionedCache->lockOnePartition(key); - auto entry = partition->get(key); + auto [entry, partitionLock] = this->getWithPartitionLock(key); + if (!entry.isOK()) { tassert(6007021, "Unexpected error code from LRU store", @@ -480,11 +493,11 @@ public: return; } - auto entryPtr = entry.getValue()->second; + auto entryPtr = *entry.getValue(); if (entryPtr->isActive == true) { std::shared_ptr<Entry> newEntry = entryPtr->clone(); newEntry->isActive = false; - partition->add(key, std::move(newEntry)); + this->put(key, std::move(newEntry), partitionLock); } } @@ -496,29 +509,26 @@ public: * for the query (if there is one). */ GetResult get(const KeyType& key) const { - std::shared_ptr<const Entry> entryPtr; + std::shared_ptr<const Entry> entrySharedPtr; CacheEntryState state; { - auto partition = _partitionedCache->lockOnePartition(key); - auto entry = partition->get(key); + auto [entry, partitionLock] = this->getWithPartitionLock(key); if (!entry.isOK()) { tassert(6007023, "Unexpected error code from LRU store", entry.getStatus() == ErrorCodes::NoSuchKey); return {CacheEntryState::kNotPresent, nullptr}; } - entryPtr = entry.getValue()->second; - state = entryPtr->isActive ? CacheEntryState::kPresentActive - : CacheEntryState::kPresentInactive; + entrySharedPtr = *entry.getValue(); + state = entrySharedPtr->isActive ? CacheEntryState::kPresentActive + : CacheEntryState::kPresentInactive; } - // The purpose of cloning 'entry' after we release the lock is to allow multiple threads to - // clone the same plan cache entry at once. 'entry' cannot be deleted by another thread even - // if the plan cache is being concurrently modified by other threads because we are holding - // a std::shared_ptr to this entry. - tassert(6007024, "LRU store must get a value or an error code", entryPtr); - + // The purpose of cloning 'entry' (in CachedPlanHolder ctor) after we release the lock + // is to allow multiple threads to clone the same plan cache entry at once. 'entry' + // cannot be deleted by another thread even if the plan cache is being concurrently + // modified by other threads because we are holding a std::shared_ptr to this entry. return {state, - std::make_unique<CachedPlanHolder<CachedPlanType, DebugInfoType>>(*entryPtr)}; + std::make_unique<CachedPlanHolder<CachedPlanType, DebugInfoType>>(*entrySharedPtr)}; } /** @@ -537,59 +547,16 @@ public: } /** - * Remove the entry with the 'key' from the cache. If there is no entry for the given key in - * the cache, this call is a no-op. - */ - void remove(const KeyType& key) { - _partitionedCache->erase(key); - } - - /** - * Remove all the entries for keys for which the predicate returns true. Return the number of - * removed entries. - */ - template <typename UnaryPredicate> - size_t removeIf(UnaryPredicate predicate) { - size_t nRemoved = 0; - for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) { - auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId); - nRemoved += lockedPartition->removeIf(predicate); - } - return nRemoved; - } - - /** - * Remove *all* cached plans. Does not clear index information. - */ - void clear() { - _partitionedCache->clear(); - } - - /** - * Reset total cache size. If the size is set to a smaller value than before, enough entries are - * evicted in order to ensure that the cache fits within the new budget. - */ - void reset(size_t cacheSize) { - for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) { - auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId); - lockedPartition->reset(cacheSize / _numPartitions); - } - } - - /** * Returns a copy of a cache entry, looked up by the plan cache key. * * If there is no entry in the cache for the 'query', returns an error Status. */ StatusWith<std::unique_ptr<Entry>> getEntry(const KeyType& key) const { - auto partition = _partitionedCache->lockOnePartition(key); - auto entry = partition->get(key); - if (!entry.isOK()) { - return entry.getStatus(); + auto result = this->lookup(key); + if (!result.isOK()) { + return {result.getStatus()}; } - invariant(entry.getValue()->second); - - return std::unique_ptr<Entry>(entry.getValue()->second->clone()); + return {result.getValue()->get()->clone()}; } /** @@ -598,26 +565,14 @@ public: std::vector<std::unique_ptr<Entry>> getAllEntries() const { std::vector<std::unique_ptr<Entry>> entries; - for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) { - auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId); - - for (auto&& [key, entry] : *lockedPartition) { - entries.emplace_back(entry->clone()); - } - } + this->forEach([&](const KeyType& key, const std::shared_ptr<Entry>& entry) { + entries.emplace_back(entry); + }); return entries; } /** - * Returns the size of the cache. - * Used for testing. - */ - size_t size() const { - return _partitionedCache->size(); - } - - /** * Iterates over the plan cache. For each entry, first filters according to the predicate * function 'cacheKeyFilterFunc', (Note that 'cacheKeyFilterFunc' could be empty, if so, we * don't filter by plan cache key.), then serializes the PlanCacheEntryBase according to @@ -634,20 +589,15 @@ public: std::vector<BSONObj> results; - for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) { - auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId); - - for (auto&& cacheEntry : *lockedPartition) { - if (cacheKeyFilterFunc && !cacheKeyFilterFunc(cacheEntry.first)) { - continue; - } - const auto& entry = cacheEntry.second; - auto serializedEntry = serializationFunc(*entry); - if (filterFunc(serializedEntry)) { - results.push_back(serializedEntry); - } + this->forEach([&](const KeyType& key, const std::shared_ptr<const Entry>& entry) { + if (cacheKeyFilterFunc && !cacheKeyFilterFunc(key)) { + return; } - } + auto serializedEntry = serializationFunc(*entry); + if (filterFunc(serializedEntry)) { + results.push_back(serializedEntry); + } + }); return results; } @@ -735,9 +685,6 @@ private: return res; } - - std::size_t _numPartitions; - std::unique_ptr<Partitioned<Lru, Partitioner>> _partitionedCache; }; } // namespace mongo diff --git a/src/mongo/db/query/plan_cache_size_parameter_test.cpp b/src/mongo/db/query/plan_cache_size_parameter_test.cpp deleted file mode 100644 index 1f0fbf76a27..00000000000 --- a/src/mongo/db/query/plan_cache_size_parameter_test.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright (C) 2021-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/query/plan_cache_size_parameter.h" - -#include "mongo/unittest/unittest.h" - -namespace mongo::plan_cache_util { - -bool operator==(const PlanCacheSizeParameter& lhs, const PlanCacheSizeParameter& rhs) { - constexpr double kEpsilon = 1e-10; - return std::abs(lhs.size - rhs.size) < kEpsilon && lhs.units == rhs.units; -} - -TEST(PlanCacheParameterTest, ParseUnitStringPercent) { - ASSERT_TRUE(PlanCacheSizeUnits::kPercent == parseUnitString("%")); -} - -TEST(PlanCacheParameterTest, ParseUnitStringMB) { - ASSERT_TRUE(PlanCacheSizeUnits::kMB == parseUnitString("MB")); - ASSERT_TRUE(PlanCacheSizeUnits::kMB == parseUnitString("mb")); - ASSERT_TRUE(PlanCacheSizeUnits::kMB == parseUnitString("mB")); - ASSERT_TRUE(PlanCacheSizeUnits::kMB == parseUnitString("Mb")); -} - -TEST(PlanCacheParameterTest, ParseUnitStringGB) { - ASSERT_TRUE(PlanCacheSizeUnits::kGB == parseUnitString("GB")); - ASSERT_TRUE(PlanCacheSizeUnits::kGB == parseUnitString("gb")); - ASSERT_TRUE(PlanCacheSizeUnits::kGB == parseUnitString("gB")); - ASSERT_TRUE(PlanCacheSizeUnits::kGB == parseUnitString("Gb")); -} - -TEST(PlanCacheParameterTest, ParseUnitStringIncorrectValue) { - ASSERT_NOT_OK(parseUnitString("").getStatus()); - ASSERT_NOT_OK(parseUnitString(" ").getStatus()); - ASSERT_NOT_OK(parseUnitString("KB").getStatus()); -} - -TEST(PlanCacheParameterTest, ParsePlanCacheSizeParameter) { - ASSERT_TRUE((PlanCacheSizeParameter{10.0, PlanCacheSizeUnits::kPercent}) == - PlanCacheSizeParameter::parse("10%")); - ASSERT_TRUE((PlanCacheSizeParameter{300.0, PlanCacheSizeUnits::kMB}) == - PlanCacheSizeParameter::parse("300MB")); - ASSERT_TRUE((PlanCacheSizeParameter{4.0, PlanCacheSizeUnits::kGB}) == - PlanCacheSizeParameter::parse("4GB")); - ASSERT_TRUE((PlanCacheSizeParameter{5.1, PlanCacheSizeUnits::kPercent}) == - PlanCacheSizeParameter::parse(" 5.1%")); - ASSERT_TRUE((PlanCacheSizeParameter{11.1, PlanCacheSizeUnits::kMB}) == - PlanCacheSizeParameter::parse("11.1 mb")); - ASSERT_TRUE((PlanCacheSizeParameter{12.1, PlanCacheSizeUnits::kGB}) == - PlanCacheSizeParameter::parse(" 12.1 Gb ")); -} -} // namespace mongo::plan_cache_util diff --git a/src/mongo/db/query/plan_enumerator.cpp b/src/mongo/db/query/plan_enumerator.cpp index 4f15905dc6d..9ad6f6b81aa 100644 --- a/src/mongo/db/query/plan_enumerator.cpp +++ b/src/mongo/db/query/plan_enumerator.cpp @@ -35,6 +35,7 @@ #include "mongo/db/query/index_tag.h" #include "mongo/db/query/indexability.h" +#include "mongo/db/query/query_planner_common.h" #include "mongo/logv2/log.h" #include "mongo/util/string_map.h" @@ -58,8 +59,8 @@ std::string getPathPrefix(std::string path) { * is a predicate that is required to use an index. */ bool expressionRequiresIndex(const MatchExpression* node) { - return CanonicalQuery::countNodes(node, MatchExpression::GEO_NEAR) > 0 || - CanonicalQuery::countNodes(node, MatchExpression::TEXT) > 0; + return QueryPlannerCommon::countNodes(node, MatchExpression::GEO_NEAR) > 0 || + QueryPlannerCommon::countNodes(node, MatchExpression::TEXT) > 0; } size_t getPathLength(const MatchExpression* expr) { @@ -261,7 +262,8 @@ PlanEnumerator::PlanEnumerator(const PlanEnumeratorParams& params) _ixisect(params.intersect), _enumerateOrChildrenLockstep(params.enumerateOrChildrenLockstep), _orLimit(params.maxSolutionsPerOr), - _intersectLimit(params.maxIntersectPerAnd) {} + _intersectLimit(params.maxIntersectPerAnd), + _disableOrPushdown(params.disableOrPushdown) {} PlanEnumerator::~PlanEnumerator() { typedef stdx::unordered_map<MemoID, NodeAssignment*> MemoMap; @@ -528,10 +530,14 @@ bool PlanEnumerator::prepMemo(MatchExpression* node, PrepMemoContext context) { // preds to 'indexedPreds'. Adding the mandatory preds directly to 'indexedPreds' would lead // to problems such as pulling a predicate beneath an OR into a set joined by an AND. getIndexedPreds(node, childContext, &indexedPreds); - // Pass in the indexed predicates as outside predicates when prepping the subnodes. + // Pass in the indexed predicates as outside predicates when prepping the subnodes. But if + // match expression optimization is disabled, skip this part: we don't want to do + // OR-pushdown because it relies on the expression being canonicalized. auto childContextCopy = childContext; - for (auto pred : indexedPreds) { - childContextCopy.outsidePreds[pred] = OutsidePredRoute{}; + if (MONGO_likely(!_disableOrPushdown)) { + for (auto pred : indexedPreds) { + childContextCopy.outsidePreds[pred] = OutsidePredRoute{}; + } } if (!prepSubNodes(node, childContextCopy, &subnodes, &mandatorySubnodes)) { return false; @@ -835,6 +841,13 @@ void PlanEnumerator::assignPredicate( MatchExpression* pred, size_t position, OneIndexAssignment* indexAssignment) { + if (MONGO_unlikely(_disableOrPushdown)) { + // If match expression optimization is disabled, we also disable OR-pushdown, + // so we should never get 'outsidePreds' here. + tassert(7059700, + "Tried to do OR-pushdown despite disableMatchExpressionOptimization", + outsidePreds.empty()); + } if (outsidePreds.find(pred) != outsidePreds.end()) { OrPushdownTag::Destination dest; dest.route = outsidePreds.at(pred).route; @@ -1289,6 +1302,8 @@ void PlanEnumerator::getIndexedPreds(MatchExpression* node, std::vector<MatchExpression*>* indexedPreds) { if (Indexability::nodeCanUseIndexOnOwnField(node)) { RelevantTag* rt = static_cast<RelevantTag*>(node->getTag()); + tassert(9074700, "RelevantTag is not assigned to the match expression node", rt != nullptr); + if (context.elemMatchExpr) { // If we're in an $elemMatch context, store the // innermost parent $elemMatch, as well as the @@ -1305,7 +1320,7 @@ void PlanEnumerator::getIndexedPreds(MatchExpression* node, indexedPreds->push_back(node); } else if (Indexability::isBoundsGeneratingNot(node)) { getIndexedPreds(node->getChild(0), context, indexedPreds); - } else if (MatchExpression::ELEM_MATCH_OBJECT == node->matchType()) { + } else if (Indexability::isBoundsGeneratingElemMatchObject(node)) { PrepMemoContext childContext; childContext.elemMatchExpr = node; for (size_t i = 0; i < node->numChildren(); ++i) { diff --git a/src/mongo/db/query/plan_enumerator.h b/src/mongo/db/query/plan_enumerator.h index b82b738c57b..9eabd5b09b9 100644 --- a/src/mongo/db/query/plan_enumerator.h +++ b/src/mongo/db/query/plan_enumerator.h @@ -44,7 +44,8 @@ namespace mongo { struct PlanEnumeratorParams { PlanEnumeratorParams() : maxSolutionsPerOr(internalQueryEnumerationMaxOrSolutions.load()), - maxIntersectPerAnd(internalQueryEnumerationMaxIntersectPerAnd.load()) {} + maxIntersectPerAnd(internalQueryEnumerationMaxIntersectPerAnd.load()), + disableOrPushdown(disableMatchExpressionOptimization.shouldFail()) {} // Do we provide solutions that use more indices than the minimum required to provide // an indexed solution? @@ -69,6 +70,11 @@ struct PlanEnumeratorParams { // all-pairs approach, we could wind up creating a lot of enumeration possibilities for // certain inputs. size_t maxIntersectPerAnd; + + // Whether to disable OR-pushdown optimization. OR-pushdown assumes that the expression has been + // simplified: for example, that single-child $or nodes are unwrapped. To avoid this, when + // the 'disableMatchExpressionOptimization' failpoint is set, we also disable OR-pushdown. + bool disableOrPushdown; }; /** @@ -594,6 +600,9 @@ private: // How many things do we want from each AND? size_t _intersectLimit; + + // Whether we should disable OR-pushdown optimization. + const bool _disableOrPushdown; }; } // namespace mongo diff --git a/src/mongo/db/query/plan_executor_factory.cpp b/src/mongo/db/query/plan_executor_factory.cpp index 0b66c3dafce..386bf38a538 100644 --- a/src/mongo/db/query/plan_executor_factory.cpp +++ b/src/mongo/db/query/plan_executor_factory.cpp @@ -31,6 +31,8 @@ #include "mongo/platform/basic.h" +#include <iostream> + #include "mongo/db/query/plan_executor_factory.h" #include "mongo/db/exec/plan_stage.h" @@ -40,6 +42,7 @@ #include "mongo/db/query/query_planner_params.h" #include "mongo/db/query/util/make_data_structure.h" #include "mongo/logv2/log.h" +#include "mongo/util/duration.h" namespace mongo::plan_executor_factory { @@ -65,6 +68,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( yieldPolicy); } + StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( const boost::intrusive_ptr<ExpressionContext>& expCtx, std::unique_ptr<WorkingSet> ws, @@ -74,6 +78,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( size_t plannerOptions, NamespaceString nss, std::unique_ptr<QuerySolution> qs) { + return make(expCtx->opCtx, std::move(ws), std::move(rt), @@ -98,6 +103,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( NamespaceString nss, PlanYieldPolicy::YieldPolicy yieldPolicy) { dassert(collection); + try { auto execImpl = new PlanExecutorImpl(opCtx, std::move(ws), @@ -128,7 +134,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( NamespaceString nss, std::unique_ptr<PlanYieldPolicySBE> yieldPolicy) { auto&& [rootStage, data] = root; - LOGV2_DEBUG(4822860, 5, "SBE plan", @@ -157,7 +162,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make( size_t plannerOptions, NamespaceString nss, std::unique_ptr<PlanYieldPolicySBE> yieldPolicy) { - LOGV2_DEBUG(4822861, 5, "SBE plan", diff --git a/src/mongo/db/query/plan_executor_factory.h b/src/mongo/db/query/plan_executor_factory.h index bf41f169af9..df7184583ec 100644 --- a/src/mongo/db/query/plan_executor_factory.h +++ b/src/mongo/db/query/plan_executor_factory.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/util/duration.h" #include <queue> #include "mongo/db/exec/sbe/stages/stages.h" diff --git a/src/mongo/db/query/plan_executor_impl.cpp b/src/mongo/db/query/plan_executor_impl.cpp index 03de8319f5c..ae40ecd070e 100644 --- a/src/mongo/db/query/plan_executor_impl.cpp +++ b/src/mongo/db/query/plan_executor_impl.cpp @@ -33,6 +33,7 @@ #include "mongo/db/query/plan_executor_impl.h" +#include "mongo/util/duration.h" #include <memory> #include "mongo/bson/simple_bsonobj_comparator.h" @@ -136,13 +137,6 @@ PlanExecutorImpl::PlanExecutorImpl(OperationContext* opCtx, invariant(!_expCtx || _expCtx->opCtx == _opCtx); invariant(!_cq || !_expCtx || _cq->getExpCtx() == _expCtx); - // If this PlanExecutor is executing a COLLSCAN, keep a pointer directly to the COLLSCAN - // stage. This is used for change streams in order to keep the the latest oplog timestamp - // and post batch resume token up to date as the oplog scan progresses. - if (auto collectionScan = getStageByType(_root.get(), STAGE_COLLSCAN)) { - _collScanStage = static_cast<CollectionScan*>(collectionScan); - } - // If we don't yet have a namespace string, then initialize it from either 'collection' or // '_cq'. if (_nss.isEmpty()) { @@ -174,6 +168,13 @@ PlanExecutorImpl::PlanExecutorImpl(OperationContext* opCtx, _planExplainer->updateEnumeratorExplainInfo( subplanStage->compositeSolution()->_enumeratorExplainInfo); } + + // If this PlanExecutor is executing a COLLSCAN, keep a pointer directly to the COLLSCAN + // stage. This is used for change streams in order to keep the the latest oplog timestamp + // and post batch resume token up to date as the oplog scan progresses. + if (auto collectionScan = getStageByType(_root.get(), STAGE_COLLSCAN)) { + _collScanStage = static_cast<CollectionScan*>(collectionScan); + } } Status PlanExecutorImpl::_pickBestPlan() { diff --git a/src/mongo/db/query/plan_executor_impl.h b/src/mongo/db/query/plan_executor_impl.h index 672cd75f243..ac593cd710c 100644 --- a/src/mongo/db/query/plan_executor_impl.h +++ b/src/mongo/db/query/plan_executor_impl.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/util/duration.h" #include <boost/optional.hpp> #include <queue> diff --git a/src/mongo/db/query/plan_executor_sbe.cpp b/src/mongo/db/query/plan_executor_sbe.cpp index c8d516cb718..e1909acf95c 100644 --- a/src/mongo/db/query/plan_executor_sbe.cpp +++ b/src/mongo/db/query/plan_executor_sbe.cpp @@ -41,6 +41,7 @@ #include "mongo/db/query/sbe_stage_builder.h" #include "mongo/logv2/log.h" #include "mongo/s/resharding/resume_token_gen.h" +#include "mongo/util/duration.h" namespace mongo { // This failpoint is defined by the classic executor but is also accessed here. diff --git a/src/mongo/db/query/plan_executor_sbe.h b/src/mongo/db/query/plan_executor_sbe.h index 547f0fb0a07..c11d97392ee 100644 --- a/src/mongo/db/query/plan_executor_sbe.h +++ b/src/mongo/db/query/plan_executor_sbe.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/util/duration.h" #include <queue> #include "mongo/db/exec/sbe/stages/stages.h" diff --git a/src/mongo/db/query/plan_explainer.h b/src/mongo/db/query/plan_explainer.h index b53a10e5655..ffaabecc43a 100644 --- a/src/mongo/db/query/plan_explainer.h +++ b/src/mongo/db/query/plan_explainer.h @@ -29,12 +29,14 @@ #pragma once +#include "mongo/bson/bsonobj.h" #include "mongo/db/exec/plan_stats.h" #include "mongo/db/query/classic_plan_cache.h" #include "mongo/db/query/explain_options.h" #include "mongo/db/query/plan_enumerator_explain_info.h" #include "mongo/db/query/plan_summary_stats.h" #include "mongo/db/query/query_solution.h" +#include "mongo/util/duration.h" namespace mongo { /** diff --git a/src/mongo/db/query/plan_explainer_factory.cpp b/src/mongo/db/query/plan_explainer_factory.cpp index 544ab33fdd2..9307ec349f6 100644 --- a/src/mongo/db/query/plan_explainer_factory.cpp +++ b/src/mongo/db/query/plan_explainer_factory.cpp @@ -34,6 +34,8 @@ #include "mongo/db/exec/plan_cache_util.h" #include "mongo/db/query/plan_explainer_impl.h" #include "mongo/db/query/plan_explainer_sbe.h" +#include "mongo/util/duration.h" +#include <ratio> namespace mongo::plan_explainer_factory { std::unique_ptr<PlanExplainer> make(PlanStage* root) { diff --git a/src/mongo/db/query/plan_explainer_factory.h b/src/mongo/db/query/plan_explainer_factory.h index 5e24a755747..0064df7cb39 100644 --- a/src/mongo/db/query/plan_explainer_factory.h +++ b/src/mongo/db/query/plan_explainer_factory.h @@ -36,6 +36,7 @@ #include "mongo/db/query/plan_explainer.h" #include "mongo/db/query/query_solution.h" #include "mongo/db/query/sbe_plan_ranker.h" +#include "mongo/util/duration.h" namespace mongo::plan_explainer_factory { std::unique_ptr<PlanExplainer> make(PlanStage* root); diff --git a/src/mongo/db/query/plan_explainer_impl.cpp b/src/mongo/db/query/plan_explainer_impl.cpp index c36bc330826..a67c9e98a60 100644 --- a/src/mongo/db/query/plan_explainer_impl.cpp +++ b/src/mongo/db/query/plan_explainer_impl.cpp @@ -655,7 +655,6 @@ boost::optional<double> getWinningPlanScore(PlanStage* root) { void PlanExplainerImpl::getSummaryStats(PlanSummaryStats* statsOut) const { invariant(statsOut); - // We can get some of the fields we need from the common stats stored in the // root stage of the plan tree. const CommonStats* common = _root->getCommonStats(); diff --git a/src/mongo/db/query/plan_explainer_impl.h b/src/mongo/db/query/plan_explainer_impl.h index 73ef81ae825..f73a2ec6250 100644 --- a/src/mongo/db/query/plan_explainer_impl.h +++ b/src/mongo/db/query/plan_explainer_impl.h @@ -29,10 +29,12 @@ #pragma once +#include "mongo/bson/bsonobj.h" #include "mongo/db/exec/plan_stage.h" #include "mongo/db/query/plan_enumerator_explain_info.h" #include "mongo/db/query/plan_explainer.h" #include "mongo/db/query/query_solution.h" +#include "mongo/util/duration.h" namespace mongo { /** @@ -47,7 +49,6 @@ public: PlanExplainerImpl(PlanStage* root, const PlanEnumeratorExplainInfo& explainInfo) : PlanExplainer{explainInfo}, _root{root} {} PlanExplainerImpl(PlanStage* root) : _root{root} {} - const ExplainVersion& getVersion() const final; bool isMultiPlan() const final; std::string getPlanSummary() const final; diff --git a/src/mongo/db/query/plan_explainer_sbe.h b/src/mongo/db/query/plan_explainer_sbe.h index 5dc97f90641..a256e1371c6 100644 --- a/src/mongo/db/query/plan_explainer_sbe.h +++ b/src/mongo/db/query/plan_explainer_sbe.h @@ -35,6 +35,7 @@ #include "mongo/db/query/plan_explainer.h" #include "mongo/db/query/query_solution.h" #include "mongo/db/query/sbe_plan_ranker.h" +#include "mongo/util/duration.h" namespace mongo { /** diff --git a/src/mongo/db/query/plan_summary_stats.h b/src/mongo/db/query/plan_summary_stats.h index ac80b6505f3..5c153ef3e39 100644 --- a/src/mongo/db/query/plan_summary_stats.h +++ b/src/mongo/db/query/plan_summary_stats.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/util/duration.h" #include <optional> #include <string> diff --git a/src/mongo/db/query/planner_access.cpp b/src/mongo/db/query/planner_access.cpp index 2e8e77c44b5..de294ad661a 100644 --- a/src/mongo/db/query/planner_access.cpp +++ b/src/mongo/db/query/planner_access.cpp @@ -45,14 +45,21 @@ #include "mongo/db/matcher/expression.h" #include "mongo/db/matcher/expression_array.h" #include "mongo/db/matcher/expression_geo.h" +#include "mongo/db/matcher/expression_internal_expr_comparison.h" +#include "mongo/db/matcher/expression_leaf.h" #include "mongo/db/matcher/expression_text.h" +#include "mongo/db/matcher/expression_tree.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/query/index_bounds.h" #include "mongo/db/query/index_bounds_builder.h" #include "mongo/db/query/index_tag.h" #include "mongo/db/query/indexability.h" +#include "mongo/db/query/parsed_find_command.h" #include "mongo/db/query/planner_wildcard_helpers.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/db/query/query_planner.h" #include "mongo/db/query/query_planner_common.h" +#include "mongo/db/query/record_id_range.h" #include "mongo/db/record_id_helpers.h" #include "mongo/logv2/log.h" #include "mongo/util/transitional_tools_do_not_use/vector_spooling.h" @@ -269,7 +276,8 @@ bool compatibleCollator(const QueryPlannerParams& params, void handleRIDRangeMinMax(const CanonicalQuery& query, CollectionScanNode* collScan, const QueryPlannerParams& params, - const CollatorInterface* collator) { + const CollatorInterface* collator, + RecordIdRange& recordRange) { BSONObj minObj = query.getFindCommandRequest().getMin(); BSONObj maxObj = query.getFindCommandRequest().getMax(); if (minObj.isEmpty() && maxObj.isEmpty()) { @@ -289,16 +297,16 @@ void handleRIDRangeMinMax(const CanonicalQuery& query, if (!maxObj.isEmpty() && compatibleCollator(params, collator, maxObj.firstElement())) { // max() is exclusive. // Assumes clustered collection scans are only supported with the forward direction. - collScan->boundInclusion = - CollectionScanParams::ScanBoundInclusion::kIncludeStartRecordOnly; - setLowestRecord(collScan->maxRecord, - IndexBoundsBuilder::objFromElement(maxObj.firstElement(), collator)); + recordRange.maybeNarrowMax( + IndexBoundsBuilder::objFromElement(maxObj.firstElement(), collator), + false /* NOT inclusive*/); } if (!minObj.isEmpty() && compatibleCollator(params, collator, minObj.firstElement())) { // The min() is inclusive as are bounded collection scans by default. - setHighestRecord(collScan->minRecord, - IndexBoundsBuilder::objFromElement(minObj.firstElement(), collator)); + recordRange.maybeNarrowMin( + IndexBoundsBuilder::objFromElement(minObj.firstElement(), collator), + true /* inclusive*/); } } @@ -323,7 +331,8 @@ void handleRIDRangeMinMax(const CanonicalQuery& query, [[nodiscard]] bool handleRIDRangeScan(const MatchExpression* conjunct, CollectionScanNode* collScan, const QueryPlannerParams& params, - const CollatorInterface* collator) { + const CollatorInterface* collator, + RecordIdRange& recordRange) { invariant(params.clusteredInfo); if (conjunct == nullptr) { @@ -334,7 +343,8 @@ void handleRIDRangeMinMax(const CanonicalQuery& query, if (andMatchPtr != nullptr) { bool atLeastOneConjunctCompatibleCollation = false; for (size_t index = 0; index < andMatchPtr->numChildren(); index++) { - if (handleRIDRangeScan(andMatchPtr->getChild(index), collScan, params, collator)) { + if (handleRIDRangeScan( + andMatchPtr->getChild(index), collScan, params, collator, recordRange)) { atLeastOneConjunctCompatibleCollation = true; } } @@ -378,31 +388,35 @@ void handleRIDRangeMinMax(const CanonicalQuery& query, } } + // {min,max}RecordId will bound the range of ids scanned to the highest and lowest present + // in the InMatchExpression, but the filter is still required to filter to _exactly_ the + // requested matches. + // Finally, tighten the collscan bounds with the min/max bounds for the $in. - if (minBound) { - setHighestRecord(collScan->minRecord, *minBound); - } - if (maxBound) { - setLowestRecord(collScan->maxRecord, *maxBound); - } + recordRange.intersectRange(minBound, maxBound); return allEltsCollationCompatible; } - auto match = dynamic_cast<const ComparisonMatchExpression*>(conjunct); + auto match = dynamic_cast<const ComparisonMatchExpressionBase*>(conjunct); if (match == nullptr) { return false; // Not a comparison match expression. } const auto& element = match->getData(); - // Set coarse min/max bounds based on type in case we can't set tight bounds. - BSONObjBuilder minb; - minb.appendMinForType("", element.type()); - setHighestRecord(collScan->minRecord, minb.obj()); + if (!ComparisonMatchExpressionBase::isInternalExprComparison(match->matchType())) { + // Internal comparisons e.g., $_internalExprGt do _not_ carry type bracketing + // semantics (consistent with `$expr{$gt:[a,b]}`). + // For other comparisons which _do_ perform type bracketing, the RecordId bounds + // may be tightened here. + BSONObjBuilder minb; + minb.appendMinForType("", element.type()); + recordRange.maybeNarrowMin(minb.obj(), true /* inclusive */); - BSONObjBuilder maxb; - maxb.appendMaxForType("", element.type()); - setLowestRecord(collScan->maxRecord, maxb.obj()); + BSONObjBuilder maxb; + maxb.appendMaxForType("", element.type()); + recordRange.maybeNarrowMax(maxb.obj(), true /* inclusive */); + } bool compatible = compatibleCollator(params, collator, element); if (!compatible) { @@ -410,19 +424,33 @@ void handleRIDRangeMinMax(const CanonicalQuery& query, } // Even if the collations don't match at this point, it's fine, - // because the bounds exclude values that use it - const auto collated = IndexBoundsBuilder::objFromElement(element, collator); - if (dynamic_cast<const EqualityMatchExpression*>(match)) { - setHighestRecord(collScan->minRecord, collated); - setLowestRecord(collScan->maxRecord, collated); - } else if (dynamic_cast<const LTMatchExpression*>(match) || - dynamic_cast<const LTEMatchExpression*>(match)) { - setLowestRecord(collScan->maxRecord, collated); - } else if (dynamic_cast<const GTMatchExpression*>(match) || - dynamic_cast<const GTEMatchExpression*>(match)) { - setHighestRecord(collScan->minRecord, collated); + // because the bounds exclude values that use it. + const BSONObj collated = IndexBoundsBuilder::objFromElement(element, collator); + using MType = MatchExpression::MatchType; + switch (match->matchType()) { + case MType::EQ: + case MType::INTERNAL_EXPR_EQ: + recordRange.maybeNarrowMin(collated, true /* inclusive */); + recordRange.maybeNarrowMax(collated, true /* inclusive */); + break; + case MType::LT: + case MType::INTERNAL_EXPR_LT: + recordRange.maybeNarrowMax(collated, false /* EXclusive */); + break; + case MType::LTE: + case MType::INTERNAL_EXPR_LTE: + recordRange.maybeNarrowMax(collated, true /* inclusive */); + break; + case MType::GT: + case MType::INTERNAL_EXPR_GT: + recordRange.maybeNarrowMin(collated, false /* EXclusive */); + break; + case MType::GTE: + case MType::INTERNAL_EXPR_GTE: + recordRange.maybeNarrowMin(collated, true /* inclusive */); + break; + default:; } - return true; } @@ -528,11 +556,21 @@ std::unique_ptr<QuerySolutionNode> QueryPlannerAccess::makeCollectionScan( // query is guaranteed to exclude values of the cluster key which are affected by collation. // If so, then even if the query and collection collations differ, the collation difference // won't affect the query results. In that case, we can say hasCompatibleCollation is true. + + RecordIdRange recordRange; + // min/max records may have been set if oplog or change collection. + recordRange.intersectRange(csn->minRecord, csn->maxRecord); bool compatibleCollation = - handleRIDRangeScan(csn->filter.get(), csn.get(), params, queryCollator); + handleRIDRangeScan(csn->filter.get(), csn.get(), params, queryCollator, recordRange); csn->hasCompatibleCollation |= compatibleCollation; - handleRIDRangeMinMax(query, csn.get(), params, queryCollator); + handleRIDRangeMinMax(query, csn.get(), params, queryCollator, recordRange); + + csn->minRecord = recordRange.getMin(); + csn->maxRecord = recordRange.getMax(); + + csn->boundInclusion = CollectionScanParams::makeInclusion(recordRange.isMinInclusive(), + recordRange.isMaxInclusive()); } return csn; diff --git a/src/mongo/db/query/planner_access_test.cpp b/src/mongo/db/query/planner_access_test.cpp index 52df32c871e..d30544b43fa 100644 --- a/src/mongo/db/query/planner_access_test.cpp +++ b/src/mongo/db/query/planner_access_test.cpp @@ -38,9 +38,7 @@ namespace mongo { namespace { BSONObj serializeMatcher(Matcher* matcher) { - BSONObjBuilder builder; - matcher->getMatchExpression()->serialize(&builder); - return builder.obj(); + return matcher->getMatchExpression()->serialize(); } TEST(PlannerAccessTest, PrepareForAccessPlanningSortsEqualNodesByTheirChildren) { diff --git a/src/mongo/db/query/planner_ixselect.cpp b/src/mongo/db/query/planner_ixselect.cpp index c2e06a1027a..b63484717f2 100644 --- a/src/mongo/db/query/planner_ixselect.cpp +++ b/src/mongo/db/query/planner_ixselect.cpp @@ -256,10 +256,9 @@ void QueryPlannerIXSelect::getFields(const MatchExpression* node, if (Indexability::nodeCanUseIndexOnOwnField(node)) { bool supportSparse = Indexability::nodeSupportedBySparseIndex(node); (*out)[prefix + node->path().toString()] = {supportSparse}; - } else if (Indexability::arrayUsesIndexOnChildren(node) && !node->path().empty()) { + } else if (Indexability::isBoundsGeneratingElemMatchObject(node)) { // If the array uses an index on its children, it's something like // {foo : {$elemMatch: {bar: 1}}}, in which case the predicate is really over foo.bar. - // Note we skip empty path components since they are not allowed in index key patterns. prefix += node->path().toString() + "."; for (size_t i = 0; i < node->numChildren(); ++i) { @@ -440,10 +439,13 @@ bool QueryPlannerIXSelect::_compatible(const BSONElement& keyPatternElt, const auto* child = node->getChild(0); const MatchExpression::MatchType childtype = child->matchType(); - // Can't index negations of MOD, REGEX, TYPE_OPERATOR, or ELEM_MATCH_VALUE. + // Can't index negations of MOD, REGEX, TYPE_OPERATOR, or ELEM_MATCH_VALUE; and, as + // above, we can't use a btree-indexed field for geo expressions (or their negations). if (MatchExpression::REGEX == childtype || MatchExpression::MOD == childtype || MatchExpression::TYPE_OPERATOR == childtype || - MatchExpression::ELEM_MATCH_VALUE == childtype) { + MatchExpression::ELEM_MATCH_VALUE == childtype || + MatchExpression::GEO == childtype || MatchExpression::GEO_NEAR == childtype || + MatchExpression::INTERNAL_BUCKET_GEO_WITHIN == childtype) { return false; } diff --git a/src/mongo/db/query/planner_ixselect_test.cpp b/src/mongo/db/query/planner_ixselect_test.cpp index 1df4d714e67..2507e65ce56 100644 --- a/src/mongo/db/query/planner_ixselect_test.cpp +++ b/src/mongo/db/query/planner_ixselect_test.cpp @@ -1333,6 +1333,45 @@ TEST(QueryPlannerIXSelectTest, HashedSparseIndexShouldBeRelevantForExistsTrue) { testRateIndices("{a: {$exists: true}}", "", kSimpleCollator, {entry}, "a", expectedIndices); } +TEST(QueryPlannerIXSelectTest, GeoPredicateCanOnlyUse2dsphereIndex) { + std::vector<IndexEntry> indices; + auto btreeEntry = buildSimpleIndexEntry(BSON("loc" << 1)); + auto twodSphereEntry = buildSimpleIndexEntry(BSON("loc" + << "2dsphere")); + indices.push_back(btreeEntry); + indices.push_back(twodSphereEntry); + std::set<size_t> expectedIndices = {1}; + testRateIndices(R"({loc: {$geoWithin: {$geometry: {type: 'Polygon', + coordinates: [[[0,0],[0,1],[1,0],[0,0]]]}}}})", + "", + kSimpleCollator, + indices, + "loc", + expectedIndices); +} + +TEST(QueryPlannerIXSelectTest, GeoPredicateWithNotCanOnlyUse2dsphereIndex) { + std::vector<IndexEntry> indices; + auto btreeEntry = buildSimpleIndexEntry(BSON("loc" << 1)); + auto twodSphereEntry = buildSimpleIndexEntry(BSON("loc" + << "2dsphere")); + indices.push_back(btreeEntry); + indices.push_back(twodSphereEntry); + // This query gets parsed to {$not: {$and: {$geoWithin: <>}}} and then tags + // the 2dsphere index as relevant. If the $and is optimized away ({$not: {$geoWithin: <>}}), + // that tagging is skipped. + // TODO SERVER-92427: The tagging behavior should be made consistent so that this query has no + // expectedIndices. + std::set<size_t> expectedIndices = {1}; + testRateIndices(R"({loc: {$not: {$geoWithin: {$geometry: {type: 'Polygon', + coordinates: [[[0,0],[0,1],[1,0],[0,0]]]}}}}})", + "", + kSimpleCollator, + indices, + "loc", + expectedIndices); +} + /* * Will compare 'keyPatterns' with 'entries'. As part of comparing, it will sort both of them. */ diff --git a/src/mongo/db/query/projection.cpp b/src/mongo/db/query/projection.cpp index b55fc03602a..2646b39bd8c 100644 --- a/src/mongo/db/query/projection.cpp +++ b/src/mongo/db/query/projection.cpp @@ -227,6 +227,13 @@ void optimizeProjection(ProjectionPathASTNode* root) { Projection::Projection(ProjectionPathASTNode root, ProjectType type) : _root(std::move(root)), _type(type), _deps(analyzeProjection(&_root, type)) {} +void Projection::optimize() { + if (!_projOptimized) { + optimizeProjection(&_root); + _deps = analyzeProjection(&_root, _type); + _projOptimized = true; + } +} namespace { /** diff --git a/src/mongo/db/query/projection.h b/src/mongo/db/query/projection.h index 97b8d1e0d30..95d41caba02 100644 --- a/src/mongo/db/query/projection.h +++ b/src/mongo/db/query/projection.h @@ -142,14 +142,25 @@ public: return _deps.containsElemMatch; } + /** + * Optimizes the projection tree. Additionally, re-computes dependencies in case anything + * changes as in projection {x: {$and: [false, "$b"]}} - which when optimized will no longer + * depend on "b". + */ + void optimize(); + private: ProjectionPathASTNode _root; ProjectType _type; ProjectionDependencies _deps; + bool _projOptimized = false; }; /** - * Walks the projection AST and optimizes each node. + * Walks the projection AST and optimizes each node. Note if you have a 'Projection' instance you + * should prefer to use Projection::optimize() since it will additionally re-compute dependencies in + * case anything changes as in projection {x: {$and: [false, "$b"]}} - which when optimized will no + * longer depend on "b". */ void optimizeProjection(ProjectionPathASTNode* root); diff --git a/src/mongo/db/query/projection_ast.h b/src/mongo/db/query/projection_ast.h index 247a91537f3..1304c68efe5 100644 --- a/src/mongo/db/query/projection_ast.h +++ b/src/mongo/db/query/projection_ast.h @@ -275,7 +275,7 @@ public: ExpressionASTNode(boost::intrusive_ptr<Expression> expr) : _expr(expr) {} ExpressionASTNode(const ExpressionASTNode& other) : ASTNode(other) { BSONObjBuilder bob; - bob << "" << other._expr->serialize(false); + bob << "" << other._expr->serialize(); // TODO SERVER-31003: add a clone() method to Expression. // Temporary stop expression counters while processing the cloned expression. diff --git a/src/mongo/db/query/projection_ast_test.cpp b/src/mongo/db/query/projection_ast_test.cpp index 619bd9a1dde..ea1eb2b683d 100644 --- a/src/mongo/db/query/projection_ast_test.cpp +++ b/src/mongo/db/query/projection_ast_test.cpp @@ -41,6 +41,7 @@ #include "mongo/db/query/projection_ast_util.h" #include "mongo/db/query/projection_parser.h" #include "mongo/db/query/query_planner_test_fixture.h" +#include "mongo/db/query/query_shape/serialization_options.h" namespace { @@ -772,4 +773,62 @@ TEST_F(ProjectionASTTest, ShouldThrowWithPositionalOnExclusion) { DBException, 31395); } + +TEST_F(ProjectionASTTest, TestASTRedaction) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + + auto proj = fromjson("{'a.b': 1}"); + BSONObj output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options); + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"HASH<a>":{"HASH<b>":true},"HASH<_id>":true})", + output); + + proj = fromjson("{'a.b': 0}"); + output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options); + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"HASH<a>":{"HASH<b>":false}})", + output); + + proj = fromjson("{a: 1, b: 1}"); + output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options); + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"HASH<a>":true,"HASH<b>":true,"HASH<_id>":true})", + output); + + // ElemMatch projection + proj = fromjson("{f: {$elemMatch: {foo: 'bar'}}}"); + output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options); + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"HASH<f>":{"$elemMatch":{"HASH<foo>":{"$eq":"?string"}}},"HASH<_id>":true})", + output); + + // Positional projection + proj = fromjson("{'x.$': 1}"); + output = projection_ast::serialize( + *parseWithFindFeaturesEnabled(proj, fromjson("{'x.a': 2}")).root(), {}); + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"x.$":true,"_id":true})", + output); + + // Slice (first form) + proj = fromjson("{a: {$slice: 1}}"); + output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options); + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"HASH<a>":{"$slice":"?number"}})", + output); + + // Slice (second form) + proj = fromjson("{a: {$slice: [1, 3]}}"); + output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options); + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"HASH<a>":{"$slice":["?number","?number"]}})", + output); + + /// $meta projection + proj = fromjson("{foo: {$meta: 'indexKey'}}"); + output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options); + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"HASH<foo>":{"$meta":"indexKey"}})", + output); +} } // namespace diff --git a/src/mongo/db/query/projection_ast_util.cpp b/src/mongo/db/query/projection_ast_util.cpp index e5b4cc1a9c4..23c7b6d9582 100644 --- a/src/mongo/db/query/projection_ast_util.cpp +++ b/src/mongo/db/query/projection_ast_util.cpp @@ -29,28 +29,26 @@ #include "mongo/platform/basic.h" -#include "mongo/db/query/projection_ast_util.h" - #include "mongo/db/query/projection_ast_path_tracking_visitor.h" +#include "mongo/db/query/projection_ast_util.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/db/query/tree_walker.h" namespace mongo::projection_ast { namespace { struct BSONVisitorContext { std::stack<BSONObjBuilder> builders; + bool underElemMatch = false; }; class BSONPreVisitor : public ProjectionASTConstVisitor { public: - BSONPreVisitor(PathTrackingVisitorContext<BSONVisitorContext>* context) - : _context(context), _builders(context->data().builders) {} - - virtual void visit(const MatchExpressionASTNode* node) { - static_cast<const MatchExpressionASTNode*>(node)->matchExpression()->serialize( - &_builders.top(), true); - } + using ProjectionASTConstVisitor::visit; + BSONPreVisitor(PathTrackingVisitorContext<BSONVisitorContext>* context, + SerializationOptions options) + : _context(context), _builders(context->data().builders), _options(std::move(options)) {} - virtual void visit(const ProjectionPathASTNode* node) { + void visit(const ProjectionPathASTNode* node) override { if (!node->parent()) { // No root of the tree, thus this node has no field name. _builders.push(BSONObjBuilder()); @@ -59,47 +57,46 @@ public: } } - virtual void visit(const ProjectionPositionalASTNode* node) { - // ProjectionPositional always has the original query's match expression node as its - // child. Serialize as: {"positional.projection.field.$": <original match expression>}. - _context->data().builders.push(_builders.top().subobjStart(getFieldName() + ".$")); - } - - virtual void visit(const ProjectionSliceASTNode* node) { + void visit(const ProjectionSliceASTNode* node) override { BSONObjBuilder sub(_builders.top().subobjStart(getFieldName())); if (node->skip()) { - sub.appendArray("$slice", BSON_ARRAY(*node->skip() << node->limit())); + sub.appendArray("$slice", + BSON_ARRAY(_options.serializeLiteral(*node->skip()) + << _options.serializeLiteral(node->limit()))); } else { - sub.appendNumber("$slice", node->limit()); + _options.appendLiteral(&sub, "$slice", node->limit()); } } - virtual void visit(const ProjectionElemMatchASTNode* node) { - // Defer to the child, match expression node. - } - virtual void visit(const ExpressionASTNode* node) { - node->expression()->serialize(false).addToBsonObj(&_builders.top(), getFieldName()); + void visit(const ExpressionASTNode* node) override { + node->expression()->serialize(_options).addToBsonObj(&_builders.top(), getFieldName()); } - virtual void visit(const BooleanConstantASTNode* node) { + void visit(const BooleanConstantASTNode* node) override { _builders.top().append(getFieldName(), node->value()); } -private: + void visit(const ProjectionPositionalASTNode* node) override = 0; + void visit(const ProjectionElemMatchASTNode* node) override = 0; + void visit(const MatchExpressionASTNode* node) override = 0; + +protected: std::string getFieldName() { - return _context->childPath(); + return _options.serializeFieldPathFromString(_context->childPath()); } PathTrackingVisitorContext<BSONVisitorContext>* _context; std::stack<BSONObjBuilder>& _builders; + SerializationOptions _options; }; class BSONPostVisitor : public ProjectionASTConstVisitor { public: + using ProjectionASTConstVisitor::visit; BSONPostVisitor(BSONVisitorContext* context) : _context(context) {} - virtual void visit(const ProjectionPathASTNode* node) { + void visit(const ProjectionPathASTNode* node) override { // Don't pop the top builder. if (node->parent()) { // Pop the BSONObjBuilder that was added in the pre visitor. @@ -107,25 +104,97 @@ public: } } - virtual void visit(const ProjectionPositionalASTNode* node) { + void visit(const ProjectionSliceASTNode* node) override {} + void visit(const ExpressionASTNode* node) override {} + void visit(const BooleanConstantASTNode* node) override {} + void visit(const MatchExpressionASTNode* node) override {} + + void visit(const ProjectionPositionalASTNode* node) override = 0; + void visit(const ProjectionElemMatchASTNode* node) override = 0; + +protected: + BSONVisitorContext* _context; +}; + +class DebugPreVisitor : public BSONPreVisitor { +public: + using BSONPreVisitor::visit; + DebugPreVisitor(PathTrackingVisitorContext<BSONVisitorContext>* context) + : BSONPreVisitor(context, SerializationOptions{}) {} + + void visit(const ProjectionPositionalASTNode* node) override { + // ProjectionPositional always has the original query's match expression node as its + // child. Serialize as: {"positional.projection.field.$": <original match expression>}. + _context->data().builders.push(_builders.top().subobjStart(getFieldName() + ".$")); + } + + void visit(const ProjectionElemMatchASTNode* node) override { + // Defer to the child, match expression node. + } + + void visit(const MatchExpressionASTNode* node) override { + static_cast<const MatchExpressionASTNode*>(node)->matchExpression()->serialize( + &_builders.top(), {}); + } +}; + +class DebugPostVisitor : public BSONPostVisitor { +public: + using BSONPostVisitor::visit; + DebugPostVisitor(BSONVisitorContext* context) : BSONPostVisitor(context) {} + + void visit(const ProjectionPositionalASTNode* node) override { _context->builders.pop(); } - virtual void visit(const MatchExpressionASTNode* node) {} - virtual void visit(const ProjectionSliceASTNode* node) {} - virtual void visit(const ProjectionElemMatchASTNode* node) {} - virtual void visit(const ExpressionASTNode* node) {} - virtual void visit(const BooleanConstantASTNode* node) {} + void visit(const ProjectionElemMatchASTNode* node) override {} +}; -private: - BSONVisitorContext* _context; +class SerializationPreVisitor : public BSONPreVisitor { +public: + using BSONPreVisitor::visit; + SerializationPreVisitor(PathTrackingVisitorContext<BSONVisitorContext>* context, + const SerializationOptions& options) + : BSONPreVisitor(context, options) {} + + void visit(const ProjectionPositionalASTNode* node) override { + tassert(73488, + "Positional projection should not appear below an $elemMatch projection.", + !_context->data().underElemMatch); + _builders.top().append(getFieldName() + ".$", true); + } + + void visit(const ProjectionElemMatchASTNode* node) override { + // The child match expression node should begin with $elemMatch. + _context->data().underElemMatch = true; + } + + void visit(const MatchExpressionASTNode* node) override { + if (_context->data().underElemMatch) { + static_cast<const MatchExpressionASTNode*>(node)->matchExpression()->serialize( + &_builders.top(), _options); + } + } }; + +class SerializationPostVisitor : public BSONPostVisitor { +public: + using BSONPostVisitor::visit; + SerializationPostVisitor(BSONVisitorContext* context) : BSONPostVisitor(context) {} + + void visit(const ProjectionPositionalASTNode* node) override {} + void visit(const ProjectionElemMatchASTNode* node) override { + _context->underElemMatch = false; + } +}; + } // namespace BSONObj astToDebugBSON(const ASTNode* root) { PathTrackingVisitorContext<BSONVisitorContext> context; - BSONPreVisitor preVisitor{&context}; - BSONPostVisitor postVisitor{&context.data()}; + DebugPreVisitor preVisitor{&context}; + DebugPostVisitor postVisitor{&context.data()}; + PathTrackingWalker walker{&context, {&preVisitor}, {&postVisitor}}; tree_walker::walk<true, projection_ast::ASTNode>(root, &walker); @@ -133,4 +202,15 @@ BSONObj astToDebugBSON(const ASTNode* root) { invariant(context.data().builders.size() == 1); return context.data().builders.top().obj(); } + +BSONObj serialize(const ProjectionPathASTNode& root, const SerializationOptions& options) { + PathTrackingVisitorContext<BSONVisitorContext> context; + SerializationPreVisitor preVisitor{&context, options}; + SerializationPostVisitor postVisitor{&context.data()}; + PathTrackingWalker walker{&context, {&preVisitor}, {&postVisitor}}; + tree_walker::walk<true, projection_ast::ASTNode>(&root, &walker); + + invariant(context.data().builders.size() == 1); + return context.data().builders.top().obj(); +} } // namespace mongo::projection_ast diff --git a/src/mongo/db/query/projection_ast_util.h b/src/mongo/db/query/projection_ast_util.h index af89254a9a1..de102dfbbd7 100644 --- a/src/mongo/db/query/projection_ast_util.h +++ b/src/mongo/db/query/projection_ast_util.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/db/query/projection.h" #include "mongo/db/query/projection_ast.h" namespace mongo { @@ -37,5 +38,7 @@ namespace projection_ast { * This is intended to be used for debug output, not for serialization. */ BSONObj astToDebugBSON(const ASTNode* root); + +BSONObj serialize(const ProjectionPathASTNode& root, const SerializationOptions& options); } // namespace projection_ast } // namespace mongo diff --git a/src/mongo/db/query/projection_parser.cpp b/src/mongo/db/query/projection_parser.cpp index 85ada560ccc..3495f99f8f2 100644 --- a/src/mongo/db/query/projection_parser.cpp +++ b/src/mongo/db/query/projection_parser.cpp @@ -510,7 +510,16 @@ void parseSubObject(ParseContext* ctx, // It was likely intended to be an expression. Check if it's a valid field path or not to // confirm. try { - FieldPath fp(obj.firstElementFieldNameStringData()); + const auto elementFieldName = obj.firstElementFieldNameStringData(); + if (!hasPositionalOperator(elementFieldName)) { + FieldPath fp(elementFieldName); + } else { + // The 'FieldPath' parser doesn't take positional operators into account, but those + // are valid path projections so trim it off for this validation. + StringData pathWithoutPositionalOperator = + elementFieldName.substr(0, elementFieldName.size() - 2); + FieldPath fp(pathWithoutPositionalOperator); + } } catch (const DBException&) { uasserted(31325, str::stream() diff --git a/src/mongo/db/query/query_feature_flags.idl b/src/mongo/db/query/query_feature_flags.idl index f97cec95cc2..7e5995a022f 100644 --- a/src/mongo/db/query/query_feature_flags.idl +++ b/src/mongo/db/query/query_feature_flags.idl @@ -152,6 +152,12 @@ feature_flags: cpp_varname: gFeatureFlagSbeFull default: false + featureFlagQueryStats: + description: "Feature flag for enabling full queryStats collection." + cpp_varname: gFeatureFlagQueryStats + default: true + version: 6.0 + featureFlagShardedSearchCustomSort: description: "Feature flag to enable user specified sort for sharded $search queries." cpp_varname: gFeatureFlagShardedSearchCustomSort diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl index 0748cee3a33..f0f24d3c3dd 100644 --- a/src/mongo/db/query/query_knobs.idl +++ b/src/mongo/db/query/query_knobs.idl @@ -29,8 +29,9 @@ global: cpp_namespace: "mongo" cpp_includes: - - "mongo/db/query/plan_cache_size_parameter.h" - "mongo/db/query/sbe_plan_cache_on_parameter_change.h" + - "mongo/db/query/util/memory_util.h" + - "mongo/db/query/query_stats/query_stats_on_parameter_change.h" - "mongo/platform/atomic_proxy.h" - "mongo/platform/atomic_word.h" @@ -675,7 +676,7 @@ server_parameters: set_at: [ startup, runtime ] cpp_varname: "internalQueryForceClassicEngine" cpp_vartype: AtomicWord<bool> - default: false + default: true internalQueryAppendIdToSetWindowFieldsSort: description: "If true, appends _id to the sort stage generated by desugaring $setWindowFields to @@ -906,6 +907,69 @@ server_parameters: expr: 8 * 1024 * 1024 # 8MB default: 0 + internalQueryStatsRateLimit: + description: "The maximum number of queries per second that are sampled for query stats. + If the rate of queries goes above this number, then rate limiting will kick in, and any + further queries will not be sampled. To sample all queries, this can be set to -1. This can be + set to 0 to turn queryStats off completely." + set_at: [ startup, runtime ] + cpp_varname: "internalQueryStatsRateLimit" + cpp_vartype: AtomicWord<int> + default: 0 + validator: + gte: -1 + on_update: query_stats_util::onQueryStatsSamplingRateUpdate + + internalQueryStatsCacheSize: + description: "The maximum amount of memory that the system will allocate for the query queryStats + cache. This will accept values in either of the following formats: + 1. <number>% indicates a percentage of the physical memory available to the process. E.g.: 15%. + 2. <number>(MB|GB), indicates the amount of memory in MB or GB. E.g.: 1.5GB, 100MB. + The default value is 1%, which means 1% of the physical memory available to the process." + set_at: [ startup, runtime ] + cpp_varname: "internalQueryStatsCacheSize" + cpp_vartype: synchronized_value<std::string> + default: "1%" + on_update: query_stats_util::onQueryStatsStoreSizeUpdate + validator: + callback: query_stats_util::validateQueryStatsStoreSize + + internalQueryStatsErrorsAreCommandFatal: + description: "Whether errors in the $queryStats stage cause the aggregation pipeline to + immediately fail and report the error. Note that this is always the case for debug builds." + set_at: [ startup, runtime ] + cpp_varname: "internalQueryStatsErrorsAreCommandFatal" + cpp_vartype: AtomicWord<bool> + default: false + + internalQueryAggMulticastTimeoutMS: + description: "Timeout in MS for requests to shard servers when aggregations are sent to all shard servers" + set_at: [ startup ] + cpp_vartype: int + cpp_varname: internalQueryAggMulticastTimeoutMS + default: 60000 + validator: + gte: 0 + + internalQueryAggMulticastMaxConcurrency: + description: "Max number of concurrent requests when aggregations are sent to all shard servers" + set_at: startup + cpp_vartype: int + cpp_varname: internalQueryAggMulticastMaxConcurrency + default: 100 + validator: + gte: 1 + + internalQuerySpillingMaxWaitTimeout: + description: "Timeout in MS that the storage engine will block a spilling operation when the + cache is under pressure." + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: internalQuerySpillingMaxWaitTimeout + default: 1000 + validator: + gte: 0 + # Note for adding additional query knobs: # # When adding a new query knob, you should consider whether or not you need to add an 'on_update' diff --git a/src/mongo/db/query/query_planner_common.h b/src/mongo/db/query/query_planner_common.h index 6d441155b54..97e94bb0a69 100644 --- a/src/mongo/db/query/query_planner_common.h +++ b/src/mongo/db/query/query_planner_common.h @@ -65,6 +65,20 @@ public: } /** + * Returns a count of 'type' nodes in expression tree. + */ + static size_t countNodes(const MatchExpression* root, MatchExpression::MatchType type) { + size_t sum = 0; + if (type == root->matchType()) { + sum = 1; + } + for (size_t i = 0; i < root->numChildren(); ++i) { + sum += countNodes(root->getChild(i), type); + } + return sum; + } + + /** * Assumes the provided BSONObj is of the form {field1: -+1, ..., field2: -+1} * Returns a BSONObj with the values negated. */ diff --git a/src/mongo/db/query/query_planner_test_lib.cpp b/src/mongo/db/query/query_planner_test_lib.cpp index bec219226b5..849ce6fb6e8 100644 --- a/src/mongo/db/query/query_planner_test_lib.cpp +++ b/src/mongo/db/query/query_planner_test_lib.cpp @@ -1234,7 +1234,10 @@ Status QueryPlannerTestLib::solutionMatches(const BSONObj& testSoln, } BSONObjBuilder bob; - actualGroupNode->groupByExpression->serialize(true).addToBsonObj(&bob, "_id"); + actualGroupNode->groupByExpression + ->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}) + .addToBsonObj(&bob, "_id"); auto actualGroupByObj = bob.done(); if (!SimpleBSONObjComparator::kInstance.evaluate(actualGroupByObj == expectedGroupByElem.Obj())) { @@ -1247,7 +1250,10 @@ Status QueryPlannerTestLib::solutionMatches(const BSONObj& testSoln, BSONArrayBuilder actualAccs; for (auto& acc : actualGroupNode->accumulators) { BSONObjBuilder bob; - acc.expr.argument->serialize(true).addToBsonObj(&bob, acc.expr.name); + acc.expr.argument + ->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}) + .addToBsonObj(&bob, acc.expr.name); actualAccs.append(BSON(acc.fieldName << bob.done())); } auto expectedAccsObj = expectedGroupObj["accs"].Obj(); diff --git a/src/mongo/db/query/query_planner_tree_test.cpp b/src/mongo/db/query/query_planner_tree_test.cpp index 5b7055dd720..7ea6d55dfb3 100644 --- a/src/mongo/db/query/query_planner_tree_test.cpp +++ b/src/mongo/db/query/query_planner_tree_test.cpp @@ -434,6 +434,39 @@ TEST_F(QueryPlannerTest, RootedOrOfAndDontCollapseDifferentBounds) { "bounds: {c: [[3,3,true,true]], d: [[4,4,true,true]]}}}]}}}}"); } +TEST_F(QueryPlannerTest, DontCrashTryingToPushToSingleChildIndexedOr1) { + FailPointEnableBlock failPoint("disableMatchExpressionOptimization"); + addIndex(BSON("indexed" << 1)); + runQuery( + fromjson("{ $and : [\n" + " { $and : [ { indexed : { $gt : 5 } },\n" + " { unindexed : 42 } ] },\n" + " { $or : [ { indexed: { $lt : 100 } } ] }\n" + " ] }")); + + assertNumSolutions(3U); +} + +TEST_F(QueryPlannerTest, DontCrashTryingToPushToSingleChildIndexedOr2) { + // Test that queries with single-child $and, $or do not crash when match-expression optimization + // is disabled. Normally these single-child nodes are eliminated, so when they are left in place + // it can confuse OR-pushdown optimization. + // + // Originally designed to reproduce SERVER-70597, which would only happen when the + // INDEX_INTERSECTION option is enabled. + FailPointEnableBlock failPoint("disableMatchExpressionOptimization"); + addIndex(BSON("a" << 1 << "b" << 1)); + + params.options |= QueryPlannerParams::INDEX_INTERSECTION; + runQuery( + fromjson("{ $and : [\n" + " { $and : [ { a : 2 } ] },\n" + " { $or : [ { b : 3 } ] }\n" + " ] }")); + + assertNumSolutions(2U); +} + // SERVER-13960: properly handle $or with a mix of exact and inexact predicates. TEST_F(QueryPlannerTest, OrInexactWithExact) { addIndex(BSON("name" << 1)); diff --git a/src/mongo/db/query/query_request_test.cpp b/src/mongo/db/query/query_request_test.cpp index 1493c352b28..52c6dd4aca7 100644 --- a/src/mongo/db/query/query_request_test.cpp +++ b/src/mongo/db/query/query_request_test.cpp @@ -36,6 +36,7 @@ #include "mongo/base/error_codes.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_mock.h" +#include "mongo/db/cursor_id.h" #include "mongo/db/dbmessage.h" #include "mongo/db/json.h" #include "mongo/db/namespace_string.h" diff --git a/src/mongo/db/query/query_shape.cpp b/src/mongo/db/query/query_shape.cpp new file mode 100644 index 00000000000..02d4c97d25f --- /dev/null +++ b/src/mongo/db/query/query_shape.cpp @@ -0,0 +1,292 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape.h" + +#include "mongo/base/status.h" +#include "mongo/db/query/find_command_gen.h" +#include "mongo/db/query/projection_ast_util.h" +#include "mongo/db/query/query_request_helper.h" +#include "mongo/db/query/query_shape_gen.h" +#include "mongo/db/query/query_stats/query_stats.h" +#include "mongo/db/query/shape_helpers.h" +#include "mongo/db/query/sort_pattern.h" + +namespace mongo::query_shape { + +BSONObj debugPredicateShape(const MatchExpression* predicate) { + SerializationOptions opts; + opts.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + return predicate->serialize(opts); +} +BSONObj representativePredicateShape(const MatchExpression* predicate) { + SerializationOptions opts; + opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue; + return predicate->serialize(opts); +} + +BSONObj debugPredicateShape(const MatchExpression* predicate, + std::function<std::string(StringData)> transformIdentifiersCallback) { + SerializationOptions opts; + opts.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + opts.transformIdentifiersCallback = transformIdentifiersCallback; + opts.transformIdentifiers = true; + return predicate->serialize(opts); +} + +BSONObj representativePredicateShape( + const MatchExpression* predicate, + std::function<std::string(StringData)> transformIdentifiersCallback) { + SerializationOptions opts; + opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue; + opts.transformIdentifiersCallback = transformIdentifiersCallback; + opts.transformIdentifiers = true; + return predicate->serialize(opts); +} + +BSONObj extractSortShape(const BSONObj& sortSpec, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts) { + if (sortSpec.isEmpty()) { + return sortSpec; + } + auto natural = sortSpec[query_request_helper::kNaturalSortField]; + + if (!natural) { + return SortPattern{sortSpec, expCtx} + .serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts) + .toBson(); + } + // This '$natural' will fail to parse as a valid SortPattern since it is not a valid field + // path - it is usually considered and converted into a hint. For the query shape, we'll + // keep it unmodified. + BSONObjBuilder bob; + for (auto&& elem : sortSpec) { + if (elem.isABSONObj()) { + // We expect this won't work or parse on the main command path, but for shapification we + // don't really care, just treat it as a literal and don't bother parsing. + opts.appendLiteral( + &bob, opts.serializeFieldPathFromString(elem.fieldNameStringData()), elem); + } else if (elem.fieldNameStringData() == natural.fieldNameStringData()) { + bob.append(elem); + } else { + bob.appendAs(elem, opts.serializeFieldPathFromString(elem.fieldNameStringData())); + } + } + return bob.obj(); +} + +void addShapeLiterals(BSONObjBuilder* bob, + const FindCommandRequest& findCommand, + const SerializationOptions& opts) { + if (auto limit = findCommand.getLimit()) { + opts.appendLiteral( + bob, FindCommandRequest::kLimitFieldName, static_cast<long long>(*limit)); + } + if (auto skip = findCommand.getSkip()) { + opts.appendLiteral(bob, FindCommandRequest::kSkipFieldName, static_cast<long long>(*skip)); + } +} + +static std::vector< + std::pair<StringData, std::function<const OptionalBool(const FindCommandRequest&)>>> + boolArgMap = { + {FindCommandRequest::kSingleBatchFieldName, &FindCommandRequest::getSingleBatch}, + {FindCommandRequest::kAllowDiskUseFieldName, &FindCommandRequest::getAllowDiskUse}, + {FindCommandRequest::kReturnKeyFieldName, &FindCommandRequest::getReturnKey}, + {FindCommandRequest::kShowRecordIdFieldName, &FindCommandRequest::getShowRecordId}, + {FindCommandRequest::kTailableFieldName, &FindCommandRequest::getTailable}, + {FindCommandRequest::kAwaitDataFieldName, &FindCommandRequest::getAwaitData}, +}; +std::vector<std::pair<StringData, std::function<const BSONObj(const FindCommandRequest&)>>> + objArgMap = { + {FindCommandRequest::kCollationFieldName, &FindCommandRequest::getCollation}, + +}; + +void addRemainingFindCommandFields(BSONObjBuilder* bob, + const FindCommandRequest& findCommand, + const SerializationOptions& opts) { + for (auto [fieldName, getterFunction] : boolArgMap) { + auto optBool = getterFunction(findCommand); + optBool.serializeToBSON(fieldName, bob); + } + + if (auto optOplogReplay = findCommand.getOplogReplay()) { + if (optOplogReplay.has_value()) { + opts.appendLiteral( + bob, FindCommandRequest::kOplogReplayFieldName, optOplogReplay.value_or(false)); + } + } + + auto collation = findCommand.getCollation(); + if (!collation.isEmpty()) { + bob->append(FindCommandRequest::kCollationFieldName, collation); + } +} + + +/** + * In a let specification all field names are variable names, and all values are either + * expressions or constants. + */ +BSONObj extractLetSpecShape(BSONObj letSpec, + const SerializationOptions& opts, + const boost::intrusive_ptr<ExpressionContext>& expCtx) { + + BSONObjBuilder bob; + for (BSONElement elem : letSpec) { + auto expr = Expression::parseOperand(expCtx.get(), elem, expCtx->variablesParseState); + auto redactedValue = expr->serialize(opts); + // Note that this will throw on deeply nested let variables. + redactedValue.addToBsonObj(&bob, opts.serializeFieldPathFromString(elem.fieldName())); + } + return bob.obj(); +} + +void appendCmdNs(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts) { + BSONObjBuilder nsObj = bob.subobjStart("cmdNs"); + shape_helpers::appendNamespaceShape(nsObj, nss, opts); + nsObj.doneFast(); +} + +BSONObj extractQueryShape(const ParsedFindCommand& findRequest, + const SerializationOptions& opts, + const boost::intrusive_ptr<ExpressionContext>& expCtx) { + const auto& findCmd = *findRequest.findCommandRequest; + BSONObjBuilder bob; + // Serialize the namespace as part of the query shape. + { + auto ns = findCmd.getNamespaceOrUUID(); + if (ns.nss().has_value()) { + appendCmdNs(bob, *ns.nss(), opts); + } else { + BSONObjBuilder cmdNs = bob.subobjStart("cmdNs"); + cmdNs.append("uuid", opts.serializeIdentifier(ns.uuid()->toString())); + cmdNs.append("db", opts.serializeIdentifier(ns.db())); + cmdNs.doneFast(); + } + } + + bob.append("command", "find"); + std::unique_ptr<MatchExpression> filterExpr; + // Filter. + bob.append(FindCommandRequest::kFilterFieldName, findRequest.filter->serialize(opts)); + // Let Spec. + if (auto letSpec = findCmd.getLet()) { + auto redactedObj = extractLetSpecShape(letSpec.get(), opts, expCtx); + auto ownedObj = redactedObj.getOwned(); + bob.append(FindCommandRequest::kLetFieldName, std::move(ownedObj)); + } + + if (findRequest.proj) { + bob.append(FindCommandRequest::kProjectionFieldName, + projection_ast::serialize(*findRequest.proj->root(), opts)); + } + + if (!findCmd.getMax().isEmpty()) { + bob.append(FindCommandRequest::kMaxFieldName, + shape_helpers::extractMinOrMaxShape(findCmd.getMax(), opts)); + } + if (!findCmd.getMin().isEmpty()) { + bob.append(FindCommandRequest::kMinFieldName, + shape_helpers::extractMinOrMaxShape(findCmd.getMin(), opts)); + } + + // Sort. + if (findRequest.sort) { + bob.append( + FindCommandRequest::kSortFieldName, + findRequest.sort + ->serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts) + .toBson()); + } + + // Fields for literal redaction. Adds limit and skip. + addShapeLiterals(&bob, findCmd, opts); + + // Add the fields that require no redaction. + addRemainingFindCommandFields(&bob, findCmd, opts); + + return bob.obj(); +} + +BSONObj extractQueryShape(const AggregateCommandRequest& aggregateCommand, + const Pipeline& pipeline, + const SerializationOptions& opts, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const NamespaceString& nss) { + BSONObjBuilder bob; + + // namespace + appendCmdNs(bob, nss, opts); + bob.append("command", "aggregate"); + + // pipeline + { + BSONArrayBuilder pipelineBab( + bob.subarrayStart(AggregateCommandRequest::kPipelineFieldName)); + auto serializedPipeline = pipeline.serializeToBson(opts); + for (const auto& stage : serializedPipeline) { + pipelineBab.append(stage); + } + pipelineBab.doneFast(); + } + + // explain + if (aggregateCommand.getExplain().has_value()) { + bob.append(AggregateCommandRequest::kExplainFieldName, true); + } + + // allowDiskUse + if (auto param = aggregateCommand.getAllowDiskUse(); param.has_value()) { + bob.append(AggregateCommandRequest::kAllowDiskUseFieldName, param.value_or(false)); + } + + // collation + if (auto param = aggregateCommand.getCollation()) { + bob.append(AggregateCommandRequest::kCollationFieldName, param.get()); + } + + // let + if (auto letSpec = aggregateCommand.getLet()) { + auto redactedObj = extractLetSpecShape(letSpec.get(), opts, expCtx); + auto ownedObj = redactedObj.getOwned(); + bob.append(FindCommandRequest::kLetFieldName, std::move(ownedObj)); + } + return bob.obj(); +} + +QueryShapeHash hash(const BSONObj& queryShape) { + return QueryShapeHash::computeHash(reinterpret_cast<const uint8_t*>(queryShape.objdata()), + queryShape.objsize()); +} +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/README.md b/src/mongo/db/query/query_shape/README.md new file mode 100644 index 00000000000..b3c02d28e69 --- /dev/null +++ b/src/mongo/db/query/query_shape/README.md @@ -0,0 +1,69 @@ +# Query Shape +A query shape is a transformed version of a command with literal values replaced by a "canonical" +BSON Type placeholder. Hence, different instances of a command would be considered to have the same +query shape if they are identical once their literal values are abstracted. + +For example, these two queries would have the same shape: +```js +db.example.findOne({x: 24}); +db.example.findOne({x: 53}); +``` +While these queries would each have a distinct shape: +```js +db.example.findOne({x: 53, y: 1}); +db.example.findOne({x: 53}); +db.example.findOne({x: "string"}); +``` +While different literal _values_ result in the same shape (matching `x` for 23 vs 53), different +BSON _types_ of the literal are considered distinct shapes (matching `x` for 53 vs "string"). + +The concept of a query shape exists not just for the find command, but for many of the CRUD commands +and aggregate. It also includes most (but not all) components of these commands, not just the query +predicate (MatchExpresssion). In these ways, "query" is meant more generally. While some components +included in the query shape are shared across the different types of commands (e.g., the "hint" +field), some are unique. For example, a find command would include a `filter` while an aggregate +command would have a `pipeline`. + +You can see which components are considered part of the query shape or not for each specific shape +type in their respective "shape component" classes, whose purpose is to determine which components +are relevant and should be included for determining the shape for specific type of command. The +structure is as follows: +- [`CmdSpecificShapeComponents`](query_shape.h#L65) + - [`LetShapeComponent`](cmd_with_let_shape.h#L48) + - [`AggCmdShapeComponents`](agg_cmd_shape.h#L82) + - [`FindCmdShapeComponents`](find_cmd_shape.h#L48) + +See more information for the different shapes in their respective classes, structured as follows: +- [`Shape`](query_shape.h) + - [`CmdWithLetShape`](cmd_with_let_shape.h) + - [`AggCmdShape`](agg_cmd_shape.h) + - [`FindCmdShape`](find_cmd_shape.h) + +## Serialization Options +`SerializationOptions` describes the way we serialize literal values. + +There are 3 different serialization options: +- `kUnchanged`: literals are serialized unmodified + - `{x: 5, y: "hello"}` -> `{x: 5, y: "hello"}` +- `kToDebugTypeString`: human readable format, type string of the literal is serialized + - `{x: 5, y: "hello"}` -> `{x: "?number", y: "?string"}` +- `kToRepresentativeParseableValue`: literal serialized to one canonical value for given type, which + must be parseable + - `{x: 5, y: "hello"}` -> `{x: 1, y: "?"}` + - An example of a query which is serialized differently due to the parseable requirement is `{x: + {$regex: "^p.*"}}`. If we serialized the pattern as if it were a normal string we would end up + with `{x: {$regex: "?"}}` however `"?"` is not a valid regex pattern, so this would fail + parsing. Instead we will serialize it this way to maintain parseability, `{x: {$regex: + "\\?"}}`, since `"\\?"` is valid regex. + +See [serialization_options.h](serialization_options.h) for more details. + +When we compute the [query shape hash](query_shape.cpp#L99-107), we use the +`kToRepresentativeParseableValue`, since all literals of the same type will become the same value. +This allows us to group together queries that have the same structure but different literal values +into the same shape, since they will result in the same hash. The term we use to refer to this is +"shapify", as we simplify the queries into their query shape. + +When shapifying, we try to get as close as possible to the original user input, but there are some +stages like `$jsonSchema` and `$setWindowFields` that output "internal" stages that are already +transformed from user input. diff --git a/src/mongo/db/query/query_shape/SConscript b/src/mongo/db/query/query_shape/SConscript new file mode 100644 index 00000000000..d4bddba4934 --- /dev/null +++ b/src/mongo/db/query/query_shape/SConscript @@ -0,0 +1,42 @@ +# -*- mode: python -*- + +Import([ + "env", + "get_option", +]) + +env = env.Clone() + +env.Library( + target='query_shape', source=['query_shape.cpp', 'shape_helpers.cpp'], LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/exec/document_value/document_value', + '$BUILD_DIR/mongo/db/pipeline/field_path', + 'query_shape_common', + ], LIBDEPS_PRIVATE=[ + ]) + +env.Library( + target='query_shape_common', source=[ + 'query_shape.idl', + 'serialization_options.cpp', + ], LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/exec/document_value/document_value', + '$BUILD_DIR/mongo/db/pipeline/field_path', + ], LIBDEPS_PRIVATE=[ + ]) + +env.CppUnitTest( + target="db_query_query_shape_test", + source=[ + "query_shape_test.cpp", + "query_shape_test.idl", + ], + LIBDEPS=[ + "$BUILD_DIR/mongo/db/auth/authmocks", + "$BUILD_DIR/mongo/db/query/query_test_service_context", + "$BUILD_DIR/mongo/db/service_context_d_test_fixture", + "query_shape", + ], +) diff --git a/src/mongo/db/query/query_shape/agg_cmd_shape.cpp b/src/mongo/db/query/query_shape/agg_cmd_shape.cpp new file mode 100644 index 00000000000..e997150ecc6 --- /dev/null +++ b/src/mongo/db/query/query_shape/agg_cmd_shape.cpp @@ -0,0 +1,125 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/agg_cmd_shape.h" + +#include "mongo/db/query/query_shape/shape_helpers.h" + +namespace mongo::query_shape { + +AggCmdShapeComponents::AggCmdShapeComponents( + const AggregateCommandRequest& aggRequest, + stdx::unordered_set<NamespaceString> involvedNamespaces_, + std::vector<BSONObj> pipeline) + : allowDiskUse(aggRequest.getAllowDiskUse()), + involvedNamespaces(std::move(involvedNamespaces_)), + representativePipeline(std::move(pipeline)) {} + +AggCmdShapeComponents::AggCmdShapeComponents( + OptionalBool allowDiskUse, + stdx::unordered_set<NamespaceString> involvedNamespaces_, + std::vector<BSONObj> pipeline) + : allowDiskUse(allowDiskUse), + involvedNamespaces(std::move(involvedNamespaces_)), + representativePipeline(std::move(pipeline)) {} + +void AggCmdShapeComponents::HashValue(absl::HashState state) const { + state = absl::HashState::combine(std::move(state), allowDiskUse); + for (auto&& shapifiedStage : representativePipeline) { + state = absl::HashState::combine(std::move(state), simpleHash(shapifiedStage)); + } +} + +void AggCmdShape::appendLetCmdSpecificShapeComponents( + BSONObjBuilder& bob, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts) const { + tassert(7633000, + "We don't support serializing to the unmodified shape here, since we have already " + "shapified and stored the representative query - we've lost the original literals", + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged); + + if (opts == SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + // We have this copy stored already! + return _components.appendTo(bob); + } else { + // The cached pipeline shape doesn't match the requested options, so we have to + // re-parse the pipeline from the initial request. + expCtx->inMongos = _inMongos; + expCtx->addResolvedNamespaces(_components.involvedNamespaces); + auto reparsed = Pipeline::parse(_components.representativePipeline, expCtx); + auto serializedPipeline = reparsed->serializeToBson(opts); + AggCmdShapeComponents{ + _components.allowDiskUse, _components.involvedNamespaces, serializedPipeline} + .appendTo(bob); + } +} + +void AggCmdShapeComponents::appendTo(BSONObjBuilder& bob) const { + bob.append("command", "aggregate"); + + // pipeline + bob.append(AggregateCommandRequest::kPipelineFieldName, representativePipeline); + + // allowDiskUse + if (allowDiskUse.has_value()) { + bob.append(AggregateCommandRequest::kAllowDiskUseFieldName, bool(allowDiskUse)); + } +} + +// As part of the size, we must track the allocation of elements in the representative +// pipeline, as well as the elements in the unordered set of involved namespaces. +size_t AggCmdShapeComponents::size() const { + return sizeof(AggCmdShapeComponents) + shape_helpers::containerSize(representativePipeline) + + shape_helpers::containerSize(involvedNamespaces); +} + +AggCmdShape::AggCmdShape(const AggregateCommandRequest& aggregateCommand, + NamespaceString origNss, + stdx::unordered_set<NamespaceString> involvedNamespaces_, + const Pipeline& pipeline, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : CmdWithLetShape(aggregateCommand.getLet(), + expCtx, + _components, + std::move(origNss), + aggregateCommand.getCollation().value_or(BSONObj())), + _components(aggregateCommand, + std::move(involvedNamespaces_), + pipeline.serializeToBson( + SerializationOptions::kRepresentativeQueryShapeSerializeOptions)), + _inMongos(expCtx->inMongos) {} + +size_t AggCmdShape::extraSize() const { + // To account for possible padding, we calculate the extra space with the difference instead of + // using sizeof(bool); + return sizeof(AggCmdShape) - sizeof(CmdWithLetShape) - sizeof(AggCmdShapeComponents); +} + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/agg_cmd_shape.h b/src/mongo/db/query/query_shape/agg_cmd_shape.h new file mode 100644 index 00000000000..c0ef5a7b06f --- /dev/null +++ b/src/mongo/db/query/query_shape/agg_cmd_shape.h @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <boost/intrusive_ptr.hpp> + +#include "mongo/db/pipeline/aggregate_command_gen.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/pipeline.h" +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" +#include "mongo/db/query/query_shape/query_shape.h" + +namespace mongo::query_shape { + +/** + * A struct representing the aggregate command's specific components that are to be considered part + * of the query shape. + * + * This struct stores the shapified version of the pipeline as a memory optimization. We'll need to + * store the BSON version in either case, since often the parsed version needs that BSON to survive + * as backing memory, so we store the representative pipeline shape so that we are able to parse the + * pipeline again if we need to compute a different shape. + */ +struct AggCmdShapeComponents : public query_shape::CmdSpecificShapeComponents { + AggCmdShapeComponents(const AggregateCommandRequest&, + stdx::unordered_set<NamespaceString> involvedNamespaces, + std::vector<BSONObj> shapifiedPipeline); + + AggCmdShapeComponents(OptionalBool allowDiskUse, + stdx::unordered_set<NamespaceString> involvedNamespaces, + std::vector<BSONObj> shapifiedPipeline); + + size_t size() const final; + + void appendTo(BSONObjBuilder&) const; + + void HashValue(absl::HashState state) const final; + + OptionalBool allowDiskUse; + + stdx::unordered_set<NamespaceString> involvedNamespaces; + + // The representative query shape of the pipeline. + std::vector<BSONObj> representativePipeline; +}; + +/** + * A class representing the query shape of an aggregate command. The components are listed above. + * This class knows how to utilize those components to serialize to BSON with any + * SerializationOptions. Mostly this involves correctly setting up an ExpressionContext to re-parse + * the request if needed. + */ +class AggCmdShape : public CmdWithLetShape { +public: + AggCmdShape(const AggregateCommandRequest&, + NamespaceString origNss, + stdx::unordered_set<NamespaceString> involvedNamespaces, + const Pipeline&, + const boost::intrusive_ptr<ExpressionContext>&); + + void appendLetCmdSpecificShapeComponents(BSONObjBuilder& bob, + const boost::intrusive_ptr<ExpressionContext>&, + const SerializationOptions&) const final; + size_t extraSize() const final override; + +private: + AggCmdShapeComponents _components; + // Flag to denote if the query was run on mongos. Needed to rebuild the "dummy" expression + // context for re-parsing. + bool _inMongos; +}; +static_assert(sizeof(AggCmdShape) <= + sizeof(CmdWithLetShape) + sizeof(AggCmdShapeComponents) + 8 /* bool and padding*/, + "If the class' members have changed, this assert and the extraSize() calculation may " + "need to be updated with a new value."); +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp b/src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp new file mode 100644 index 00000000000..c617391f4e7 --- /dev/null +++ b/src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp @@ -0,0 +1,266 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/bson/json.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_shape/agg_cmd_shape.h" +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" +#include "mongo/db/query/query_test_service_context.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_shape { + +namespace { +static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl"); + +class AggCmdShapeTest : public unittest::Test { +public: + void setUp() final { + _queryTestServiceContext = std::make_unique<QueryTestServiceContext>(); + _operationContext = _queryTestServiceContext->makeOperationContext(); + _expCtx = make_intrusive<ExpressionContextForTest>(); + } + + std::unique_ptr<AggregateCommandRequest> makeAggregateCommandRequest( + std::vector<StringData> stagesJson, + boost::optional<StringData> letJson = boost::none, + boost::optional<StringData> collationJson = boost::none) { + std::vector<BSONObj> pipeline; + for (auto&& stage : stagesJson) { + pipeline.push_back(fromjson(stage.rawData())); + } + + auto aggRequest = + std::make_unique<AggregateCommandRequest>(kDefaultTestNss, std::move(pipeline)); + if (letJson) { + aggRequest->setLet(fromjson(letJson->rawData())); + } + if (collationJson) { + aggRequest->setCollation(fromjson(collationJson->rawData())); + } + return aggRequest; + } + + std::unique_ptr<AggCmdShape> makeShapeFromPipeline( + std::vector<StringData> stagesJson, + boost::optional<StringData> letJson = boost::none, + boost::optional<StringData> collationJson = boost::none) { + + auto aggRequest = makeAggregateCommandRequest( + std::move(stagesJson), std::move(letJson), std::move(collationJson)); + + auto parsedPipeline = Pipeline::parse(aggRequest->getPipeline(), _expCtx); + return std::make_unique<AggCmdShape>(*aggRequest, + kDefaultTestNss, + stdx::unordered_set<NamespaceString>{kDefaultTestNss}, + *parsedPipeline, + _expCtx); + } + std::unique_ptr<AggCmdShapeComponents> makeShapeComponentsFromPipeline( + std::vector<StringData> stagesJson, OptionalBool allowDiskUse = {}) { + auto aggRequest = makeAggregateCommandRequest(std::move(stagesJson)); + + auto parsedPipeline = Pipeline::parse(aggRequest->getPipeline(), _expCtx); + return std::make_unique<AggCmdShapeComponents>( + *aggRequest, + stdx::unordered_set<NamespaceString>{kDefaultTestNss}, + parsedPipeline->serializeToBson( + SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + } + + std::unique_ptr<QueryTestServiceContext> _queryTestServiceContext; + + ServiceContext::UniqueOperationContext _operationContext; + boost::intrusive_ptr<ExpressionContext> _expCtx; +}; + +TEST_F(AggCmdShapeTest, BasicPipelineShape) { + auto shape = + makeShapeFromPipeline({R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "cmdNs": { + "db": "testDB", + "coll": "testColl" + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "$and": [ + { + "x": { + "$eq": "?number" + } + }, + { + "y": { + "$lte": "?number" + } + } + ] + } + }, + { + "$group": { + "_id": "$y", + "z": { + "$max": "$z" + }, + "w": { + "$avg": "$w" + } + } + } + ] + })", + shape->toBson(_operationContext.get(), + SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +TEST_F(AggCmdShapeTest, IncludesLet) { + auto shape = makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}, + R"({x: 4, y: "str"})"_sd); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "cmdNs": { + "db": "testDB", + "coll": "testColl" + }, + "let": { + "x": "?number", + "y": "?string" + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "x": { + "$eq": "?number" + } + } + }, + { + "$limit": "?number" + } + ] + })", + shape->toBson(_operationContext.get(), + SerializationOptions::kDebugQueryShapeSerializeOptions)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "cmdNs": { + "db": "testDB", + "coll": "testColl" + }, + "let": { + "x": { + "$const": 1 + }, + "y": { + "$const": "?" + } + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "x": { + "$eq": 1 + } + } + }, + { + "$limit": 1 + } + ] + })", + shape->toBson(_operationContext.get(), + SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); +} + +TEST_F(AggCmdShapeTest, SizeOfAggCmdShapeComponents) { + auto aggComponents = makeShapeComponentsFromPipeline( + {R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}, + false /*allowDiskUse*/); + + // The sizes of any members of AggCmdShapeComponents are typically accounted for by + // sizeof(AggCmdShapeComponents). The important part of the test here is to ensure that any + // additional memory allocations are also included in the size() operation. In our case, + // we expect additional memory use from the representative pipeline and the involved + // namespaces set. + const auto pipelineSize = shape_helpers::containerSize(aggComponents->representativePipeline); + const auto involvedNamespacesSize = sizeof(kDefaultTestNss) + + kDefaultTestNss.size(); // kDefaultTestNss is the only value in the unordered set. + + ASSERT_EQ(aggComponents->size(), + sizeof(AggCmdShapeComponents) + pipelineSize + involvedNamespacesSize); +} + +TEST_F(AggCmdShapeTest, EquivalentAggCmdShapeComponentSizes) { + auto aggComponentsDiskUseFalse = makeShapeComponentsFromPipeline( + {R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}, + false /*allowDiskUse*/); + auto aggComponentsDiskUseTrue = makeShapeComponentsFromPipeline( + {R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}, + true /*allowDiskUse*/); + ASSERT_EQ(aggComponentsDiskUseFalse->size(), aggComponentsDiskUseTrue->size()); +} + +TEST_F(AggCmdShapeTest, DifferentAggCmdShapeComponentSizes) { + auto smallAggComponents = makeShapeComponentsFromPipeline({R"({$match: {x: 3, y: {$lte: 3}}})"}, + false /*allowDiskUse*/); + auto largeAggComponents = makeShapeComponentsFromPipeline( + {R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}, + false /*allowDiskUse*/); + ASSERT_LT(smallAggComponents->size(), largeAggComponents->size()); +} + +TEST_F(AggCmdShapeTest, SizeOfAggCmdShapeWithAndWithoutLet) { + auto shapeWithoutLet = makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}); + auto shapeWithLet = makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}, + R"({x: 4, y: "str"})"_sd); + ASSERT_LT(shapeWithoutLet->size(), shapeWithLet->size()); +} + +TEST_F(AggCmdShapeTest, SizeOfAggCmdShapeWithAndWithoutCollation) { + auto shapeWithoutCollation = + makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}); + auto shapeWithCollation = makeShapeFromPipeline( + {R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}, boost::none, R"({locale: "en_US"})"_sd); + ASSERT_LT(shapeWithoutCollation->size(), shapeWithCollation->size()); +} +} // namespace +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/cmd_with_let_shape.cpp b/src/mongo/db/query/query_shape/cmd_with_let_shape.cpp new file mode 100644 index 00000000000..2bbb6dfeadc --- /dev/null +++ b/src/mongo/db/query/query_shape/cmd_with_let_shape.cpp @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" + +namespace mongo::query_shape { + +namespace { +BSONObj extractLetShape(BSONObj letSpec, + const SerializationOptions& opts, + const boost::intrusive_ptr<ExpressionContext>& expCtx) { + if (letSpec.isEmpty()) { + // Fast path for the common case. + return letSpec; + } + + BSONObjBuilder bob; + for (BSONElement elem : letSpec) { + auto expr = Expression::parseOperand(expCtx.get(), elem, expCtx->variablesParseState); + auto redactedValue = expr->serialize(opts); + // Note that this will throw on deeply nested let variables. + redactedValue.addToBsonObj(&bob, opts.serializeFieldPathFromString(elem.fieldName())); + } + return bob.obj(); +} + +auto representativeLetShape(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx) { + return let ? extractLetShape( + *let, SerializationOptions::kRepresentativeQueryShapeSerializeOptions, expCtx) + : BSONObj(); +} +} // namespace + +LetShapeComponent::LetShapeComponent(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const CmdSpecificShapeComponents& unownedInnerComponents_) + : shapifiedLet(representativeLetShape(let, expCtx)), + hasLet(bool(let)), + unownedInnerComponents(unownedInnerComponents_) {} + +void LetShapeComponent::HashValue(absl::HashState state) const { + state = absl::HashState::combine( + std::move(state), hasLet, simpleHash(shapifiedLet), unownedInnerComponents); +} + +size_t LetShapeComponent::size() const { + return sizeof(LetShapeComponent) + shapifiedLet.objsize() + unownedInnerComponents.size(); +} + +void LetShapeComponent::addLetBson(BSONObjBuilder& bob, + const SerializationOptions& opts, + const boost::intrusive_ptr<ExpressionContext>& expCtx) const { + if (hasLet) { + auto shapeToAppend = shapifiedLet; + if (opts != SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + // We have the representative query cached/stored here, but the caller is asking for a + // different format, so we must re-compute. + shapeToAppend = extractLetShape(shapifiedLet, opts, expCtx); + } + bob.append(FindCommandRequest::kLetFieldName, shapeToAppend); + } +} + +void CmdWithLetShape::appendCmdSpecificShapeComponents(BSONObjBuilder& bob, + OperationContext* opCtx, + const SerializationOptions& opts) const { + auto expCtx = + ExpressionContext::makeBlankExpressionContext(opCtx, nssOrUUID, _let.shapifiedLet); + _let.addLetBson(bob, opts, expCtx); + appendLetCmdSpecificShapeComponents(bob, expCtx, opts); +} + +CmdWithLetShape::CmdWithLetShape(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const CmdSpecificShapeComponents& unownedInnerComponents, + NamespaceStringOrUUID nssOrUUID, + BSONObj collation) + : Shape(nssOrUUID, collation), _let(let, expCtx, unownedInnerComponents) {} + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/cmd_with_let_shape.h b/src/mongo/db/query/query_shape/cmd_with_let_shape.h new file mode 100644 index 00000000000..a1c127b9999 --- /dev/null +++ b/src/mongo/db/query/query_shape/cmd_with_let_shape.h @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/bson/bsonobj.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/query/parsed_find_command.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/serialization_options.h" + +namespace mongo::query_shape { + +/** + * This struct is bit of a weird one. We want to use it as the shape's _entire_ "specific + * components" (rather than introduce more virtual functions to that interface). So, we track here + * the let component (as the name suggests) but we also keep an unowned reference to the specific + * components of CmdWithLetShape sub-classes. This class doesn't really do all that much with those + * components except track a reference to them and ensure their size is accounted for and their hash + * value is incorporated. + */ +struct LetShapeComponent : public CmdSpecificShapeComponents { + LetShapeComponent(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const CmdSpecificShapeComponents& unownedInnerComponents); + + /** + * Hashes to include the shapified let parameters and also the hash of 'unownedInnerComponents'. + */ + void HashValue(absl::HashState state) const final; + + /** + * Includes the size of the let parameters and the size of 'unownedInnerComponents.' + */ + size_t size() const final; + + /** + * Adds _only_ the let params. + */ + void addLetBson(BSONObjBuilder&, + const SerializationOptions&, + const boost::intrusive_ptr<ExpressionContext>&) const; + + BSONObj shapifiedLet; + bool hasLet; + // Tracked so that this can be hash combined correctly. + const CmdSpecificShapeComponents& unownedInnerComponents; +}; + +/** + * The 'let' command argument is semi-generic in that it is supported in a couple commands. However + * it is treated specially since it supports using expressions as the let constants. Using + * expressions induces a library dependency that we don't want in the Shape interface itself. So + * this class handles tracking and adding the 'let' component of the shape for sub-classes. + */ +class CmdWithLetShape : public Shape { +public: + CmdWithLetShape(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const CmdSpecificShapeComponents& unownedInnerComponents, + NamespaceStringOrUUID, + BSONObj collation_); + + const CmdSpecificShapeComponents& specificComponents() const final { + return _let; + } + +protected: + void appendCmdSpecificShapeComponents(BSONObjBuilder&, + OperationContext* opCtx, + const SerializationOptions& opts) const final; + virtual void appendLetCmdSpecificShapeComponents( + BSONObjBuilder&, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions&) const = 0; + + LetShapeComponent _let; +}; +static_assert(sizeof(CmdWithLetShape) == sizeof(Shape) + sizeof(LetShapeComponent), + "If the class' members have changed, this assert and the extraSize() calculation may " + "need to be updated with a new value."); + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp b/src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp new file mode 100644 index 00000000000..21812d97fc7 --- /dev/null +++ b/src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp @@ -0,0 +1,78 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_shape { + +namespace { +static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl"); + +class CmdWithLetShapeTest : public unittest::Test {}; + + +struct DummyInnerComponent : public CmdSpecificShapeComponents { + DummyInnerComponent(){}; + void HashValue(absl::HashState state) const {} + size_t size() const final { + return sizeof(*this); + } +}; + +TEST_F(CmdWithLetShapeTest, SizeOfLetShapeComponent) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto let = fromjson(R"({x: 4, y: "str"})"); + auto innerComponents = std::make_unique<DummyInnerComponent>(); + auto components = std::make_unique<LetShapeComponent>(let, expCtx, *innerComponents); + + const auto minimumSize = sizeof(CmdSpecificShapeComponents) + sizeof(BSONObj) + sizeof(bool) + + sizeof(void*) /*CmdSpecificShapeComponents&*/ + + static_cast<size_t>(components->shapifiedLet.objsize()) + + components->unownedInnerComponents.size(); + + ASSERT_GTE(components->size(), minimumSize); + ASSERT_LTE(components->size(), minimumSize + 8 /*padding*/); +} + +TEST_F(CmdWithLetShapeTest, SizeOfComponentWithAndWithoutLet) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto let = fromjson(R"({x: 4, y: "str"})"); + auto innerComponents = std::make_unique<DummyInnerComponent>(); + auto componentsWithLet = std::make_unique<LetShapeComponent>(let, expCtx, *innerComponents); + auto componentsWithNoLet = + std::make_unique<LetShapeComponent>(boost::none, expCtx, *innerComponents); + + ASSERT_LT(componentsWithNoLet->size(), componentsWithLet->size()); +} + +} // namespace +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/find_cmd_shape.cpp b/src/mongo/db/query/query_shape/find_cmd_shape.cpp new file mode 100644 index 00000000000..2d018de2619 --- /dev/null +++ b/src/mongo/db/query/query_shape/find_cmd_shape.cpp @@ -0,0 +1,227 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/find_cmd_shape.h" + +#include "mongo/db/query/projection_ast_util.h" +#include "mongo/db/query/query_shape/shape_helpers.h" + +namespace mongo::query_shape { +namespace { + +BSONObj projectionShape(const boost::optional<projection_ast::Projection>& proj, + const SerializationOptions& opts = + SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + return proj ? projection_ast::serialize(*proj->root(), opts) : BSONObj(); +} + +BSONObj sortShape(const boost::optional<SortPattern>& sort, + const SerializationOptions& opts = + SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + return sort + ? sort->serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts) + .toBson() + : BSONObj(); +} + +void maybeAddWithName(const OptionalBool& optBool, BSONObjBuilder& bob, StringData name) { + if (optBool.has_value()) { + bob.append(name, bool(optBool)); + } +} + +void addRemainingFindCommandFields(const FindCmdShapeComponents& components, BSONObjBuilder& bob) { + maybeAddWithName(components.singleBatch, bob, FindCommandRequest::kSingleBatchFieldName); + maybeAddWithName(components.allowDiskUse, bob, FindCommandRequest::kAllowDiskUseFieldName); + maybeAddWithName(components.returnKey, bob, FindCommandRequest::kReturnKeyFieldName); + maybeAddWithName(components.showRecordId, bob, FindCommandRequest::kShowRecordIdFieldName); + maybeAddWithName(components.tailable, bob, FindCommandRequest::kTailableFieldName); + maybeAddWithName(components.awaitData, bob, FindCommandRequest::kAwaitDataFieldName); + maybeAddWithName(components.oplogReplay, bob, FindCommandRequest::kOplogReplayFieldName); +} + +} // namespace + +FindCmdShapeComponents::FindCmdShapeComponents( + const ParsedFindCommand& request, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts) + : filter(request.filter->serialize(opts)), + projection(projectionShape(request.proj, opts)), + sort(sortShape(request.sort, opts)), + min(shape_helpers::extractMinOrMaxShape(request.findCommandRequest->getMin(), opts)), + max(shape_helpers::extractMinOrMaxShape(request.findCommandRequest->getMax(), opts)), + singleBatch(request.findCommandRequest->getSingleBatch()), + allowDiskUse(request.findCommandRequest->getAllowDiskUse().has_value() + ? boost::optional<bool>(bool(request.findCommandRequest->getAllowDiskUse())) + : boost::none), + returnKey(request.findCommandRequest->getReturnKey()), + showRecordId(request.findCommandRequest->getShowRecordId()), + tailable(request.findCommandRequest->getTailable()), + awaitData(request.findCommandRequest->getAwaitData()), + oplogReplay(request.findCommandRequest->getOplogReplay()), + hasField(), + serializationOpts(opts) { + hasField.projection = request.proj.has_value(); + hasField.sort = request.sort.has_value(); + hasField.limit = request.findCommandRequest->getLimit().has_value(); + hasField.skip = request.findCommandRequest->getSkip().has_value(); +} + +void FindCmdShapeComponents::appendTo(BSONObjBuilder& bob) const { + + bob.append("command", "find"); + + std::unique_ptr<MatchExpression> filterExpr; + // Filter. + bob.append(FindCommandRequest::kFilterFieldName, filter); + + if (hasField.projection) { + bob.append(FindCommandRequest::kProjectionFieldName, projection); + } + + if (!max.isEmpty()) { + bob.append(FindCommandRequest::kMaxFieldName, max); + } + if (!min.isEmpty()) { + bob.append(FindCommandRequest::kMinFieldName, min); + } + + // Sort. + if (hasField.sort) { + bob.append(FindCommandRequest::kSortFieldName, sort); + } + + // The values here don't matter (assuming we're not using the 'kUnchanged' policy). + tassert(7973601, + "Serialization policy not supported - original values have been discarded", + serializationOpts.literalPolicy != LiteralSerializationPolicy::kUnchanged); + if (hasField.limit) { + serializationOpts.appendLiteral(&bob, FindCommandRequest::kLimitFieldName, 1ll); + } + if (hasField.skip) { + serializationOpts.appendLiteral(&bob, FindCommandRequest::kSkipFieldName, 1ll); + } + + // Add the fields that require no transformation. + addRemainingFindCommandFields(*this, bob); +} + +void FindCmdShapeComponents::HashValue(absl::HashState state) const { + absl::HashState::combine(std::move(state), + simpleHash(filter), + simpleHash(projection), + simpleHash(sort), + simpleHash(min), + simpleHash(max), + singleBatch, + allowDiskUse, + returnKey, + showRecordId, + tailable, + awaitData, + oplogReplay, + hasField); +} + +std::unique_ptr<FindCommandRequest> FindCmdShape::toFindCommandRequest() const { + auto fcr = std::make_unique<FindCommandRequest>(nssOrUUID); + + fcr->setFilter(components.filter); + if (components.hasField.projection) + fcr->setProjection(components.projection); + if (components.hasField.sort) + fcr->setSort(components.sort); + + fcr->setMin(components.min); + fcr->setMax(components.max); + + // Doesn't matter what value to use for limit and skip in the context of a shape. + if (components.hasField.limit) + fcr->setLimit(1ll); + if (components.hasField.skip) + fcr->setSkip(1ll); + + // All the booleans. + if (components.singleBatch.has_value()) + fcr->setSingleBatch(bool(components.singleBatch)); + if (components.allowDiskUse.has_value()) + fcr->setAllowDiskUse(bool(components.allowDiskUse)); + if (components.returnKey.has_value()) + fcr->setReturnKey(bool(components.returnKey)); + if (components.showRecordId.has_value()) + fcr->setShowRecordId(bool(components.showRecordId)); + if (components.tailable.has_value()) + fcr->setTailable(bool(components.tailable)); + if (components.awaitData.has_value()) + fcr->setAwaitData(bool(components.awaitData)); + if (components.oplogReplay.has_value()) + fcr->setOplogReplay(bool(components.oplogReplay)); + + // Common shape components. + if (_let.hasLet) + fcr->setLet(_let.shapifiedLet); + if (!collation.isEmpty()) + fcr->setCollation(collation); + + + return fcr; +} + +FindCmdShape::FindCmdShape(const ParsedFindCommand& findRequest, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : CmdWithLetShape(findRequest.findCommandRequest->getLet(), + expCtx, + components, + findRequest.findCommandRequest->getNamespaceOrUUID(), + findRequest.findCommandRequest->getCollation()), + components(findRequest, expCtx) {} + +void FindCmdShape::appendLetCmdSpecificShapeComponents( + BSONObjBuilder& bob, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts) const { + if (opts == SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + // Fast path: we already have this. + return components.appendTo(bob); + } else { + // Slow path: we need to re-parse from our representative shapes. + auto request = uassertStatusOKWithContext( + parsed_find_command::parse(expCtx, + toFindCommandRequest(), + ExtensionsCallbackNoop(), + MatchExpressionParser::kAllowAllSpecialFeatures), + "Could not re-parse a representative query shape"); + + // This constructor will shapify according to the options. + FindCmdShapeComponents{*request, expCtx, opts}.appendTo(bob); + } +} + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/find_cmd_shape.h b/src/mongo/db/query/query_shape/find_cmd_shape.h new file mode 100644 index 00000000000..49d70b8ec27 --- /dev/null +++ b/src/mongo/db/query/query_shape/find_cmd_shape.h @@ -0,0 +1,130 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/shape_helpers.h" + +namespace mongo::query_shape { + +/** + * This struct tracks the components of a find command which are important for the find query shape. + * It attempts to only track those which are _unique_ to a find command - common elements should go + * on some super class. + * + * Data elements which are shapified like 'filter' are stored in their shapified form. By default + * and in most cases this will be the representative query shape form so that it can be re-parsed, + * but as a convenience for serializing it is also supported to construct and serialize this with + * other options. + */ +struct FindCmdShapeComponents : public CmdSpecificShapeComponents { + + FindCmdShapeComponents(const ParsedFindCommand& request, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts = + SerializationOptions::kRepresentativeQueryShapeSerializeOptions); + + /** + * Appends using the SerializationOptions given in the constructor. + */ + void appendTo(BSONObjBuilder&) const; + + size_t size() const final { + return sizeof(FindCmdShapeComponents) + filter.objsize() + projection.objsize() + + sort.objsize() + min.objsize() + max.objsize(); + } + + BSONObj filter; + BSONObj projection; + BSONObj sort; + BSONObj min; + BSONObj max; + + OptionalBool singleBatch; + OptionalBool allowDiskUse; + OptionalBool returnKey; + OptionalBool showRecordId; + OptionalBool tailable; + OptionalBool awaitData; + OptionalBool oplogReplay; + + // This anonymous struct represents the presence of the member variables as C++ bit fields. + // In doing so, each of these boolean values takes up 1 bit instead of 1 byte. + struct HasField { + HasField() : projection(false), sort(false), limit(false), skip(false) {} + bool projection : 1; + bool sort : 1; + bool limit : 1; + bool skip : 1; + } hasField; + + // We save a copy of the options used when constructed so we know how to properly append things + // like limit and skip - either a 1 or "?number". We could have the caller pass the options + // again during 'appendTo()', but this introduces a risk that the options provided are different + // than the ones we used to compute 'filter' and the other components. + SerializationOptions serializationOpts; + + void HashValue(absl::HashState state) const final; +}; + +class FindCmdShape : public CmdWithLetShape { +public: + FindCmdShape(const ParsedFindCommand& findRequest, + const boost::intrusive_ptr<ExpressionContext>& expCtx); + + /** + * Assembles a parseable FindCommandRequest representing this shape - some of the pieces are + * stored right here in the shape, others are in parent classes. + */ + std::unique_ptr<FindCommandRequest> toFindCommandRequest() const; + + FindCmdShapeComponents components; + +protected: + void appendLetCmdSpecificShapeComponents(BSONObjBuilder& bob, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts) const final; +}; + +template <typename H> +H AbslHashValue(H h, const FindCmdShapeComponents::HasField& hasField) { + return H::combine( + std::move(h), hasField.projection, hasField.sort, hasField.limit, hasField.skip); +} + +// This assertion is still active on the maintained master branch. On the v6.0 branch, we disable it +// since it is not passing on all toolchains/platforms - notably x86 macOS. The intent of the +// assertion is to prevent accidental additions of data members, which should not happen on this +// branch without first happening on the master branch and passing that assertion. +// static_assert(sizeof(FindCmdShape) == sizeof(CmdWithLetShape) + sizeof(FindCmdShapeComponents), +// "If the class' members have changed, this assert and the extraSize() calculation +// may " "need to be updated with a new value."); +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/find_cmd_shape_test.cpp b/src/mongo/db/query/query_shape/find_cmd_shape_test.cpp new file mode 100644 index 00000000000..0d839a5d3d2 --- /dev/null +++ b/src/mongo/db/query/query_shape/find_cmd_shape_test.cpp @@ -0,0 +1,238 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_shape/find_cmd_shape.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_shape { + +namespace { +/** + * TODO this was stolen from another test. Time for a library? + * Simplistic redaction strategy for testing which appends the field name to the prefix "REDACT_". + */ +std::string applyHmacForTest(StringData sd) { + return "REDACT_" + sd.toString(); +} + +static const NamespaceStringOrUUID kDefaultTestNss = + NamespaceStringOrUUID{NamespaceString("testDB.testColl")}; + +struct RequestOptions { + OptionalBool singleBatch = {}; + OptionalBool allowDiskUse = {}; + OptionalBool returnKey = {}; + OptionalBool showRecordId = {}; + OptionalBool tailable = {}; + OptionalBool awaitData = {}; + OptionalBool limit = {}; + OptionalBool skip = {}; +}; +class FindCmdShapeTest : public ServiceContextTest { +public: + void setUp() final { + _expCtx = make_intrusive<ExpressionContextForTest>(); + } + + std::unique_ptr<FindCmdShape> makeShapeFromSort(StringData sortJson) { + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setSort(fromjson(sortJson.rawData())); + auto&& parsedRequest = + uassertStatusOK(::mongo::parsed_find_command::parse(_expCtx, std::move(fcr))); + return std::make_unique<FindCmdShape>(*parsedRequest, _expCtx); + } + + BSONObj sortShape(StringData sortJson) { + auto shape = makeShapeFromSort(sortJson); + return shape->components.sort; + } + + /** + * Returns the shape of the input sort, or boost::none if the input shape was a natural sort + * which got converted into a hint. + */ + boost::optional<BSONObj> maybeRedactedSortShape(StringData sortJson) { + auto shape = makeShapeFromSort(sortJson); + SerializationOptions opts = SerializationOptions::kDebugQueryShapeSerializeOptions; + opts.transformIdentifiers = true; + opts.transformIdentifiersCallback = applyHmacForTest; + auto shapeBson = shape->toBson(_expCtx->opCtx, opts); + if (auto sortElem = shapeBson["sort"]; !sortElem.eoo()) { + return sortElem.Obj().getOwned(); + } + return boost::none; + } + + BSONObj redactedSortShape(StringData sortJson) { + return *maybeRedactedSortShape(sortJson); + } + + boost::intrusive_ptr<ExpressionContext> _expCtx; + + std::unique_ptr<FindCmdShapeComponents> makeShapeComponentsFromFilter( + BSONObj filter, const RequestOptions& requestOptions = {}) { + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + fcr->setSingleBatch(requestOptions.singleBatch); + fcr->setAllowDiskUse(requestOptions.allowDiskUse); + fcr->setReturnKey(requestOptions.returnKey); + fcr->setAllowDiskUse(requestOptions.showRecordId); + fcr->setTailable(requestOptions.tailable); + fcr->setAwaitData(requestOptions.awaitData); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)})); + return std::make_unique<FindCmdShapeComponents>(*parsedFind, _expCtx); + } + + std::unique_ptr<FindCmdShape> makeShapeFromFilter(const BSONObj& filter) { + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)})); + return std::make_unique<FindCmdShape>(*parsedFind, _expCtx); + } +}; + +TEST_F(FindCmdShapeTest, NormalSortPattern) { + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"a.b.c":1,"foo":-1})", + sortShape(R"({"a.b.c": 1, "foo": -1})")); +} + +TEST_F(FindCmdShapeTest, NaturalSortPattern) { + // $natural sorts are interpreted as a hint. Hints are not part of the shape (but should show up + // in the query stats key). + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({})", + sortShape(R"({$natural: 1})")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({})", + sortShape(R"({$natural: -1})")); +} + +TEST_F(FindCmdShapeTest, NaturalSortPatternWithMeta) { + ASSERT_THROWS_CODE( + sortShape(R"({$natural: 1, x: {$meta: "textScore"}})"), DBException, ErrorCodes::BadValue); +} + +TEST_F(FindCmdShapeTest, MetaPatternWithoutNatural) { + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"normal":1,"$computed1":{"$meta":"textScore"}})", + sortShape(R"({normal: 1, x: {$meta: "textScore"}})")); +} + +// Here we have one test to ensure that the redaction policy is accepted and applied in the +// query_shape utility, but there are more extensive redaction tests in sort_pattern_test.cpp +TEST_F(FindCmdShapeTest, RespectsRedactionPolicy) { + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"REDACT_normal":1,"REDACT_y":1})", + redactedSortShape(R"({normal: 1, y: 1})")); + + // No need to redact $natural. Again, this will be interpreted as a hint, but this test is + // interesting to ensure the $-prefix of $natural doesn't confuse us. + ASSERT(!maybeRedactedSortShape(R"({$natural: 1})")); +} + +TEST_F(FindCmdShapeTest, SizeOfShapeComponents) { + auto query = BSON("query" << 1 << "xEquals" << 42); + auto findCmdComponent = makeShapeComponentsFromFilter(query.getOwned()); + const auto querySize = findCmdComponent->filter.objsize(); + + const auto minimumSize = sizeof(FindCmdShapeComponents) + querySize; + ASSERT_GT(findCmdComponent->size(), minimumSize); + ASSERT_LTE(findCmdComponent->size(), + minimumSize + static_cast<size_t>(4 * BSONObj().objsize())); +} + +TEST_F(FindCmdShapeTest, EquivalentShapeComponentsSizes) { + auto query = BSON("query" << 1 << "xEquals" << 42); + // Tailable can not be set together with 'singleBatch' option. + auto mostlyTrueComponent = makeShapeComponentsFromFilter(query.getOwned(), + {/* singleBatch = */ false, + /* allowDiskUse = */ true, + /* returnKey = */ true, + /* showRecordId = */ true, + /* tailable = */ true, + /* awaitData = */ true, + /* limit = */ true, + /* skip = */ true}); + + auto mostlyFalseComponent = makeShapeComponentsFromFilter(query.getOwned(), + {/* singleBatch = */ false, + /* allowDiskUse = */ false, + /* returnKey = */ false, + /* showRecordId = */ false, + /* tailable = */ true, + /* awaitData = */ false, + /* limit = */ false, + /* skip = */ false}); + + ASSERT_EQ(mostlyTrueComponent->size(), mostlyFalseComponent->size()); +} + +TEST_F(FindCmdShapeTest, DifferentShapeComponentsSizes) { + auto smallQuery = BSON("query" << BSONObj()); + auto smallFindCmdComponent = makeShapeComponentsFromFilter(smallQuery.getOwned()); + + auto largeQuery = BSON("query" << 1 << "xEquals" << 42); + auto largeFindCmdComponent = makeShapeComponentsFromFilter(largeQuery.getOwned()); + + ASSERT_LT(smallQuery.objsize(), largeQuery.objsize()); + ASSERT_LT(smallFindCmdComponent->size(), largeFindCmdComponent->size()); +} + +TEST_F(FindCmdShapeTest, SizeOfShapeWithAndWithoutLet) { + auto filter = BSON("query" << 1 << "xEquals" << 42); + auto shapeWithoutLet = makeShapeFromFilter(filter.getOwned()); + + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + fcr->setLet(fromjson(R"({x: 4})")); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)})); + auto shapeWithLet = std::make_unique<FindCmdShape>(*parsedFind, _expCtx); + + ASSERT_LT(shapeWithoutLet->size(), shapeWithLet->size()); +} + +TEST_F(FindCmdShapeTest, SizeOfShapeWithAndWithoutCollation) { + auto filter = BSON("query" << 1 << "xEquals" << 42); + auto shapeWithoutCollation = makeShapeFromFilter(filter.getOwned()); + + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + fcr->setCollation(fromjson(R"({locale: "en_US"})")); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)})); + auto shapeWithCollation = std::make_unique<FindCmdShape>(*parsedFind, _expCtx); + + ASSERT_LT(shapeWithoutCollation->size(), shapeWithCollation->size()); +} + +} // namespace + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/query_shape.cpp b/src/mongo/db/query/query_shape/query_shape.cpp new file mode 100644 index 00000000000..2fa0520120e --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape.cpp @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/query_shape.h" + +#include "mongo/base/status.h" +#include "mongo/crypto/sha256_block.h" +#include "mongo/db/query/find_command_gen.h" +#include "mongo/db/query/projection_ast_util.h" +#include "mongo/db/query/query_request_helper.h" +#include "mongo/db/query/query_shape/agg_cmd_shape.h" +#include "mongo/db/query/query_shape/find_cmd_shape.h" +#include "mongo/db/query/query_shape/query_shape_gen.h" +#include "mongo/db/query/query_shape/shape_helpers.h" +#include "mongo/db/query/sort_pattern.h" + +namespace mongo::query_shape { + +namespace { +void appendCmdNs(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts) { + BSONObjBuilder nsObj = bob.subobjStart("cmdNs"); + shape_helpers::appendNamespaceShape(nsObj, nss, opts); + nsObj.doneFast(); +} +} // namespace + +Shape::Shape(NamespaceStringOrUUID nssOrUUID_, BSONObj collation_) + : nssOrUUID(nssOrUUID_), collation(std::move(collation_)) {} + + +BSONObj Shape::toBson(OperationContext* opCtx, const SerializationOptions& opts) const { + BSONObjBuilder bob; + appendCmdNsOrUUID(bob, opts); + if (!collation.isEmpty()) { + // Collation is never shapified. We use find command's collation name definition, but it + // should be the same for all requests. + bob.append(FindCommandRequest::kCollationFieldName, collation); + } + appendCmdSpecificShapeComponents(bob, opCtx, opts); + return bob.obj(); +} + +size_t Shape::size() const { + return sizeof(Shape) + shape_helpers::optionalObjSize(collation) + specificComponents().size() + + extraSize(); +} + +QueryShapeHash Shape::sha256Hash(OperationContext* opCtx) const { + // The Query Shape Hash should use the representative query shape. + auto serialized = + toBson(opCtx, SerializationOptions::kRepresentativeQueryShapeSerializeOptions); + return SHA256Block::computeHash((const uint8_t*)serialized.sharedBuffer().get(), + serialized.objsize()); +} + +void Shape::appendCmdNsOrUUID(BSONObjBuilder& bob, const SerializationOptions& opts) const { + if (nssOrUUID.nss()) { + appendCmdNs(bob, *nssOrUUID.nss(), opts); + } else { + BSONObjBuilder cmdNs = bob.subobjStart("cmdNs"); + cmdNs.append("uuid", opts.serializeIdentifier(nssOrUUID.uuid()->toString())); + cmdNs.append("db", opts.serializeIdentifier(nssOrUUID.db())); + cmdNs.doneFast(); + } +} + +void Shape::appendCmdNs(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts) const { + BSONObjBuilder nsObj = bob.subobjStart("cmdNs"); + shape_helpers::appendNamespaceShape(nsObj, nss, opts); + nsObj.doneFast(); +} + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/query_shape.h b/src/mongo/db/query/query_shape/query_shape.h new file mode 100644 index 00000000000..dc83cbab127 --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape.h @@ -0,0 +1,165 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/matcher/expression.h" +#include "mongo/db/pipeline/aggregate_command_gen.h" +#include "mongo/db/query/find_command_gen.h" +#include "mongo/db/query/query_request_helper.h" +#include "mongo/db/query/query_shape/shape_helpers.h" + +namespace mongo::query_shape { + +/** + * Each type of "query" command likely has different fields/options that are considered important + * for the shape. For example, a find command has a skip and a limit, and an aggregate command has a + * pipeline. This interface is used to allow different sub-commands to diverge in this way but still + * ensure we can appropriately hash them to compare their shapes, and properly account for their + * size. + * + * This struct is split out as a separate inheritence hierarchy from 'Shape' to make it easier to + * ensure each piece is hashed without sub-classes needing to enumerate the parent class's member + * variables. + */ +struct CmdSpecificShapeComponents { + virtual ~CmdSpecificShapeComponents() {} + + /** + * Sub-classes should implement this in a way which includes all shape-relevant state. If two + * shapes should compare equal, they should result in the same hash value. For example for the + * find command - we would include the _shapified_ filter and projection here, but we will not + * include the comment - which is not part of the shape. + */ + virtual void HashValue(absl::HashState state) const = 0; + + /** + * It is important for shape components to accurately report their size, and to make a + * reasonable effort to maintain a minimal size. We use the query shape in memory-constrained + * data structures, so a bigger shape means we can have fewer different shapes stored (for + * example in the query stats store). + * + * We cannot just use sizeof() because there are some variable size data members (like BSON + * objects) which depend on the particular instance. + */ + virtual size_t size() const = 0; + + // Some template boilerplate to allow sub-classes to overload the hash implementation. + template <typename H> + friend H AbslHashValue(H state, const CmdSpecificShapeComponents& value) { + value.HashValue(absl::HashState::Create(&state)); + return std::move(state); + } +}; + +using QueryShapeHash = SHA256Block; + +/** + * A query "shape" is a version of a command with literal values abstracted so that two instances of + * the command may compare/hash equal even if they use slightly different literal values. This + * concept exists not just the find command, but planned for many of the CRUD commands + aggregate. + * It also includes most (but not all) components of these commands, not just the query predicate + * (MatchExpresssion). In these ways, "query" is meant more generally. + * + * A "Query Shape" can vary depending on the command (e.g. find, aggregate, or distinct). This + * abstract struct is the API we must implement for each command which we want to have a "shape" + * concept. + * + * In order to properly account for the size of a query shape, the CmdSpecificShapeComponents should + * include all meaningful memory consumption, and be sure to report it in 'size()'. Subclasses of + * 'Shape' are not expected to have any meaningful memory usage outside of that struct. + */ +class Shape { +public: + virtual ~Shape() {} + + /** + * Sub-classes are expected to implement this as a mechanism for plugging in their command + * specific shape components. + */ + virtual const CmdSpecificShapeComponents& specificComponents() const = 0; + + /** + * Note this may involve re-parsing command BSON and so is not necessarily cheap. + */ + BSONObj toBson(OperationContext*, const SerializationOptions&) const; + + /** + * The Query Shape Hash is defined to be the SHA256 Hash of the representatice query shape. This + * helper computes that. + */ + QueryShapeHash sha256Hash(OperationContext*) const; + + /** + * The size of a query shape is important, since we store these in space-constrained + * environments like the query stats store. + */ + size_t size() const; + + /** + * This should be overriden by a child class if it has members whose sizes are not included in + * specificComponents().size(). + */ + virtual size_t extraSize() const { + return 0; + } + template <typename H> + friend H AbslHashValue(H h, const Shape& shape) { + h = H::combine(std::move(h), shape.nssOrUUID, shape.specificComponents()); + if (!shape.collation.isEmpty()) + h = H::combine(std::move(h), simpleHash(shape.collation)); + return h; + } + + + // Not shapified but it is an identifier so it may be transformed. + NamespaceStringOrUUID nssOrUUID; + + // Never shapified. If it's empty, leave it off. + BSONObj collation; + +protected: + Shape(NamespaceStringOrUUID, BSONObj collation_); + + /** + * Along with the hash implementation, this is the main way that shapes are 'shapified' - + * sub-classes should implement this to add the shapified versions of their literals to an + * object. Depending on 'opts', this may be eligible to be used for output in $queryStats or as + * the object to compute the QueryShapeHash. + */ + virtual void appendCmdSpecificShapeComponents(BSONObjBuilder&, + OperationContext*, + const SerializationOptions& opts) const = 0; + +private: + void appendCmdNsOrUUID(BSONObjBuilder&, const SerializationOptions&) const; + void appendCmdNs(BSONObjBuilder&, const NamespaceString&, const SerializationOptions&) const; +}; + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/query_shape.idl b/src/mongo/db/query/query_shape/query_shape.idl new file mode 100644 index 00000000000..77e71756467 --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape.idl @@ -0,0 +1,50 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. + +global: + cpp_namespace: "mongo::query_shape" + +imports: + - "mongo/idl/basic_types.idl" + + +structs: + CommandNamespace: + description: "Representation of the cmdNs sub-object of the query shape." + fields: + db: + type: string + coll: + type: string + optional: true + uuid: + type: string + optional: true + tenantId: + type: string + optional: true +
\ No newline at end of file diff --git a/src/mongo/db/query/query_shape/query_shape_test.cpp b/src/mongo/db/query/query_shape/query_shape_test.cpp new file mode 100644 index 00000000000..d6185b5c5cb --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape_test.cpp @@ -0,0 +1,767 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/db/matcher/expression_geo.h" +#include "mongo/db/matcher/extensions_callback_real.h" +#include "mongo/db/matcher/parsed_match_expression_for_test.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/query_shape_test_gen.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/db/query/query_shape/shape_helpers.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/unittest/bson_test_util.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_shape { + +namespace { +BSONObj predicateShape(const MatchExpression* expr) { + return expr->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions); +} +BSONObj predicateShape(std::string filterJson) { + return predicateShape(ParsedMatchExpressionForTest(filterJson).get()); +} + +BSONObj predicateShapeRedacted(const MatchExpression* expr) { + return expr->serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST); +} +BSONObj predicateShapeRedacted(std::string filterJson) { + return predicateShapeRedacted(ParsedMatchExpressionForTest(filterJson).get()); +} + +// TODO SERVER-87736 There is no 'auto' here, make that more clear. +#define ASSERT_SHAPE_EQ_AUTO(expected, actual) \ + ASSERT_BSONOBJ_EQ_AUTO(expected, predicateShape(actual)) + +#define ASSERT_REDACTED_SHAPE_EQ_AUTO(expected, actual) \ + ASSERT_BSONOBJ_EQ_AUTO(expected, predicateShapeRedacted(actual)) + + +TEST(QueryPredicateShape, Equals) { + ASSERT_SHAPE_EQ_AUTO( // Implicit equals + R"({"a":{"$eq":"?number"}})", + "{a: 5}"); + ASSERT_SHAPE_EQ_AUTO( // Explicit equals + R"({"a":{"$eq":"?number"}})", + "{a: {$eq: 5}}"); + ASSERT_SHAPE_EQ_AUTO( // implicit $and + R"({"$and":[{"a":{"$eq":"?number"}},{"b":{"$eq":"?number"}}]})", + "{a: 5, b: 6}"); + ASSERT_REDACTED_SHAPE_EQ_AUTO( // Implicit equals + R"({"HASH<a>":{"$eq":"?number"}})", + "{a: 5}"); + ASSERT_REDACTED_SHAPE_EQ_AUTO( // Explicit equals + R"({"HASH<a>":{"$eq":"?number"}})", + "{a: {$eq: 5}}"); + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"$and":[{"HASH<a>":{"$eq":"?number"}},{"HASH<b>":{"$eq":"?number"}}]})", + "{a: 5, b: 6}"); + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"HASH<foo>.HASH<$bar>":{"$eq":"?number"}})", + R"({"foo.$bar":0})"); +} + +TEST(QueryPredicateShape, ArraySubTypes) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + "{a: {$eq: '[]'}}", + "{a: []}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + "{a: {$eq: '?array<?number>'}}", + "{a: [2]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?number>"}})", + "{a: [2, 3]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?object>"}})", + "{a: [{}]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?object>"}})", + "{a: [{}, {}]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?array>"}})", + "{a: [[], [], []]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?array>"}})", + "{a: [[2, 3], ['string'], []]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<>"}})", + "{a: [{}, 2]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<>"}})", + "{a: [[], 2]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<>"}})", + "{a: [[{}, 'string'], 2]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<>"}})", + "{a: [[{}, 'string'], 2]}"); +} + +TEST(QueryPredicateShape, Comparisons) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$lt": "?number" + } + }, + { + "b": { + "$gt": "?number" + } + }, + { + "c": { + "$gte": "?number" + } + }, + { + "c": { + "$lte": "?number" + } + } + ] + })", + "{a: {$lt: 5}, b: {$gt: 6}, c: {$gte: 3, $lte: 10}}"); +} + +namespace { +void assertShapeIs(std::string filterJson, BSONObj expectedShape) { + ASSERT_BSONOBJ_EQ(expectedShape, predicateShape(filterJson)); +} + +void assertRedactedShapeIs(std::string filterJson, BSONObj expectedShape) { + ASSERT_BSONOBJ_EQ(expectedShape, predicateShapeRedacted(filterJson)); +} +} // namespace + +TEST(QueryPredicateShape, Regex) { + // Note/warning: 'fromjson' will parse $regex into a /regex/, so these tests can't use + // auto-updating BSON assertions. + assertShapeIs("{a: /a+/}", + BSON("a" << BSON("$regex" + << "?string"))); + assertShapeIs("{a: /a+/i}", + BSON("a" << BSON("$regex" + << "?string" + << "$options" + << "?string"))); + assertRedactedShapeIs("{a: /a+/}", + BSON("HASH<a>" << BSON("$regex" + << "?string"))); + assertRedactedShapeIs("{a: /a+/}", + BSON("HASH<a>" << BSON("$regex" + << "?string"))); +} + +TEST(QueryPredicateShape, Mod) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$mod":["?number","?number"]}})", + "{a: {$mod: [2, 0]}}"); +} + +TEST(QueryPredicateShape, Exists) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$exists":"?bool"}})", + "{a: {$exists: true}}"); +} + +TEST(QueryPredicateShape, In) { + // Any number of children in any order is always the same shape + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$in":"?array<?number>"}})", + "{a: {$in: [1]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$in":"?array<>"}})", + "{a: {$in: [1, 4, 'str', /regex/]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$in":"?array<>"}})", + "{a: {$in: ['str', /regex/, 1, 4]}}"); +} + +TEST(QueryPredicateShape, BitTestOperators) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAllSet":"?array<?number>"}})", + "{a: {$bitsAllSet: [1, 5]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAllSet":"?array<?number>"}})", + "{a: {$bitsAllSet: 50}}"); + + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAnySet":"?array<?number>"}})", + "{a: {$bitsAnySet: [1, 5]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAnySet":"?array<?number>"}})", + "{a: {$bitsAnySet: 50}}"); + + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAllClear":"?array<?number>"}})", + "{a: {$bitsAllClear: [1, 5]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAllClear":"?array<?number>"}})", + "{a: {$bitsAllClear: 50}}"); + + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAnyClear":"?array<?number>"}})", + "{a: {$bitsAnyClear: [1, 5]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAnyClear":"?array<?number>"}})", + "{a: {$bitsAnyClear: 50}}"); +} + +TEST(QueryPredicateShape, AlwaysBoolean) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"$alwaysTrue":"?number"})", + "{$alwaysTrue: 1}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"$alwaysFalse":"?number"})", + "{$alwaysFalse: 1}"); +} + +TEST(QueryPredicateShape, And) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$lt": "?number" + } + }, + { + "b": { + "$gte": "?number" + } + }, + { + "c": { + "$lte": "?number" + } + } + ] + })", + "{$and: [{a: {$lt: 5}}, {b: {$gte: 3}}, {c: {$lte: 10}}]}"); +} + +TEST(QueryPredicateShape, Or) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({ + "$or": [ + { + "a": { + "$eq": "?number" + } + }, + { + "b": { + "$in": "?array<?number>" + } + }, + { + "c": { + "$gt": "?number" + } + } + ] + })", + "{$or: [{a: 5}, {b: {$in: [1,2,3]}}, {c: {$gt: 10}}]}"); +} + +TEST(QueryPredicateShape, ElemMatch) { + // ElemMatchObjectMatchExpression + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({ + "a": { + "$elemMatch": { + "$and": [ + { + "b": { + "$eq": "?number" + } + }, + { + "c": { + "$exists": "?bool" + } + } + ] + } + } + })", + "{a: {$elemMatch: {b: 5, c: {$exists: true}}}}"); + + // ElemMatchValueMatchExpression + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$elemMatch":{"$gt":"?number","$lt":"?number"}}})", + "{a: {$elemMatch: {$gt: 5, $lt: 10}}}"); + + // Nested + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({ + "HASH<a>": { + "$elemMatch": { + "$elemMatch": { + "$gt": "?number", + "$lt": "?number" + } + } + } + })", + "{a: {$elemMatch: {$elemMatch: {$gt: 5, $lt: 10}}}}"); +} + +TEST(QueryPredicateShape, InternalBucketGeoWithinMatchExpression) { + auto query = + "{ $_internalBucketGeoWithin: {withinRegion: {$centerSphere: [[0, 0], 10]}, field: " + "\"a\"} " + "}"; + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({ + "$_internalBucketGeoWithin": { + "withinRegion": { + "$centerSphere": "?array<>" + }, + "field": "HASH<a>" + } + })", + query); +} + +TEST(QueryPredicateShape, NorMatchExpression) { + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"$nor":[{"HASH<a>":{"$lt":"?number"}},{"HASH<b>":{"$gt":"?number"}}]})", + "{ $nor: [ { a: {$lt: 5} }, { b: {$gt: 4} } ] }"); +} + +TEST(QueryPredicateShape, NotMatchExpression) { + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"HASH<price>":{"$not":{"$gt":"?number"}}})", + "{ price: { $not: { $gt: 1.99 } } }"); + // Test the special case where NotMatchExpression::serialize() reduces to $alwaysFalse. + auto emptyAnd = std::make_unique<AndMatchExpression>(); + const MatchExpression& notExpr = NotMatchExpression(std::move(emptyAnd)); + auto serialized = + notExpr.serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$alwaysFalse":"?number"})", + serialized); +} + +TEST(QueryPredicateShape, SizeMatchExpression) { + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"HASH<price>":{"$size":"?number"}})", + "{ price: { $size: 2 } }"); +} + +TEST(QueryPredicateShape, TextMatchExpression) { + TextMatchExpressionBase::TextParams params = {"coffee"}; + auto expr = ExtensionsCallbackNoop().createText(params); + auto literalAndFieldRedactOpts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$text": { + "$search": "?string", + "$language": "?string", + "$caseSensitive": "?bool", + "$diacriticSensitive": "?bool" + } + })", + expr->serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST)); +} + +TEST(QueryPredicateShape, TwoDPtInAnnulusExpression) { + const MatchExpression& expr = TwoDPtInAnnulusExpression({}, {}); + auto literalAndFieldRedactOpts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$TwoDPtInAnnulusExpression":true})", + expr.serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST)); +} + +TEST(QueryPredicateShape, WhereMatchExpression) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"$where":"?javascript"})", + "{$where: \"some_code()\"}"); +} + +BSONObj queryShapeForOptimizedExprExpression(std::string exprPredicateJson) { + ParsedMatchExpressionForTest expr(exprPredicateJson); + // We need to optimize an $expr expression in order to generate an $_internalExprEq. It's + // not clear we'd want to do optimization before computing the query shape, but we should + // support the computation on any MatchExpression, and this is the easiest way we can create + // this type of MatchExpression node. + auto optimized = MatchExpression::optimize(expr.release()); + return predicateShape(optimized.get()); +} + +TEST(QueryPredicateShape, OptimizedExprPredicates) { + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprEq": "?number" + } + }, + { + "$expr": { + "$eq": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$eq: ['$a', 2]}}")); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprLt": "?number" + } + }, + { + "$expr": { + "$lt": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$lt: ['$a', 2]}}")); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprLte": "?number" + } + }, + { + "$expr": { + "$lte": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$lte: ['$a', 2]}}")); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprGt": "?number" + } + }, + { + "$expr": { + "$gt": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$gt: ['$a', 2]}}")); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprGte": "?number" + } + }, + { + "$expr": { + "$gte": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$gte: ['$a', 2]}}")); +} + +TEST(QueryShapeIDL, ShapifyIDLStruct) { + SerializationOptions options; + options.transformIdentifiers = true; + options.transformIdentifiersCallback = [](StringData s) -> std::string { + return str::stream() << "HASH<" << s << ">"; + }; + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + + auto nested = NestedStruct("value", + ExampleEnumEnum::Value1, + "hello", + {1, 2, 3, 4}, + "field.path", + {"field.path.1", "fieldpath2"}, + NamespaceString{"db", "coll"}, + NamespaceString{"db", "coll"}, + 177, + true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "stringField": "value", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": [ + 1, + 2, + 3, + 4 + ], + "fieldpath": "field.path", + "fieldpathList": [ + "field.path.1", + "fieldpath2" + ], + "nss": "db.coll", + "plainNss": "db.coll", + "safeInt64Field": 177, + "boolField": true + })", + nested.toBSON()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "stringField": "?string", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": "?array<?number>", + "fieldpath": "HASH<field>.HASH<path>", + "fieldpathList": [ + "HASH<field>.HASH<path>.HASH<1>", + "HASH<fieldpath2>" + ], + "nss": "HASH<db.coll>", + "plainNss": "db.coll", + "safeInt64Field": "?number", + "boolField": "?bool" + })", + nested.toBSON(options)); + + + auto parent = ParentStruct(nested, nested); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "nested_shape": { + "stringField": "value", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": [ + 1, + 2, + 3, + 4 + ], + "fieldpath": "field.path", + "fieldpathList": [ + "field.path.1", + "fieldpath2" + ], + "nss": "db.coll", + "plainNss": "db.coll", + "safeInt64Field": 177, + "boolField": true + }, + "nested_no_shape": { + "stringField": "value", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": [ + 1, + 2, + 3, + 4 + ], + "fieldpath": "field.path", + "fieldpathList": [ + "field.path.1", + "fieldpath2" + ], + "nss": "db.coll", + "plainNss": "db.coll", + "safeInt64Field": 177, + "boolField": true + } + })", + parent.toBSON()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "nested_shape": { + "stringField": "?string", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": "?array<?number>", + "fieldpath": "HASH<field>.HASH<path>", + "fieldpathList": [ + "HASH<field>.HASH<path>.HASH<1>", + "HASH<fieldpath2>" + ], + "nss": "HASH<db.coll>", + "plainNss": "db.coll", + "safeInt64Field": "?number", + "boolField": "?bool" + }, + "nested_no_shape": { + "stringField": "value", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": [ + 1, + 2, + 3, + 4 + ], + "fieldpath": "field.path", + "fieldpathList": [ + "field.path.1", + "fieldpath2" + ], + "nss": "db.coll", + "plainNss": "db.coll", + "safeInt64Field": 177, + "boolField": true + } + })", + parent.toBSON(options)); +} + +} // namespace + +namespace { + +static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl"); + +struct DummyShapeSpecificComponents : public query_shape::CmdSpecificShapeComponents { + DummyShapeSpecificComponents(){}; + void HashValue(absl::HashState state) const {} + size_t size() const final { + return sizeof(DummyShapeSpecificComponents); + } +}; + +class DummyShape : public Shape { +public: + DummyShape(NamespaceStringOrUUID nssOrUUID, + BSONObj collation, + DummyShapeSpecificComponents dummyComponents) + : Shape(nssOrUUID, collation) { + components = dummyComponents; + } + + const CmdSpecificShapeComponents& specificComponents() const final { + return components; + } + + void appendCmdSpecificShapeComponents(BSONObjBuilder&, + OperationContext*, + const SerializationOptions& opts) const final {} + DummyShapeSpecificComponents components; +}; + +class DummyShapeWithExtraSize : public Shape { +public: + DummyShapeWithExtraSize(NamespaceStringOrUUID nssOrUUID, + BSONObj collation, + DummyShapeSpecificComponents dummyComponents) + : Shape(nssOrUUID, collation) { + components = dummyComponents; + } + + const CmdSpecificShapeComponents& specificComponents() const final { + return components; + } + + // Random number for testing purposes. + size_t extraSize() const final override { + return 125; + } + void appendCmdSpecificShapeComponents(BSONObjBuilder&, + OperationContext*, + const SerializationOptions& opts) const final {} + + DummyShapeSpecificComponents components; +}; + +class UniversalShapeTest : public ServiceContextTest {}; + +TEST_F(UniversalShapeTest, SizeOfSpecificComponents) { + auto innerComponents = std::make_unique<DummyShapeSpecificComponents>(); + ASSERT_EQ(innerComponents->size(), sizeof(CmdSpecificShapeComponents)); + ASSERT_EQ(innerComponents->size(), sizeof(void*) /*vtable ptr*/); +} + +TEST_F(UniversalShapeTest, SizeOfShape) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + // Make shape for testing. + auto collation = BSONObj{}; + auto innerComponents = std::make_unique<DummyShapeSpecificComponents>(); + auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents); + + ASSERT_EQ(innerComponents->size(), shape->specificComponents().size()); + ASSERT_EQ(shape->size(), + sizeof(NamespaceStringOrUUID) + sizeof(BSONObj) + sizeof(void*) /*vtable ptr*/ + + shape->specificComponents().size() + static_cast<size_t>(collation.objsize())); +} + +TEST_F(UniversalShapeTest, SizeOfShapeWithExtraSize) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + // Make shape for testing. + auto collation = BSONObj{}; + auto innerComponents = std::make_unique<DummyShapeSpecificComponents>(); + auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents); + auto shapeWithExtraSize = + std::make_unique<DummyShapeWithExtraSize>(kDefaultTestNss, collation, *innerComponents); + + ASSERT_EQ(shapeWithExtraSize->size(), shape->size() + shapeWithExtraSize->extraSize()); +} +} // namespace +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/query_shape_test.idl b/src/mongo/db/query/query_shape/query_shape_test.idl new file mode 100644 index 00000000000..06efb7ed1ef --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape_test.idl @@ -0,0 +1,91 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/idl/basic_types.idl" + +enums: + ExampleEnum: + description: "" + type: string + values: + Value1: "EnumValue1" + Value2: "EnumValue2" + +structs: + NestedStruct: + query_shape_component: true + strict: true + description: "" + fields: + stringField: + query_shape: literal + type: string + enumField: + query_shape: parameter + type: ExampleEnum + stringIntVariantEnum: + query_shape: parameter + type: + variant: [string, int] + arrayOfInts: + query_shape: literal + type: array<int> + fieldpath: + query_shape: anonymize + type: string + fieldpathList: + query_shape: anonymize + type: array<string> + nss: + query_shape: custom + type: namespacestring + plainNss: + query_shape: parameter + type: namespacestring + safeInt64Field: + query_shape: literal + type: safeInt64 + boolField: + query_shape: literal + type: bool + + ParentStruct: + query_shape_component: true + strict: true + description: "" + fields: + nested_shape: + query_shape: literal + type: NestedStruct + nested_no_shape: + query_shape: parameter + type: NestedStruct diff --git a/src/mongo/db/query/query_shape/serialization_options.cpp b/src/mongo/db/query/query_shape/serialization_options.cpp new file mode 100644 index 00000000000..e6008f8579b --- /dev/null +++ b/src/mongo/db/query/query_shape/serialization_options.cpp @@ -0,0 +1,515 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "serialization_options.h" +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + +#include "mongo/db/query/query_shape/serialization_options.h" + +#include <boost/optional.hpp> +#include <string> + +#include "mongo/base/string_data.h" +#include "mongo/bson/timestamp.h" +#include "mongo/db/exec/document_value/document.h" +#include "mongo/db/exec/document_value/value.h" +#include "mongo/db/query/explain_options.h" +#include "mongo/logv2/log.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + +namespace { + +// We'll pre-declare all of these strings so that we can avoid the allocations when we reference +// them later. +static constexpr StringData kUndefinedTypeString = "?undefined"_sd; +static constexpr StringData kStringTypeString = "?string"_sd; +static constexpr StringData kNumberTypeString = "?number"_sd; +static constexpr StringData kMinKeyTypeString = "?minKey"_sd; +static constexpr StringData kObjectTypeString = "?object"_sd; +static constexpr StringData kArrayTypeString = "?array"_sd; +static constexpr StringData kBinDataTypeString = "?binData"_sd; +static constexpr StringData kObjectIdTypeString = "?objectId"_sd; +static constexpr StringData kBoolTypeString = "?bool"_sd; +static constexpr StringData kDateTypeString = "?date"_sd; +static constexpr StringData kNullTypeString = "?null"_sd; +static constexpr StringData kRegexTypeString = "?regex"_sd; +static constexpr StringData kDbPointerTypeString = "?dbPointer"_sd; +static constexpr StringData kJavascriptTypeString = "?javascript"_sd; +static constexpr StringData kJavascriptWithScopeTypeString = "?javascriptWithScope"_sd; +static constexpr StringData kTimestampTypeString = "?timestamp"_sd; +static constexpr StringData kMaxKeyTypeString = "?maxKey"_sd; + +static const StringMap<StringData> kArrayTypeStringConstants{ + {kUndefinedTypeString.rawData(), "?array<?undefined>"_sd}, + {kStringTypeString.rawData(), "?array<?string>"_sd}, + {kNumberTypeString.rawData(), "?array<?number>"_sd}, + {kMinKeyTypeString.rawData(), "?array<?minKey>"_sd}, + {kObjectTypeString.rawData(), "?array<?object>"_sd}, + {kArrayTypeString.rawData(), "?array<?array>"_sd}, + {kBinDataTypeString.rawData(), "?array<?binData>"_sd}, + {kObjectIdTypeString.rawData(), "?array<?objectId>"_sd}, + {kBoolTypeString.rawData(), "?array<?bool>"_sd}, + {kDateTypeString.rawData(), "?array<?date>"_sd}, + {kNullTypeString.rawData(), "?array<?null>"_sd}, + {kRegexTypeString.rawData(), "?array<?regex>"_sd}, + {kDbPointerTypeString.rawData(), "?array<?dbPointer>"_sd}, + {kJavascriptTypeString.rawData(), "?array<?javascript>"_sd}, + {kJavascriptWithScopeTypeString.rawData(), "?array<?javascriptWithScope>"_sd}, + {kTimestampTypeString.rawData(), "?array<?timestamp>"_sd}, + {kMaxKeyTypeString.rawData(), "?array<?maxKey>"_sd}, +}; + +static constexpr auto kRepresentativeString = "?"_sd; +static constexpr auto kRepresentativeNumber = 1; +static const auto kRepresentativeObject = BSON("?" + << "?"); +static const auto kRepresentativeArray = BSONArray(); +static constexpr auto kRepresentativeBinData = BSONBinData(); +static const auto kRepresentativeObjectId = OID::max(); +static constexpr auto kRepresentativeBool = true; +static const auto kRepresentativeDate = Date_t::fromMillisSinceEpoch(0); +static const auto kRepresentativeRegex = BSONRegEx("/\?/"); +static const auto kRepresentativeDbPointer = BSONDBRef("?.?", OID::max()); +static const auto kRepresentativeJavascript = BSONCode("return ?;"); +static const auto kRepresentativeJavascriptWithScope = BSONCodeWScope("return ?;", BSONObj()); +static const auto kRepresentativeTimestamp = Timestamp::min(); + +/** + * A default redaction strategy that generates easy to check results for testing purposes. + */ +std::string applyHmacForTest(StringData s) { + // Avoid ending in a parenthesis since the results will occur in a raw string where the )" + // sequence will accidentally terminate the string. + return str::stream() << "HASH<" << s << ">"; +} + +/** + * Computes a debug string meant to represent "any value of type t", where "t" is the type of the + * provided argument. For example "?number" for any number (int, double, etc.). + */ +StringData debugTypeString(BSONType t) { + // This is tightly coupled with 'canonicalizeBSONType' and therefore also with + // sorting/comparison semantics. + switch (t) { + case EOO: + case Undefined: + return kUndefinedTypeString; + case Symbol: + case String: + return kStringTypeString; + case NumberInt: + case NumberLong: + case NumberDouble: + case NumberDecimal: + return kNumberTypeString; + case MinKey: + return kMinKeyTypeString; + case Object: + return kObjectTypeString; + case Array: + // This case should only happen if we have an array within an array. + return kArrayTypeString; + case BinData: + return kBinDataTypeString; + case jstOID: + return kObjectIdTypeString; + case Bool: + return kBoolTypeString; + case Date: + return kDateTypeString; + case jstNULL: + return kNullTypeString; + case RegEx: + return kRegexTypeString; + case DBRef: + return kDbPointerTypeString; + case Code: + return kJavascriptTypeString; + case CodeWScope: + return kJavascriptWithScopeTypeString; + case bsonTimestamp: + return kTimestampTypeString; + case MaxKey: + return kMaxKeyTypeString; + default: + MONGO_UNREACHABLE_TASSERT(7539806); + } +} + +/** + * Returns an arbitrary value of the same type as the one given. For any number, this will be the + * number 1. For any boolean this will be true. + * TODO if you need a different value to make sure it will parse, you should not use this API. + */ +ImplicitValue defaultLiteralOfType(BSONType t) { + // This is tightly coupled with 'canonicalizeBSONType' and therefore also with + // sorting/comparison semantics. + switch (t) { + case EOO: + case Undefined: + return BSONUndefined; + case Symbol: + case String: + return kRepresentativeString; + case NumberInt: + case NumberLong: + case NumberDouble: + case NumberDecimal: + return kRepresentativeNumber; + case MinKey: + return MINKEY; + case Object: + return kRepresentativeObject; + case Array: + // This case should only happen if we have an array within an array. + return kRepresentativeArray; + case BinData: + return kRepresentativeBinData; + case jstOID: + return kRepresentativeObjectId; + case Bool: + return kRepresentativeBool; + case Date: + return kRepresentativeDate; + case jstNULL: + return BSONNULL; + case RegEx: + return kRepresentativeRegex; + case DBRef: + return kRepresentativeDbPointer; + case Code: + return kRepresentativeJavascript; + case CodeWScope: + return kRepresentativeJavascriptWithScope; + case bsonTimestamp: + return kRepresentativeTimestamp; + case MaxKey: + return MAXKEY; + default: + MONGO_UNREACHABLE_TASSERT(7539803); + } +} + +/** + * A struct representing the sub-type information for an array. + */ +struct ArraySubtypeInfo { + /** + * Whether the values of an array are all the same BSON type or not (mixed). + */ + enum class NTypes { kEmpty, kOneType, kMixed }; + ArraySubtypeInfo(NTypes nTypes_) : nTypes(nTypes_) {} + ArraySubtypeInfo(BSONType oneType) : nTypes(NTypes::kOneType), singleType(oneType) {} + + NTypes nTypes; + boost::optional<BSONType> singleType = boost::none; +}; + +template <typename ValueType> +using GetTypeFn = std::function<BSONType(ValueType)>; + +static GetTypeFn<BSONElement> getBSONElementType = [](const BSONElement& e) { return e.type(); }; +static GetTypeFn<Value> getValueType = [](const Value& v) { return v.getType(); }; + +/** + * Scans 'arrayOfValues' to see if all values are of the same type or not. Returns this info in a + * struct - see the struct definition for how it is represented. + * + * Templated algorithm to handle both iterators of BSONElements or iterators of Values. + * 'getTypeCallback' is provided to abstract away the different '.type()' vs '.getType()' APIs. + */ +template <typename ArrayType, typename ValueType> +ArraySubtypeInfo determineArraySubType(const ArrayType& arrayOfValues, + GetTypeFn<ValueType> getTypeCallback) { + boost::optional<BSONType> firstType = boost::none; + for (auto&& v : arrayOfValues) { + if (!firstType) { + firstType.emplace(getTypeCallback(v)); + } else if (*firstType != getTypeCallback(v)) { + return {ArraySubtypeInfo::NTypes::kMixed}; + } + } + return firstType ? ArraySubtypeInfo{*firstType} + : ArraySubtypeInfo{ArraySubtypeInfo::NTypes::kEmpty}; +} + +ArraySubtypeInfo determineArraySubType(const BSONObj& arrayAsObj) { + return determineArraySubType<BSONObj, BSONElement>(arrayAsObj, getBSONElementType); +} +ArraySubtypeInfo determineArraySubType(const std::vector<Value>& values) { + return determineArraySubType<std::vector<Value>, Value>(values, getValueType); +} + +template <typename ValueType> +StringData debugTypeString( + const ValueType& v, + GetTypeFn<ValueType> getTypeCallback, + std::function<ArraySubtypeInfo(ValueType)> determineArraySubTypeCallback) { + if (getTypeCallback(v) == BSONType::Array) { + // Iterating the array as .Obj(), as if it were a BSONObj (with field names '0', '1', etc.) + // is faster than converting the whole thing to an array which would force a copy. + auto typeInfo = determineArraySubTypeCallback(v); + switch (typeInfo.nTypes) { + case ArraySubtypeInfo::NTypes::kEmpty: + return "[]"_sd; + case ArraySubtypeInfo::NTypes::kOneType: + return kArrayTypeStringConstants.at(debugTypeString(*typeInfo.singleType)); + case ArraySubtypeInfo::NTypes::kMixed: + return "?array<>"; + default: + MONGO_UNREACHABLE_TASSERT(7539801); + } + } + return debugTypeString(getTypeCallback(v)); +} + +template <typename ValueType> +ImplicitValue defaultLiteralOfType( + const ValueType& v, + GetTypeFn<ValueType> getTypeCallback, + std::function<ArraySubtypeInfo(ValueType)> determineArraySubTypeCallback) { + if (getTypeCallback(v) == BSONType::Array) { + auto typeInfo = determineArraySubTypeCallback(v); + switch (typeInfo.nTypes) { + case ArraySubtypeInfo::NTypes::kEmpty: + return BSONArray(); + case ArraySubtypeInfo::NTypes::kOneType: + return std::vector<Value>{defaultLiteralOfType(*typeInfo.singleType)}; + case ArraySubtypeInfo::NTypes::kMixed: + // We don't care which types, we'll use a number and a string as the canonical + // mixed type array regardless. This is to ensure we don't get 2^N possibilities + // for mixed type scenarios - we wish to collapse all "mixed type" arrays to one + // canonical mix. The choice of int and string is mostly arbitrary - hopefully + // somewhat comprehensible at a glance. + return std::vector<Value>{Value(2), Value("or more types"_sd)}; + default: + MONGO_UNREACHABLE_TASSERT(7539805); + } + } + return defaultLiteralOfType(getTypeCallback(v)); +} + +ArraySubtypeInfo getSubTypeFromBSONElemArray(BSONElement arrayElem) { + // Iterating the array as .Obj(), as if it were a BSONObj (with field names '0', '1', etc.) + // is faster than converting the whole thing to an array which would force a copy. + return determineArraySubType(arrayElem.Obj()); +} +ArraySubtypeInfo getSubTypeFromValueArray(const Value& arrayVal) { + return determineArraySubType(arrayVal.getArray()); +} + +void appendDefaultOfNonArrayType(BSONObjBuilder* bob, StringData name, const BSONElement& e) { + switch (e.type()) { + case EOO: + case Undefined: + bob->appendUndefined(name); + return; + case Symbol: + case String: + bob->append(name, kRepresentativeString); + return; + case NumberInt: + case NumberLong: + case NumberDouble: + case NumberDecimal: + bob->append(name, kRepresentativeNumber); + return; + case MinKey: + bob->appendMinKey(name); + return; + case Object: + bob->append(name, kRepresentativeObject); + return; + case Array: + // This case is more complicated and callers should use a more generic helper. + MONGO_UNREACHABLE_TASSERT(8094100); + case BinData: + bob->append(name, kRepresentativeBinData); + return; + case jstOID: + bob->append(name, kRepresentativeObjectId); + return; + case Bool: + bob->append(name, kRepresentativeBool); + return; + case Date: + bob->append(name, kRepresentativeDate); + return; + case jstNULL: + bob->appendNull(name); + return; + case RegEx: + bob->append(name, kRepresentativeRegex); + return; + case DBRef: + bob->append(name, kRepresentativeDbPointer); + return; + case Code: + bob->append(name, kRepresentativeJavascript); + return; + case CodeWScope: + bob->append(name, kRepresentativeJavascriptWithScope); + return; + case bsonTimestamp: + bob->append(name, kRepresentativeTimestamp); + return; + case MaxKey: + bob->appendMaxKey(name); + return; + default: + MONGO_UNREACHABLE_TASSERT(8094101); + }; +} +} // namespace + +const SerializationOptions SerializationOptions::kRepresentativeQueryShapeSerializeOptions = + SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + +const SerializationOptions SerializationOptions::kDebugQueryShapeSerializeOptions = + SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + +SerializationOptions::SerializationOptions(LiteralSerializationPolicy policy) + : literalPolicy(policy) {} +SerializationOptions::SerializationOptions( + boost::optional<ExplainOptions::Verbosity> explainVerbosity) + : verbosity(explainVerbosity) {} + +SerializationOptions::SerializationOptions(LiteralSerializationPolicy policy, + bool transformIdentifiers, + TokenizeIdentifierFunc transformIdentifiersCallbackFn) + : literalPolicy(policy), + transformIdentifiers(transformIdentifiers), + transformIdentifiersCallback(transformIdentifiersCallbackFn) {} + +const SerializationOptions SerializationOptions::kMarkIdentifiers_FOR_TEST{ + LiteralSerializationPolicy::kUnchanged, true, applyHmacForTest}; + +const SerializationOptions SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST{ + LiteralSerializationPolicy::kToDebugTypeString, true, applyHmacForTest}; + +// Overloads for BSONElem and Value. +StringData debugTypeString(BSONElement e) { + return debugTypeString<BSONElement>(e, getBSONElementType, getSubTypeFromBSONElemArray); +} +StringData debugTypeString(const Value& v) { + return debugTypeString<Value>(v, getValueType, getSubTypeFromValueArray); +} + +// Overloads for BSONElem and Value. +ImplicitValue defaultLiteralOfType(const Value& v) { + return defaultLiteralOfType<Value>(v, getValueType, getSubTypeFromValueArray); +} +ImplicitValue defaultLiteralOfType(BSONElement e) { + return defaultLiteralOfType<BSONElement>(e, getBSONElementType, getSubTypeFromBSONElemArray); +} + +void SerializationOptions::appendLiteral(BSONObjBuilder* bob, const BSONElement& e) const { + appendLiteral(bob, e.fieldNameStringData(), e); +} +void SerializationOptions::appendLiteral(BSONObjBuilder* bob, + StringData name, + const BSONElement& e) const { + // The first two cases are particularly performance sensitive. We could answer everything here + // with the code inside the 'kToDebugTypeString' branch, but there are some relatively easy ways + // to accomplish the first two policy cases (in the common cases), so we'll special case those + // in order to avoid constructing a temporary Value. + switch (literalPolicy) { + case LiteralSerializationPolicy::kUnchanged: + bob->appendAs(e, name); + return; + case LiteralSerializationPolicy::kToRepresentativeParseableValue: { + if (e.type() != BSONType::Array) { + appendDefaultOfNonArrayType(bob, name, e); + return; + } + // If it's an array we'll default to the slow but general codepath below. + [[fallthrough]]; + } + case LiteralSerializationPolicy::kToDebugTypeString: { + // Performance isn't as sensitive here. + return serializeLiteral(e).addToBsonObj(bob, name); + } + default: + MONGO_UNREACHABLE_TASSERT(8094102); + } +} + +void SerializationOptions::appendLiteral(BSONObjBuilder* bob, + StringData fieldName, + const ImplicitValue& v, + const boost::optional<Value>& representativeValue) const { + serializeLiteral(v, representativeValue).addToBsonObj(bob, fieldName); +} + +Value SerializationOptions::serializeLiteral( + const BSONElement& e, const boost::optional<Value>& representativeValue) const { + switch (literalPolicy) { + case LiteralSerializationPolicy::kUnchanged: + return Value(e); + case LiteralSerializationPolicy::kToDebugTypeString: + return Value(debugTypeString(e)); + case LiteralSerializationPolicy::kToRepresentativeParseableValue: + return representativeValue.value_or(defaultLiteralOfType(e)); + default: + MONGO_UNREACHABLE_TASSERT(7539802); + } +} + +Value SerializationOptions::serializeLiteral( + const ImplicitValue& v, const boost::optional<Value>& representativeValue) const { + switch (literalPolicy) { + case LiteralSerializationPolicy::kUnchanged: + return v; + case LiteralSerializationPolicy::kToDebugTypeString: + return Value(debugTypeString(v)); + case LiteralSerializationPolicy::kToRepresentativeParseableValue: + return representativeValue.value_or(defaultLiteralOfType(v)); + default: + MONGO_UNREACHABLE_TASSERT(7539804); + } +} + +std::string SerializationOptions::serializeFieldPathFromString(StringData path) const { + if (transformIdentifiers) { + try { + return serializeFieldPath(FieldPath(path, false)); + } catch (DBException& ex) { + LOGV2_DEBUG(7549808, + 1, + "Failed to convert a path string to a FieldPath", + "pathString"_attr = path, + "failure"_attr = ex.toStatus()); + return serializeFieldPath("invalidFieldPathPlaceholder"); + } + } + return path.toString(); +} +} // namespace mongo diff --git a/src/mongo/db/query/query_shape/serialization_options.h b/src/mongo/db/query/query_shape/serialization_options.h new file mode 100644 index 00000000000..226da7689d3 --- /dev/null +++ b/src/mongo/db/query/query_shape/serialization_options.h @@ -0,0 +1,236 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/exec/document_value/document.h" +#include "mongo/db/exec/document_value/value.h" +#include "mongo/db/pipeline/field_path.h" +#include "mongo/db/query/explain_options.h" +#include "mongo/util/assert_util.h" +#include <boost/optional.hpp> +#include <string> + +namespace mongo { +namespace { +// Should never be called, throw to ensure we catch this in tests. +std::string defaultHmacStrategy(StringData s) { + MONGO_UNREACHABLE_TASSERT(7332410); +} +} // namespace + +/** + * A policy enum for how to serialize literal values. + */ +enum class LiteralSerializationPolicy { + // The default way to serialize. Just serialize whatever literals were given if they are still + // available, or whatever you parsed them to. This is expected to be able to parse again, since + // it worked the first time. + kUnchanged, + // Serialize any literal value as "?number" or similar. For example "?bool" for any boolean. Use + // 'debugTypeString()' helper. + kToDebugTypeString, + // Serialize any literal value to one canonical value of the given type, with the constraint + // that the chosen representative value should be parseable in this context. There are some + // default implementations that will usually work (e.g. using the number 1 almost always works + // for numbers), but serializers should be careful to think about and test this if their parsers + // reject certain values. + kToRepresentativeParseableValue, +}; + +/** + * A struct with options for how you want to serialize a match or aggregation expression. + */ +struct SerializationOptions { + using TokenizeIdentifierFunc = std::function<std::string(StringData)>; + + // The default serialization options for a query shape. No need to redact identifiers for the + // this purpose. We may do that on the $queryStats read path. + static const SerializationOptions kRepresentativeQueryShapeSerializeOptions; + static const SerializationOptions kDebugQueryShapeSerializeOptions; + static const SerializationOptions kMarkIdentifiers_FOR_TEST; + static const SerializationOptions kDebugShapeAndMarkIdentifiers_FOR_TEST; + + SerializationOptions() = default; + SerializationOptions(LiteralSerializationPolicy policy); + SerializationOptions(boost::optional<ExplainOptions::Verbosity> explain); + SerializationOptions(LiteralSerializationPolicy policy, + bool transformIdentifiers, + TokenizeIdentifierFunc transformIdentifiersCallbackFn); + + /** + * Checks if this SerializationOptions represents the same options as another + * SerializationOptions. Note it cannot compare whether the two 'transformIdentifiersCallback's + * are the same - the language purposefully leaves the comparison operator undefined. + */ + bool operator==(const SerializationOptions& other) const { + return this->transformIdentifiers == other.transformIdentifiers && + // You cannot well determine std::function equivalence in C++, so this is the best we'll + // do. + (this->transformIdentifiersCallback == nullptr) == + (other.transformIdentifiersCallback == nullptr) && + this->literalPolicy == other.literalPolicy && this->verbosity == other.verbosity; + } + bool operator!=(const SerializationOptions& other) const { + return !(*this == other); + } + + // Helper function for removing identifiable information (like collection/db names). + // Note: serializeFieldPath/serializeFieldPathFromString should be used for field + // names. + std::string serializeIdentifier(StringData str) const { + if (transformIdentifiers) { + return transformIdentifiersCallback(str); + } + return str.toString(); + } + + std::string serializeFieldPath(FieldPath path) const { + if (transformIdentifiers) { + std::stringstream hmaced; + for (size_t i = 0; i < path.getPathLength(); ++i) { + if (i > 0) { + hmaced << "."; + } + hmaced << transformIdentifiersCallback(path.getFieldName(i)); + } + return hmaced.str(); + } + return path.fullPath(); + } + + std::string serializeFieldPathWithPrefix(FieldPath path) const { + return "$" + serializeFieldPath(path); + } + + std::string serializeFieldPathFromString(StringData path) const; + + std::vector<std::string> serializeFieldPathFromString( + const std::vector<std::string>& paths) const { + std::vector<std::string> result; + result.reserve(paths.size()); + for (auto& p : paths) { + result.push_back(serializeFieldPathFromString(p)); + } + return result; + } + + // Helper functions for applying hmac to BSONObj. Does not take into account anything to do with + // MQL semantics, removes all field names and literals in the passed in obj. + void addHmacedArrayToBuilder(BSONArrayBuilder* bab, std::vector<BSONElement> array) const { + for (const auto& elem : array) { + if (elem.type() == BSONType::Object) { + BSONObjBuilder subObj(bab->subobjStart()); + addHmacedObjToBuilder(&subObj, elem.Obj()); + subObj.done(); + } else if (elem.type() == BSONType::Array) { + BSONArrayBuilder subArr(bab->subarrayStart()); + addHmacedArrayToBuilder(&subArr, elem.Array()); + subArr.done(); + } else { + *bab << serializeLiteral(elem); + } + } + } + + void addHmacedObjToBuilder(BSONObjBuilder* bob, BSONObj objToHmac) const { + for (const auto& elem : objToHmac) { + auto fieldName = serializeFieldPath(elem.fieldName()); + if (elem.type() == BSONType::Object) { + BSONObjBuilder subObj(bob->subobjStart(fieldName)); + addHmacedObjToBuilder(&subObj, elem.Obj()); + subObj.done(); + } else if (elem.type() == BSONType::Array) { + BSONArrayBuilder subArr(bob->subarrayStart(fieldName)); + addHmacedArrayToBuilder(&subArr, elem.Array()); + subArr.done(); + } else { + appendLiteral(bob, fieldName, elem); + } + } + } + + /** + * Helper method to call 'serializeLiteral()' on 'e' and append the resulting value to 'bob' + * using the same name as 'e'. + */ + void appendLiteral(BSONObjBuilder* bob, const BSONElement& e) const; + void appendLiteral(BSONObjBuilder* bob, StringData name, const BSONElement& e) const; + /** + * Helper method to call 'serializeLiteral()' on 'v' and append the result to 'bob' using field + * name 'fieldName'. + */ + void appendLiteral(BSONObjBuilder* bob, + StringData fieldName, + const ImplicitValue& v, + const boost::optional<Value>& representativeValue = boost::none) const; + + /** + * Depending on the configured 'literalPolicy', serializeLiteral will return the appropriate + * value for adding literals to serialization output: + * - If 'literalPolicy' is 'kUnchanged', returns the input value unmodified. + * - If it is 'kToDebugTypeString', computes and returns the type string as a string Value. + * - If it is 'kToRepresentativeValue', it returns an arbitrary value of the same type as the + * one given. For any number, this will be the number 1. For any boolean this will be true. + * If the 'representativeValue' parameter if it is not none, returns it (regardless of type). + * + * Example usage: BSON("myArg" << options.serializeLiteral(_myArg)); + */ + Value serializeLiteral(const BSONElement& e, + const boost::optional<Value>& representativeValue = boost::none) const; + Value serializeLiteral(const ImplicitValue& v, + const boost::optional<Value>& representativeValue = boost::none) const; + + // 'literalPolicy' is an independent option to serialize in a general format with the aim of + // similar "shaped" queries serializing to the same object. For example, if set to + // 'kToDebugTypeString', then the serialization of {a: {$gt: 2}} should result in {a: {$gt: + // '?number'}}, as will the serialization of {a: {$gt: 3}}. + // + // "Literal" here is meant to stand in contrast to expression arguments, as in the $gt + // expressions in {$and: [{a: {$gt: 3}}, {b: {$gt: 4}}]}. There the only literals are 3 and 4, + // so the serialization expected for 'kToDebugTypeString' would be {$and: [{a: {$gt: + // '?number'}}, {b: {$lt: '?number'}}]}. + LiteralSerializationPolicy literalPolicy = LiteralSerializationPolicy::kUnchanged; + + // If true the caller must set transformIdentifiersCallback. 'transformIdentifiers' if set along + // with a strategy the redaction strategy will be called on any personal identifiable + // information (e.g., field paths/names, collection names) encountered before serializing them. + bool transformIdentifiers = false; + std::function<std::string(StringData)> transformIdentifiersCallback = defaultHmacStrategy; + + // For aggregation indicate whether we should use the more verbose serialization format. + boost::optional<ExplainOptions::Verbosity> verbosity = boost::none; + + // If set to true, serializes each stage and expression as needed for query analysis. + bool serializeForQueryAnalysis = false; +}; + +} // namespace mongo diff --git a/src/mongo/db/query/query_shape/shape_helpers.cpp b/src/mongo/db/query/query_shape/shape_helpers.cpp new file mode 100644 index 00000000000..8eea475ab78 --- /dev/null +++ b/src/mongo/db/query/query_shape/shape_helpers.cpp @@ -0,0 +1,108 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/shape_helpers.h" + +#include "mongo/db/query/query_shape/query_shape_gen.h" + +namespace mongo::shape_helpers { + +static constexpr StringData hintSpecialField = "$hint"_sd; +// A "Flat" object is one with only top-level fields. We won't descend recursively to shapify any +// sub-objects. +BSONObj shapifyFlatObj(BSONObj obj, const SerializationOptions& opts, bool valuesAreLiterals) { + if (obj.isEmpty()) { + // fast-path for the common case. + return obj; + } + + BSONObjBuilder bob; + for (BSONElement elem : obj) { + if (hintSpecialField.compare(elem.fieldNameStringData()) == 0) { + if (elem.type() == BSONType::String) { + bob.append(hintSpecialField, opts.serializeFieldPathFromString(elem.String())); + } else if (elem.type() == BSONType::Object) { + opts.appendLiteral(&bob, hintSpecialField, elem.Obj()); + } else { + // SERVER-85500: $hint syntax will not be validated if the collection does not + // exist, so we should accept a value that is neither string nor object here. + opts.appendLiteral(&bob, hintSpecialField, elem); + } + continue; + } + + // $natural doesn't need to be redacted. + if (elem.fieldNameStringData().compare(query_request_helper::kNaturalSortField) == 0) { + bob.append(elem); + continue; + } + + if (valuesAreLiterals) { + opts.appendLiteral(&bob, opts.serializeFieldPathFromString(elem.fieldName()), elem); + } else { + bob.appendAs(elem, opts.serializeFieldPathFromString(elem.fieldName())); + } + } + return bob.obj(); +} + +BSONObj extractHintShape(BSONObj hintObj, const SerializationOptions& opts) { + return shapifyFlatObj(hintObj, opts, /* valuesAreLiterals = */ false); +} + +BSONObj extractMinOrMaxShape(BSONObj obj, const SerializationOptions& opts) { + return shapifyFlatObj(obj, opts, /* valuesAreLiterals = */ true); +} + +void appendNamespaceShape(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts) { + bob.append("db", opts.serializeIdentifier(nss.db())); + bob.append("coll", opts.serializeIdentifier(nss.coll())); +} + +NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt) { + tassert(7632900, "cmdNs must be an object.", cmdNsElt.type() == BSONType::Object); + auto cmdNs = query_shape::CommandNamespace::parse("cmdNs"_sd, cmdNsElt.embeddedObject()); + + if (cmdNs.getColl().has_value()) { + tassert(7632903, + "Exactly one of 'uuid' and 'coll' can be defined.", + !cmdNs.getUuid().has_value()); + return NamespaceString(cmdNs.getDb(), cmdNs.getColl().value()); + } else { + tassert(7632904, + "Exactly one of 'uuid' and 'coll' can be defined.", + !cmdNs.getColl().has_value()); + UUID uuid = uassertStatusOK(UUID::parse(cmdNs.getUuid().value().toString())); + return NamespaceStringOrUUID(cmdNs.getDb().toString(), uuid); + } +} + +} // namespace mongo::shape_helpers diff --git a/src/mongo/db/query/query_shape/shape_helpers.h b/src/mongo/db/query/query_shape/shape_helpers.h new file mode 100644 index 00000000000..4d0fadb4a47 --- /dev/null +++ b/src/mongo/db/query/query_shape/shape_helpers.h @@ -0,0 +1,101 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/bson/simple_bsonobj_comparator.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/serialization_options.h" + +namespace mongo::shape_helpers { + +int64_t inline optionalObjSize(boost::optional<BSONObj> optionalObj) { + if (!optionalObj) + return 0; + return optionalObj->objsize(); +} + +template <typename T> +int64_t optionalSize(boost::optional<T> optionalVal) { + if (!optionalVal) + return 0; + return optionalVal->size(); +} + +template <typename T> +std::function<size_t(size_t, const T&)> sizeAccumulatorFunc() { + MONGO_UNREACHABLE; // Don't know how to compute the size of this template type. +}; + +template <> +inline std::function<size_t(size_t, const BSONObj&)> sizeAccumulatorFunc<BSONObj>() { + return [](size_t total, const BSONObj& obj) { + return total + sizeof(BSONObj) + static_cast<size_t>(obj.objsize()); + }; +} + +template <> +inline std::function<size_t(size_t, const NamespaceString&)> +sizeAccumulatorFunc<NamespaceString>() { + return [](size_t total, const NamespaceString& nss) { + // For each element, we have to track the size of the + // nss as well as the size allocated by the nss. It would be + // ideal to be able to ask the underlying namespace string for + // its capacity, but it's not something we have access to. + // Further, namespace strings appear to shrink to fit (i.e + // resize to correct size), so it may not be necessary. Should + // we also try to consider short string optimization? At the + // very least, the current approach gives us a good upper bound + // memory usage (assuming shrink to fit). + return total + sizeof(nss) + nss.size(); + }; +} + +template <typename Container> +size_t containerSize(const Container& container) { + return std::accumulate(container.begin(), + container.end(), + 0, + sizeAccumulatorFunc<typename Container::value_type>()); +} + +/** + * Serializes the given 'hintObj' in accordance with the options. Assumes the hint is correct and + * contains field names. It is possible that this hint doesn't actually represent an index, but we + * can't detect that here. + */ +BSONObj extractHintShape(BSONObj hintObj, const SerializationOptions& opts); +BSONObj extractMinOrMaxShape(BSONObj obj, const SerializationOptions& opts); + +NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt); +void appendNamespaceShape(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts); + +} // namespace mongo::shape_helpers diff --git a/src/mongo/db/query/query_solution.cpp b/src/mongo/db/query/query_solution.cpp index 669777b61ae..191218b2f4d 100644 --- a/src/mongo/db/query/query_solution.cpp +++ b/src/mongo/db/query/query_solution.cpp @@ -1566,11 +1566,11 @@ void GroupNode::appendToString(str::stream* ss, int indent) const { if (idx > 0) { *ss << ", "; } - *ss << "{" << groupName << ": " << exprObj->serialize(false).toString() << "}"; + *ss << "{" << groupName << ": " << exprObj->serialize().toString() << "}"; ++idx; } } else { - *ss << "{_id: " << groupByExpression->serialize(false).toString() << "}"; + *ss << "{_id: " << groupByExpression->serialize().toString() << "}"; } *ss << '\n'; addIndent(ss, indent + 1); @@ -1581,7 +1581,11 @@ void GroupNode::appendToString(str::stream* ss, int indent) const { } auto& acc = accumulators[idx]; *ss << "{" << acc.fieldName << ": {" << acc.expr.name << ": " - << acc.expr.argument->serialize(true).toString() << "}}"; + << acc.expr.argument + ->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}) + .toString() + << "}}"; } *ss << "]" << '\n'; addCommon(ss, indent); diff --git a/src/mongo/db/query/query_stats/README.md b/src/mongo/db/query/query_stats/README.md new file mode 100644 index 00000000000..6f2667fbfd9 --- /dev/null +++ b/src/mongo/db/query/query_stats/README.md @@ -0,0 +1,200 @@ +# Query Stats +This directory is the home of the infrastructure related to recording runtime query statistics for +the database. It is not to be confused with `src/mongo/db/query/stats/` which is the home of the +logic for computing and maintaining statistics about a collection or index's data distribution - for +use by the query planner. + +The system will collect metrics for each query execution, and the results will be aggregated in a +structure called the [`QueryStatsStore`](#querystatsstore) upon completion of each successful +execution. Metrics will be aggregated according to an abstracted version of the query known as the +query stats key and will be collected on any mongod or mongos process for which they are configured, +including primaries and secondaries. + +## QueryStatsStore +At the center of everything here is the [`QueryStatsStore`](query_stats.h#93-97), which is a +partitioned hash table that maps the hash of a [Query Stats Key](#glossary) (also known as the +_Query Stats Store Key_) to some metrics about how often each one occurs. + +### Computing the Query Stats Store Key +A query stats store key contains various dimensions that distinctify a specific query. One main +attribute to the query stats store key, is the query shape (`query_shape::Shape`). For example, if +the client does this: +```js +db.example.findOne({x: 24}); +db.example.findOne({x: 53}); +``` +then the `QueryStatsStore` should contain an entry for a single query shape which would record 2 +executions and some related statistics (see [`QueryStatsEntry`](query_stats_entry.h) for details). + +For more information on query shape, see the [query_shape](../query_shape/README.md) directory. + +The query stats store has _more_ dimensions (i.e. more granularity) to group incoming queries than +just the query shape. For example, these queries would all three have the same shape but the first +would have a different query stats store entry from the other two: +```js +db.example.find({x: 55}); +db.example.find({x: 55}).batchSize(2); +db.example.find({x: 55}).batchSize(3); +``` +There are two distinct query stats store entries here - both the examples which include the batch +size will be treated separately from the example which does not specify a batch size. + +The dimensions considered will depend on the command, but can generally be found in the +[`KeyGenerator`](key_generator.h) interface, which will generate the query stats store keys by which +we accumulate statistics. As one example, you can find the +[`FindKey`](find_key.h) which will include all the things tracked in the +`FindCmdQueryStatsStoreKeyComponents` (including `batchSize` shown in this example). + +### Query Stats Store Cache Size +The size of the`QueryStatsStore` can be set by the server parameter +[`internalQueryStatsCacheSize`](#server-parameters), and the partitions will be created based off +that. See [`queryStatsStoreManagerRegisterer`](query_stats.cpp#L138-L154) for more details about how +the number of partitions and their size is determined; Each partition is an LRU cache, therefore, if +adding a new entry to the partition makes it go over its size limit, the least recently used entries +will be evicted to drop below the max size. Eviction will be tracked in the new [server status +metrics](#server-status-metrics) for queryStats. + +## Metric Collection +At a high level, when a query is run and collection of query stats is enabled, during planning we +call [`registerRequest`]((query_stats.h#L195-L198)) in which the query stats store key will be +generated based on the query's shape and the various other dimensions. The key will always be serialized +and stored on the `opDebug`, and also on the cursor in the case that there are `getMore`s, so that we can +continue to aggregate the operation's metrics. Once the query execution is fully complete, +[`writeQueryStats`](query_stats.h#L200-216) will be called and will either retrieve the entry for +the key from the store if it exists and update it, or create a new one and add it to the store. See +more details in the [comments](query_stats.h#L158-L216). + +### Rate Limiting +Whether or not query stats will be recorded for a specific query execution depends on a Rate +Limiter, which limits the number of recordings per second based on the server parameter +[internalQueryStatsRateLimit](#server-parameters). The goal of the rate limiter is to minimize +impact to overall system performance through restricting excessive traffic. If a query is run but +the rate limit has been reached, the query will still execute as expected but query stats will not +be updated in the query stats store. Our rate limiter uses the sliding window algorithm; see details +[here](rate_limiting.h#82-87). + +## Metric Retrieval +To retrieve the stats gathered in the `QueryStatsStore`, there is a new aggregation stage, +`$queryStats`. This stage must be the first in a pipeline and it must be run against the admin +database. The structure of the command is as follows (note `aggregate: 1` reflecting there is no collection): +```js +db.adminCommand({ + aggregate: 1, + pipeline: [{ + $queryStats: { + tranformIdentifiers: { + algorithm: "hmac-sha-256", + hmacKey: BinData(8, "87c4082f169d3fef0eef34dc8e23458cbb457c3sf3n2") /* bindata + subtype 8 - a new type for sensitive data */, + } + } + }] +}) +``` +`transformIdentifiers` is optional. If not present, we will generate the regular Query Stats Key. If +present: +- `algorithm` is required and the only currently supported option is "hmac-sha-256". +- `hmacKey` is required +- We will generate the [One-way Tokenized](#glossary) Query Stats Key by applying the "hmac-sha-256" + to the names of any field, collection, or database. Application Name field is not transformed. + +The query stats store will output one document for each query stats key, which is structured in the +following way: +```js +{ + key: {/* Query Stats Key */}, + asOf: ISODate(/* … */), + metrics: { + execCount: 0, + firstSeenTimestamp: ISODate(/* … */), + latestSeenTimestamp: ISODate(/* … */), + docsReturned: {sum: 0, max: 0, min: 0, sumOfSquares: 0}, + firstResponseExecMicros: {sum: 0, max: 0, min: 0, sumOfSquares: 0}, + totalExecMicros: {sum: 0, max: 0, min: 0, sumOfSquares: 0}, + lastExecutionMicros: 0, + } +} +``` +- `key`: Query Stats Key. +- `asOf`: UTC time when $queryStats read this entry from the store. This will not return the same + UTC time for each result. The data structure used for the store is partitioned, and each partition + will be read at a snapshot individually. You may see up to the number of partitions in unique + timestamps returned by one $queryStats cursor. +- `metrics`: the metrics collected; these may be flawed due to: + - Server restarts, which will reset metrics. + - LRU eviction, which will reset metrics. + - Rate limiting, which will skew metrics. +- `metrics.execCount`: Number of recorded observations of this query. +- `metrics.firstSeenTimestamp`: UTC time taken at query completion (including getMores) for the + first recording of this query stats store entry. +- `metrics.lastSeenTimestamp`: UTC time taken at query completion (including getMores) for the + latest recording of this query stats store entry. +- `metrics.docsReturned`: Various broken down metrics for the number of documents returned by + observation of this query. +- `metrics.firstResponseExecMicros`: Estimated time spent computing and returning the first batch. +- `metrics.totalExecMicros`: Estimated time spent computing and returning all batches, which is the + same as the above for single-batch queries. +- `metrics.lastExecutionMicros`: Estimated time spent processing the latest query (akin to + "totalExecMicros", not "firstResponseExecMicros"). + +#### Permissions +`$queryStats` is restricted by two privilege actions: +- `queryStatsRead` privilege allows running `$queryStats` without passing the `transformIdentifiers` + options. +- `queryStatsReadTransformed` allows running `$queryStats` with `transformIdentifiers` set. These +two privileges are included in the clusterMonitor role in Atlas. + +### Server Parameters +- `internalQueryStatsCacheSize`: + * Max query stats store size, specified as a string like "4MB" or "1%". Defaults to 1% of the + machine's total memory. + * Query stats store is a LRU cache structure with partitions, so we may be under the cap due to + implementation. + +- `internalQueryStatsRateLimit`: + * The rate limit is an integer which imposes a maximum number of recordings per second. Default is + 0 which has the effect of disabling query stats collection. Setting the parameter to -1 means + there will be no rate limit. + +- `logComponentVerbosity.queryStats`: + * Controls the logging behavior for query stats. See [Logging](#logging) for details. + +### Logging +Setting `logComponentVerbosity.queryStats` will do the following for each level: +* Level 0 (default): Nothing will be logged. +* Level 1 or higher: Invocations of $queryStats will be logged if and only if the algorithm is + "hmac-sha-256". The specification of the $queryStats stage is logged, with any provided hmac key + redacted. +* Level 2 or higher: Nothing extra, reserved for future use. +* Level 3 or higher: All results of any "hmac-sha-256" $queryStats invocation are logged. Each + result will be its own entry and there will be one final entry that says "we finished". +* Levels 4 and 5 do nothing extra. + +### Server Status Metrics +The following will be added to the `serverStatus.metrics`: +```js +queryStats: { + numEvicted: NumberLong(0), + numHmacApplicationErrors: NumberLong(0), + numQueryStatsStoreWriteErrors: NumberLong(0), + numRateLimitedRequests: NumberLong(0), + queryStatsStoreSizeEstimateBytes: NumberLong(0) +} +``` + +# Glossary +**Query Execution**: This term implies the overall execution of what a client would consider one +query, but which may or may not involve one or more getMore commands to iterate a cursor. For +example, a find command and two getMore commands on the returned cursor is one query execution. An +aggregate command which returns everything in one batch is also one query execution. + +**One-way Tokenized Object**: A one-way tokenized object has an HMAC hashing function applied to +particular sensitive elements/pieces of an object. It is "one-way" because it is never meant to be +undone. This allows us to detect when two queries are using the same identifiers, but never to +reveal what those identifiers were. + +**Query Shape**: [Query Shape](../query_shape/README.md) + +**Query Stats Key**: Also known as the _Query Stats Store Key_, this is the collection of attributes +championed by the query shape which identifies one grouping of metrics. The $queryStats stage will +output one document per query stats key - output in the "key" field. diff --git a/src/mongo/db/query/query_stats/SConscript b/src/mongo/db/query/query_stats/SConscript new file mode 100644 index 00000000000..f9f3a8b1c2e --- /dev/null +++ b/src/mongo/db/query/query_stats/SConscript @@ -0,0 +1,121 @@ +# -*- mode: python -*- + +Import([ + "env", + "get_option", +]) + +env = env.Clone() + +env.Library( + target='rate_limiting', + source=[ + 'rate_limiting.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/util/clock_sources', + ], +) + +env.Library(target='query_stats_parse', source=['transform_algorithm.idl'], LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/idl/idl_parser', +]) + +env.Library( + target='query_stats', + source=[ + '$BUILD_DIR/mongo/db/curop.cpp', + 'key.cpp', + 'query_stats.cpp', + 'query_stats_entry.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/mutable/mutable_bson', + '$BUILD_DIR/mongo/db/commands', + '$BUILD_DIR/mongo/db/concurrency/lock_manager', + '$BUILD_DIR/mongo/db/exec/document_value/document_value', + '$BUILD_DIR/mongo/db/generic_cursor', + '$BUILD_DIR/mongo/db/profile_filter', + '$BUILD_DIR/mongo/db/query/command_request_response', + '$BUILD_DIR/mongo/db/query/memory_util', + '$BUILD_DIR/mongo/db/query/query_knobs', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', + '$BUILD_DIR/mongo/db/server_options', + '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/db/stats/counters', + '$BUILD_DIR/mongo/db/stats/timer_stats', + '$BUILD_DIR/mongo/db/storage/storage_engine_parameters', + '$BUILD_DIR/mongo/rpc/client_metadata', + '$BUILD_DIR/mongo/transport/service_executor', + '$BUILD_DIR/mongo/util/diagnostic_info' if get_option('use-diagnostic-latches') == 'on' else [], + '$BUILD_DIR/mongo/util/fail_point', + '$BUILD_DIR/mongo/util/net/network', + '$BUILD_DIR/mongo/util/processinfo', + '$BUILD_DIR/mongo/util/progress_meter', + 'query_stats_parse', + 'rate_limiting', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/auth/auth', + '$BUILD_DIR/mongo/db/auth/user_acquisition_stats', + '$BUILD_DIR/mongo/db/exec/projection_executor', + '$BUILD_DIR/mongo/db/prepare_conflict_tracker', + '$BUILD_DIR/mongo/db/stats/resource_consumption_metrics', + ], +) + +env.CppUnitTest( + target="db_query_query_stats_test", + source=[ + "agg_key_test.cpp", + "find_key_test.cpp", + "key_test.cpp", + "query_stats_test.cpp", + "query_stats_store_test.cpp", + "rate_limiting_test.cpp", + ], + LIBDEPS=[ + "$BUILD_DIR/mongo/db/auth/authmocks", + "$BUILD_DIR/mongo/db/query/query_shape/query_shape", + "$BUILD_DIR/mongo/db/query/query_test_service_context", + "$BUILD_DIR/mongo/db/service_context_d_test_fixture", + "query_stats", + "rate_limiting", + ], +) + +env.Benchmark( + target='rate_limiting_bm', + source=[ + 'rate_limiting_bm.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/unittest/unittest', + '$BUILD_DIR/mongo/util/processinfo', + 'rate_limiting', + ], +) + +env.Benchmark( + target='shapifying_bm', + source=[ + 'shapifying_bm.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/auth/auth', + '$BUILD_DIR/mongo/db/pipeline/pipeline', + '$BUILD_DIR/mongo/db/query/canonical_query', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', + '$BUILD_DIR/mongo/db/query/query_test_service_context', + '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/rpc/client_metadata', + '$BUILD_DIR/mongo/unittest/unittest', + '$BUILD_DIR/mongo/util/processinfo', + 'query_stats', + ], +) diff --git a/src/mongo/db/query/query_stats/agg_key.cpp b/src/mongo/db/query/query_stats/agg_key.cpp new file mode 100644 index 00000000000..1d53418d371 --- /dev/null +++ b/src/mongo/db/query/query_stats/agg_key.cpp @@ -0,0 +1,174 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_stats/agg_key.h" + +#include "mongo/db/query/explain_options.h" +#include <absl/container/node_hash_set.h> +#include <boost/cstdint.hpp> +#include <functional> +#include <initializer_list> +#include <memory> +#include <numeric> +#include <vector> + +#include <boost/move/utility_core.hpp> +#include <boost/optional/optional.hpp> +#include <boost/smart_ptr/intrusive_ptr.hpp> + +#include "mongo/crypto/fle_field_schema_gen.h" +#include "mongo/db/pipeline/exchange_spec_gen.h" +#include "mongo/db/pipeline/pipeline.h" +#include "mongo/db/query/query_shape/agg_cmd_shape.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/db/query/query_shape/shape_helpers.h" +#include "mongo/idl/basic_types_gen.h" +#include "mongo/util/assert_util.h" + +namespace mongo::query_stats { + +AggCmdComponents::AggCmdComponents(const AggregateCommandRequest& request_, + stdx::unordered_set<NamespaceString> involvedNamespaces_) + : involvedNamespaces(std::move(involvedNamespaces_)), + _bypassDocumentValidation(request_.getBypassDocumentValidation().value_or(false)), + _verbosity(request_.getExplain()), + _hasField() { + _hasField.batchSize = request_.getCursor().getBatchSize().has_value(); + _hasField.bypassDocumentValidation = request_.getBypassDocumentValidation().has_value(); + _hasField.explain = request_.getExplain().has_value(); + _hasField.passthroughToShard = request_.getPassthroughToShard().has_value(); +} + + +void AggCmdComponents::HashValue(absl::HashState state) const { + // The hashing for verbosity in this branch needed to be different because the compiler was + // complaining about the different wrappers. This is not important since this computation is + // only used locally in memory on a single machine, and the query shape is still stable. + auto verbosity = + _hasField.explain ? std::string(ExplainOptions::verbosityString(_verbosity.value())) : ""; + state = absl::HashState::combine(std::move(state), + _bypassDocumentValidation, + _hasField.batchSize, + _hasField.bypassDocumentValidation, + verbosity, + _hasField.explain, + _hasField.passthroughToShard); + // We don't need to add 'involvedNamespaces' here since they are already tracked/duplicated in + // the Pipeline component of the query shape. We just expose them here for ease of + // analysis/querying. +} + +void AggCmdComponents::appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const { + + // otherNss + if (!involvedNamespaces.empty()) { + BSONArrayBuilder otherNss = bob.subarrayStart(kOtherNssFieldName); + for (const auto& nss : involvedNamespaces) { + BSONObjBuilder otherNsEntryBob = otherNss.subobjStart(); + shape_helpers::appendNamespaceShape(otherNsEntryBob, nss, opts); + otherNsEntryBob.doneFast(); + } + otherNss.doneFast(); + } + + // bypassDocumentValidation + if (_hasField.bypassDocumentValidation) { + bob.append(AggregateCommandRequest::kBypassDocumentValidationFieldName, + _bypassDocumentValidation); + } + + // We don't store the specified batch size values since they don't matter. + // Provide an arbitrary literal long here. + + tassert(78429, + "Serialization policy not supported - original values have been discarded", + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged); + + if (_hasField.batchSize) { + // cursor + BSONObjBuilder cursorInfo = bob.subobjStart(AggregateCommandRequest::kCursorFieldName); + opts.appendLiteral(&cursorInfo, SimpleCursorOptions::kBatchSizeFieldName, 0ll); + cursorInfo.doneFast(); + } + + if (_hasField.explain) { + // The verbosity can be explicitly set by using the .explain() command, but when using the + // flag {explain: true} it is set to 'queryPlanner'. + bob.append(AggregateCommandRequest::kExplainFieldName, + ExplainOptions::verbosityString(_verbosity.value())); + } + + // The values here don't matter (assuming we're not using the 'kUnchanged' policy). + tassert(8949601, + "Serialization policy not supported - original values have been discarded", + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged); + if (_hasField.passthroughToShard) { + BSONObjBuilder passthroughToShardInfo = + bob.subobjStart(AggregateCommandRequest::kPassthroughToShardFieldName); + static const PassthroughToShardOptions representativePassthroughOptions = []() { + PassthroughToShardOptions passthroughOpts; + // The value doesn't matter since we will only use this for shapified output. + passthroughOpts.setShard("?"); + return passthroughOpts; + }(); + representativePassthroughOptions.serialize(&passthroughToShardInfo, opts); + passthroughToShardInfo.doneFast(); + } +} + +size_t AggCmdComponents::size() const { + return sizeof(AggCmdComponents) + + std::accumulate(involvedNamespaces.begin(), + involvedNamespaces.end(), + 0, + [](int64_t total, const auto& nss) { return total + nss.size(); }); +} + +void AggKey::appendCommandSpecificComponents(BSONObjBuilder& bob, + const SerializationOptions& opts) const { + return _components.appendTo(bob, opts); +} + +AggKey::AggKey(AggregateCommandRequest request, + const Pipeline& pipeline, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + stdx::unordered_set<NamespaceString> involvedNamespaces, + const NamespaceString& origNss, + query_shape::CollectionType collectionType) + : Key(expCtx->opCtx, + std::make_unique<query_shape::AggCmdShape>( + request, origNss, involvedNamespaces, pipeline, expCtx), + request.getHint(), + request.getReadConcern(), + request.getMaxTimeMS().has_value(), + collectionType), + _components(request, std::move(involvedNamespaces)) {} + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/agg_key.h b/src/mongo/db/query/query_stats/agg_key.h new file mode 100644 index 00000000000..38b80e28006 --- /dev/null +++ b/src/mongo/db/query/query_stats/agg_key.h @@ -0,0 +1,129 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <cstdint> +#include <utility> + +#include <absl/container/node_hash_map.h> +#include <boost/move/utility_core.hpp> +#include <boost/none.hpp> +#include <boost/optional/optional.hpp> +#include <boost/smart_ptr/intrusive_ptr.hpp> + +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/collection_type.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/pipeline/aggregate_command_gen.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/pipeline.h" +#include "mongo/db/pipeline/variables.h" +#include "mongo/db/query/explain_options.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_stats/key.h" + +namespace mongo::query_stats { + +/** + * Struct representing the aggregate command's unique arguments which should be included in the + * query stats key. + */ +struct AggCmdComponents : public SpecificKeyComponents { + static constexpr StringData kOtherNssFieldName = "otherNss"_sd; + + AggCmdComponents(const AggregateCommandRequest&, + stdx::unordered_set<NamespaceString> involvedNamespaces); + + void HashValue(absl::HashState state) const final; + + void appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const; + + size_t size() const; + + stdx::unordered_set<NamespaceString> involvedNamespaces; + bool _bypassDocumentValidation; + const boost::optional<mongo::ExplainOptions::Verbosity> _verbosity; + + // This anonymous struct represents the presence of the member variables as C++ bit fields. + // In doing so, each of these boolean values takes up 1 bit instead of 1 byte. + struct HasField { + HasField() : batchSize(false), bypassDocumentValidation(false), explain(false) {} + bool batchSize : 1; + bool bypassDocumentValidation : 1; + bool explain : 1; + bool passthroughToShard : 1; + } _hasField; +}; + +/** + * Handles shapification for AggregateCommandRequests. Requires a pre-parsed pipeline in order to + * avoid parsing the raw pipeline multiple times, but users should be sure to provide a + * non-optimized pipeline. + */ +class AggKey final : public Key { +public: + AggKey(AggregateCommandRequest request, + const Pipeline& pipeline, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + stdx::unordered_set<NamespaceString> involvedNamespaces, + const NamespaceString& origNss, + query_shape::CollectionType collectionType = query_shape::CollectionType::kUnknown); + + const SpecificKeyComponents& specificComponents() const final { + return _components; + } + + // The default implementation of hashing for smart pointers is not a good one for our purposes. + // Here we overload them to actually take the hash of the object, rather than hashing the + // pointer itself. + template <typename H> + friend H AbslHashValue(H h, const std::unique_ptr<const AggKey>& key) { + return H::combine(std::move(h), *key); + } + template <typename H> + friend H AbslHashValue(H h, const std::shared_ptr<const AggKey>& key) { + return H::combine(std::move(h), *key); + } + + +protected: + void appendCommandSpecificComponents(BSONObjBuilder& bob, + const SerializationOptions& opts) const final override; + +private: + const AggCmdComponents _components; +}; +static_assert( + sizeof(AggKey) == sizeof(Key) + sizeof(AggCmdComponents), + "If the class' members have changed, this assert may need to be updated with a new value."); +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/agg_key_test.cpp b/src/mongo/db/query/query_stats/agg_key_test.cpp new file mode 100644 index 00000000000..35d0ae20d86 --- /dev/null +++ b/src/mongo/db/query/query_stats/agg_key_test.cpp @@ -0,0 +1,204 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include <boost/smart_ptr/intrusive_ptr.hpp> + +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/pipeline/pipeline.h" +#include "mongo/db/query/query_shape/agg_cmd_shape.h" +#include "mongo/db/query/query_stats/agg_key.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/idl/basic_types.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/intrusive_counter.h" + +namespace mongo::query_stats { + +namespace { + +static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl"); + +static constexpr auto collectionType = query_shape::CollectionType::kCollection; + +class AggKeyTest : public ServiceContextTest { +public: + static std::unique_ptr<const Key> makeAggKeyFromRawPipeline( + const std::vector<BSONObj>& rawPipeline) { + auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss); + AggregateCommandRequest acr(kDefaultTestNss); + acr.setPipeline(rawPipeline); + auto pipeline = Pipeline::parse(rawPipeline, expCtx); + return std::make_unique<AggKey>(acr, + *pipeline, + expCtx, + pipeline->getInvolvedCollections(), + acr.getNamespace(), + collectionType); + } + size_t namespaceSize(stdx::unordered_set<NamespaceString> involvedNamespaces) { + return std::accumulate(involvedNamespaces.begin(), + involvedNamespaces.end(), + 0, + [](int64_t total, const auto& nss) { return total + nss.size(); }); + } +}; + +TEST_F(AggKeyTest, SizeOfAggCmdComponents) { + auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss); + auto rawPipeline = {fromjson(R"({ + $match: { + foo: { $in: ["a", "b"] }, + bar: { $gte: { $date: "2022-01-01T00:00:00Z" } } + } + })")}; + + AggregateCommandRequest acr(kDefaultTestNss); + acr.setPipeline(rawPipeline); + auto pipeline = Pipeline::parse(rawPipeline, expCtx); + auto namespaces = pipeline->getInvolvedCollections(); + auto aggComponents = std::make_unique<AggCmdComponents>(acr, namespaces); + + const auto minimumSize = sizeof(SpecificKeyComponents) + + sizeof(stdx::unordered_set<NamespaceString>) + 2 /*size for bool and HasField*/ + + sizeof(boost::optional<mongo::ExplainOptions::Verbosity>) + namespaceSize(namespaces); + ASSERT_GTE(aggComponents->size(), minimumSize); + ASSERT_LTE(aggComponents->size(), minimumSize + 8 /*padding*/); +} + +TEST_F(AggKeyTest, EquivalentAggCmdComponentSizes) { + auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss); + auto rawPipeline = {fromjson(R"({ + $match: { + foo: { $in: ["a", "b"] }, + bar: { $gte: { $date: "2022-01-01T00:00:00Z" } } + } + })")}; + + // Set different values in the command request. + AggregateCommandRequest acrBypassTrue(kDefaultTestNss); + acrBypassTrue.setPipeline(rawPipeline); + acrBypassTrue.setBypassDocumentValidation(true); + auto pipeline = Pipeline::parse(rawPipeline, expCtx); + auto namespaces = pipeline->getInvolvedCollections(); + auto aggComponentsBypassTrue = std::make_unique<AggCmdComponents>(acrBypassTrue, namespaces); + + + AggregateCommandRequest acrBypassFalse(kDefaultTestNss); + acrBypassFalse.setPipeline(rawPipeline); + acrBypassFalse.setBypassDocumentValidation(false); + auto aggComponentsBypassFalse = std::make_unique<AggCmdComponents>(acrBypassFalse, namespaces); + + ASSERT_EQ(aggComponentsBypassTrue->size(), aggComponentsBypassFalse->size()); +} + +TEST_F(AggKeyTest, DifferentAggCmdComponentSizes) { + auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss); + auto rawPipeline = {fromjson(R"({ + $match: { + foo: { $in: ["a", "b"] }, + bar: { $gte: { $date: "2022-01-01T00:00:00Z" } } + } + })")}; + + AggregateCommandRequest acr(kDefaultTestNss); + acr.setPipeline(rawPipeline); + // Manually creating different namespaces for testing purposes. + const auto namespaceStringOne = NamespaceString("testDB.testColl1"); + const auto namespaceStringTwo = NamespaceString("testDB.testColl2"); + + stdx::unordered_set<NamespaceString> smallNamespaces; + smallNamespaces.insert(namespaceStringOne); + + stdx::unordered_set<NamespaceString> largeNamespaces; + largeNamespaces.insert(namespaceStringOne); + largeNamespaces.insert(namespaceStringTwo); + + auto smallAggComponents = std::make_unique<AggCmdComponents>(acr, smallNamespaces); + auto largeAggComponents = std::make_unique<AggCmdComponents>(acr, largeNamespaces); + + ASSERT_LT(namespaceSize(smallNamespaces), namespaceSize(largeNamespaces)); + ASSERT_LT(smallAggComponents->size(), largeAggComponents->size()); +} + +// Testing item in opCtx that should impact key size. +TEST_F(AggKeyTest, SizeOfAggKeyWithAndWithoutWriteConcern) { + auto rawPipeline = {fromjson(R"({ + $match: { + foo: { $in: ["a", "b"] }, + bar: { $gte: { $date: "2022-01-01T00:00:00Z" } } + } + })")}; + + auto keyWithoutComment = makeAggKeyFromRawPipeline(rawPipeline); + + auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss); + AggregateCommandRequest acrWithComment(kDefaultTestNss); + acrWithComment.setPipeline(rawPipeline); + expCtx->opCtx->setComment(BSON("comment" + << " foo")); + auto pipelineWithComment = Pipeline::parse(rawPipeline, expCtx); + auto keyWithComment = std::make_unique<AggKey>(acrWithComment, + *pipelineWithComment, + expCtx, + pipelineWithComment->getInvolvedCollections(), + acrWithComment.getNamespace(), + collectionType); + + ASSERT_LT(keyWithoutComment->size(), keyWithComment->size()); +} + +// Testing item in command request that should impact key size. +TEST_F(AggKeyTest, SizeOfAggKeyWithAndWithoutReadConcern) { + auto rawPipeline = {fromjson(R"({ + $match: { + foo: { $in: ["a", "b"] }, + bar: { $gte: { $date: "2022-01-01T00:00:00Z" } } + } + })")}; + + auto keyWithoutReadConcern = makeAggKeyFromRawPipeline(rawPipeline); + + auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss); + AggregateCommandRequest acrWithReadConcern(kDefaultTestNss); + acrWithReadConcern.setPipeline(rawPipeline); + acrWithReadConcern.setReadConcern(fromjson(R"({level: "local"})")); + auto pipelineWithReadConcern = Pipeline::parse(rawPipeline, expCtx); + auto keyWithReadConcern = + std::make_unique<AggKey>(acrWithReadConcern, + *pipelineWithReadConcern, + expCtx, + pipelineWithReadConcern->getInvolvedCollections(), + acrWithReadConcern.getNamespace(), + collectionType); + + ASSERT_LT(keyWithoutReadConcern->size(), keyWithReadConcern->size()); +} +} // namespace +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/aggregate_key_generator.cpp b/src/mongo/db/query/query_stats/aggregate_key_generator.cpp new file mode 100644 index 00000000000..f175df296f2 --- /dev/null +++ b/src/mongo/db/query/query_stats/aggregate_key_generator.cpp @@ -0,0 +1,185 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_stats/aggregate_key_generator.h" + +#include "mongo/db/pipeline/pipeline.h" +#include "mongo/db/query/query_shape.h" +#include "mongo/db/query/serialization_options.h" +#include "mongo/db/query/shape_helpers.h" + +namespace mongo::query_stats { + +BSONObj AggregateKeyGenerator::generate( + OperationContext* opCtx, + boost::optional<SerializationOptions::TokenizeIdentifierFunc> hmacPolicy) const { + // TODO SERVER-76087 We will likely want to set a flag here to stop $search from calling out + // to mongot. + auto expCtx = makeDummyExpCtx(opCtx); + SerializationOptions opts{LiteralSerializationPolicy::kToDebugTypeString}; + if (hmacPolicy) { + opts.transformIdentifiers = true; + opts.transformIdentifiersCallback = *hmacPolicy; + opts.includePath = true; + opts.verbosity = boost::none; + } + + return makeQueryStatsKey(opts, expCtx); +} + +void AggregateKeyGenerator::appendCommandSpecificComponents( + BSONObjBuilder& bob, const SerializationOptions& opts) const { + // cursor + if (auto param = _request.getCursor().getBatchSize()) { + BSONObjBuilder cursorInfo = bob.subobjStart(AggregateCommandRequest::kCursorFieldName); + opts.appendLiteral(&cursorInfo, + SimpleCursorOptions::kBatchSizeFieldName, + static_cast<long long>(param.get())); + cursorInfo.doneFast(); + } + + // maxTimeMS + if (auto param = _request.getMaxTimeMS()) { + opts.appendLiteral(&bob, + AggregateCommandRequest::kMaxTimeMSFieldName, + static_cast<long long>(param.get())); + } + + // bypassDocumentValidation + if (auto param = _request.getBypassDocumentValidation()) { + opts.appendLiteral( + &bob, AggregateCommandRequest::kBypassDocumentValidationFieldName, bool(param.get())); + } + + // otherNss + if (!_involvedNamespaces.empty()) { + BSONArrayBuilder otherNss = bob.subarrayStart(kOtherNssFieldName); + for (const auto& nss : _involvedNamespaces) { + BSONObjBuilder otherNsEntryBob = otherNss.subobjStart(); + shape_helpers::appendNamespaceShape(otherNsEntryBob, nss, opts); + otherNsEntryBob.doneFast(); + } + otherNss.doneFast(); + } +} + +BSONObj AggregateKeyGenerator::makeQueryStatsKey( + const SerializationOptions& opts, const boost::intrusive_ptr<ExpressionContext>& expCtx) const { + auto pipeline = Pipeline::parse(_request.getPipeline(), expCtx); + return _makeQueryStatsKeyHelper(opts, expCtx, *pipeline); +} + +BSONObj AggregateKeyGenerator::_makeQueryStatsKeyHelper( + const SerializationOptions& opts, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const Pipeline& pipeline) const { + return generateWithQueryShape( + query_shape::extractQueryShape(_request, pipeline, opts, expCtx, _origNss), opts); +} + +namespace { + +int64_t sum(const std::initializer_list<int64_t>& sizes) { + return std::accumulate(sizes.begin(), sizes.end(), 0, std::plus{}); +} + +int64_t size(const std::vector<BSONObj>& objects) { + return std::accumulate(objects.begin(), objects.end(), 0, [](int64_t total, const auto& obj) { + // Include the 'sizeof' to account for the variable number in the vector. + return total + sizeof(BSONObj) + obj.objsize(); + }); +} + +int64_t size(const boost::optional<PassthroughToShardOptions>& passthroughToShardOpts) { + if (!passthroughToShardOpts) { + return 0; + } + return passthroughToShardOpts->getShard().size(); +} + +int64_t size(const boost::optional<ExchangeSpec>& exchange) { + if (!exchange) { + return 0; + } + return sum( + {exchange->getKey().objsize(), + (exchange->getBoundaries() ? size(exchange->getBoundaries().get()) : 0), + (exchange->getConsumerIds() ? 4 * static_cast<int64_t>(exchange->getConsumerIds()->size()) + : 0)}); +} + +int64_t size(const boost::optional<EncryptionInformation>& encryptInfo) { + if (!encryptInfo) { + return 0; + } + tasserted(7659700, + "Unexpected encryption information - not expecting to collect query shape stats on " + "encrypted querys"); +} + +int64_t size(const StringData& str) { + return str.size(); +} + +int64_t size(const boost::optional<BSONObj>& obj) { + return optionalObjSize(obj); +} + +// variadic base case. +template <typename T> +int64_t sumOfSizes(const T& t) { + return size(t); +} + +// variadic recursive case. Making the compiler expand the pluses everywhere to give us good +// formatting at the call site. sumOfSizes(x, y, z) rather than size(x) + size(y) + size(z). +template <typename T, typename... Args> +int64_t sumOfSizes(const T& t, const Args&... args) { + return size(t) + sumOfSizes(args...); +} + +int64_t aggRequestSize(const AggregateCommandRequest& request) { + return sumOfSizes(request.getPipeline(), + request.getLet(), + request.getUnwrappedReadPref(), + request.getExchange(), + request.getPassthroughToShard(), + request.getEncryptionInformation(), + request.getDbName()); +} + +} // namespace + +int64_t AggregateKeyGenerator::doGetSize() const { + return sum({sizeof(*this), + static_cast<int64_t>(_origNss.size()), + optionalObjSize(_initialQueryStatsKey), + aggRequestSize(_request)}); +} +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/aggregated_metric.h b/src/mongo/db/query/query_stats/aggregated_metric.h new file mode 100644 index 00000000000..2e933a17a98 --- /dev/null +++ b/src/mongo/db/query/query_stats/aggregated_metric.h @@ -0,0 +1,79 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <algorithm> +#include <cstdint> + +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/platform/decimal128.h" + +namespace mongo::query_stats { + +/** + * An aggregated metric stores a compressed view of data. It balances the loss of information + * with the reduction in required storage. + */ +struct AggregatedMetric { + + /** + * Aggregate an observed value into the metric. + */ + void aggregate(uint64_t val) { + sum += val; + max = std::max(val, max); + min = std::min(val, min); + sumOfSquares = sumOfSquares.add(Decimal128(val).multiply(Decimal128(val))); + } + + void appendTo(BSONObjBuilder& builder, const StringData& fieldName) const { + BSONObjBuilder metricsBuilder = builder.subobjStart(fieldName); + metricsBuilder.append("sum", (long long)sum); + metricsBuilder.append("max", (long long)max); + metricsBuilder.append("min", (long long)min); + metricsBuilder.append("sumOfSquares", sumOfSquares); + metricsBuilder.done(); + } + + uint64_t sum = 0; + // Default to the _signed_ maximum (which fits in unsigned range) because we cast to + // BSONNumeric when serializing. + uint64_t min = (uint64_t)std::numeric_limits<int64_t>::max; + uint64_t max = 0; + + /** + * The sum of squares along with (an externally stored) count will allow us to compute the + * variance/stddev. + */ + Decimal128 sumOfSquares{}; +}; + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/find_key.cpp b/src/mongo/db/query/query_stats/find_key.cpp new file mode 100644 index 00000000000..437c75aecaa --- /dev/null +++ b/src/mongo/db/query/query_stats/find_key.cpp @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_stats/find_key.h" + +namespace mongo::query_stats { + +void FindCmdComponents::appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const { + + if (_hasField.allowPartialResults) { + bob.append(FindCommandRequest::kAllowPartialResultsFieldName, _allowPartialResults); + } + + // Fields for literal redaction. Adds batchSize, and noCursorTimeOut. + + if (_hasField.noCursorTimeout) { + bob.append(FindCommandRequest::kNoCursorTimeoutFieldName, _noCursorTimeout); + } + + // We don't store the specified batch size value since it doesn't matter. + // Provide an arbitrary literal long here. + tassert(7973602, + "Serialization policy not supported - original values have been discarded", + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged); + + if (_hasField.batchSize) { + opts.appendLiteral(&bob, FindCommandRequest::kBatchSizeFieldName, 0ll); + } +} + +std::unique_ptr<FindCommandRequest> FindKey::reparse(OperationContext* opCtx) const { + auto fcr = + static_cast<const query_shape::FindCmdShape*>(universalComponents()._queryShape.get()) + ->toFindCommandRequest(); + if (_components._hasField.allowPartialResults) + fcr->setAllowPartialResults(_components._allowPartialResults); + if (_components._hasField.noCursorTimeout) + fcr->setNoCursorTimeout(_components._noCursorTimeout); + if (_components._hasField.batchSize) + fcr->setBatchSize(1ll); + return fcr; +} +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/find_key.h b/src/mongo/db/query/query_stats/find_key.h new file mode 100644 index 00000000000..8578a77e573 --- /dev/null +++ b/src/mongo/db/query/query_stats/find_key.h @@ -0,0 +1,152 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <memory> + +#include "mongo/db/namespace_string.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/query/query_shape/find_cmd_shape.h" +#include "mongo/db/query/query_stats/key.h" + +namespace mongo::query_stats { + +struct FindCmdComponents : public SpecificKeyComponents { + FindCmdComponents(const FindCommandRequest* findCmd) + : _allowPartialResults(findCmd->getAllowPartialResults().value_or(false)), + _noCursorTimeout(findCmd->getNoCursorTimeout().value_or(false)), + _hasField() { + _hasField.batchSize = findCmd->getBatchSize().has_value(); + _hasField.allowPartialResults = findCmd->getAllowPartialResults().has_value(); + _hasField.noCursorTimeout = findCmd->getNoCursorTimeout().has_value(); + } + + std::size_t size() const { + return sizeof(FindCmdComponents); + } + + void HashValue(absl::HashState state) const final { + absl::HashState::combine( + std::move(state), _hasField, _allowPartialResults, _noCursorTimeout); + } + + void appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const; + + // Avoid using boost::optional here because it creates extra padding at the beginning of the + // struct. Since each QueryStatsEntry can have its own FindKey, it's better to + // minimize the struct's size as much as possible. + + // Preserved literal. + bool _allowPartialResults; + bool _noCursorTimeout; + + // This anonymous struct represents the presence of the member variables as C++ bit fields. + // In doing so, each of these boolean values takes up 1 bit instead of 1 byte. + struct HasField { + HasField() : batchSize(false), allowPartialResults(false), noCursorTimeout(false) {} + bool batchSize : 1; + bool allowPartialResults : 1; + bool noCursorTimeout : 1; + bool operator==(const HasField& other) const { + return batchSize == other.batchSize && + allowPartialResults == other.allowPartialResults && + noCursorTimeout == other.noCursorTimeout; + } + + } _hasField; + + template <typename H> + friend H AbslHashValue(H h, const HasField& hasField) { + return H::combine(std::move(h), + hasField.batchSize, + hasField.noCursorTimeout, + hasField.allowPartialResults); + } +}; + +// This static assert checks to ensure that the struct's size is changed thoughtfully. If adding +// or otherwise changing the members, this assert may be updated with care. +static_assert( + // Expecting two bytes for allowPartialResults and noCursorTimeout, and another + // byte for _hasField. For alignment reasons (alignment is 8 bytes here), this means the trailer + // will bring up the total bytecount to a multiple of 8. + sizeof(FindCmdComponents) <= sizeof(SpecificKeyComponents) + 8, + "Size of FindCmdComponents is too large! " + "Make sure that the struct has been align- and padding-optimized. " + "If the struct's members have changed, this assert may need to be updated with a new " + "value."); + +class FindKey final : public Key { +public: + FindKey(const boost::intrusive_ptr<ExpressionContext>& expCtx, + const ParsedFindCommand& request, + query_shape::CollectionType collectionType = query_shape::CollectionType::kUnknown) + : Key(expCtx->opCtx, + std::make_unique<query_shape::FindCmdShape>(request, expCtx), + request.findCommandRequest->getHint(), + request.findCommandRequest->getReadConcern(), + request.findCommandRequest->getMaxTimeMS().has_value(), + collectionType), + _components(request.findCommandRequest.get()) {} + + // The default implementation of hashing for smart pointers is not a good one for our purposes. + // Here we overload them to actually take the hash of the object, rather than hashing the + // pointer itself. + template <typename H> + friend H AbslHashValue(H h, const std::unique_ptr<const FindKey>& key) { + return H::combine(std::move(h), *key); + } + template <typename H> + friend H AbslHashValue(H h, const std::shared_ptr<const FindKey>& key) { + return H::combine(std::move(h), *key); + } + + const SpecificKeyComponents& specificComponents() const { + return _components; + } + +private: + void appendCommandSpecificComponents(BSONObjBuilder& bob, + const SerializationOptions& opts) const final { + _components.appendTo(bob, opts); + } + + std::unique_ptr<FindCommandRequest> reparse(OperationContext* opCtx) const; + + FindCmdComponents _components; +}; + +// This static assert checks to ensure that the struct's size is changed thoughtfully. If adding +// or otherwise changing the members, this assert may be updated with care. +static_assert(sizeof(FindKey) == sizeof(Key) + sizeof(FindCmdComponents), + "If the class' members have changed, this assert may need to be updated with a new " + "value and the size calcuation will need to be changed."); + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/find_key_test.cpp b/src/mongo/db/query/query_stats/find_key_test.cpp new file mode 100644 index 00000000000..6c34ba5a606 --- /dev/null +++ b/src/mongo/db/query/query_stats/find_key_test.cpp @@ -0,0 +1,133 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/parsed_find_command.h" +#include "mongo/db/query/query_stats/find_key.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_stats { + +namespace { +static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl"); + +static constexpr auto collectionType = query_shape::CollectionType::kCollection; + +class FindKeyTest : public ServiceContextTest { +public: + static std::unique_ptr<const Key> makeFindKeyFromQuery(const BSONObj& filter) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcr)})); + return std::make_unique<FindKey>(expCtx, *parsedFind, collectionType); + } +}; + +TEST_F(FindKeyTest, SizeOfFindCmdComponents) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + auto query = BSON("query" << 1 << "xEquals" << 42); + fcr->setFilter(query.getOwned()); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcr)})); + auto findComponents = std::make_unique<FindCmdComponents>(parsedFind->findCommandRequest.get()); + + ASSERT_GTE(findComponents->size(), sizeof(SpecificKeyComponents) + 3 /*bools and HasField*/); + ASSERT_LTE(findComponents->size(), + sizeof(SpecificKeyComponents) + 8 /*bools, HasField, and padding*/); +} + +TEST_F(FindKeyTest, EquivalentFindCmdComponentsSizes) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto query = BSON("query" << 1 << "xEquals" << 42); + + // Set different fields in the find commands. + auto fcrCursorTimeout = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcrCursorTimeout->setFilter(query.getOwned()); + fcrCursorTimeout->setNoCursorTimeout(true); + auto parsedFindCursorTimeout = + uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrCursorTimeout)})); + auto findComponentsCursorTimeout = + std::make_unique<FindCmdComponents>(parsedFindCursorTimeout->findCommandRequest.get()); + + auto fcrAllowPartial = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcrAllowPartial->setFilter(query.getOwned()); + fcrAllowPartial->setAllowPartialResults(true); + auto parsedFindAllowPartial = + uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrAllowPartial)})); + auto findComponentsAllowPartial = + std::make_unique<FindCmdComponents>(parsedFindAllowPartial->findCommandRequest.get()); + + ASSERT_EQ(findComponentsCursorTimeout->size(), findComponentsAllowPartial->size()); +} + +// Testing item from opCtx that should impact key size. +TEST_F(FindKeyTest, SizeOfFindKeyWithAndWithoutComment) { + auto query = BSON("query" << 1 << "xEquals" << 42); + + auto keyWithoutComment = makeFindKeyFromQuery(query); + + auto opCtx = makeOperationContext(); + auto fcrWithComment = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcrWithComment->setFilter(query.getOwned()); + opCtx->setComment(BSON("comment" + << " foo")); + auto expCtxWithComment = make_intrusive<ExpressionContext>( + opCtx.get(), *fcrWithComment, nullptr, true /* mayDbProfile*/); + auto parsedFindWithComment = + uassertStatusOK(parsed_find_command::parse(expCtxWithComment, {std::move(fcrWithComment)})); + auto keyWithComment = std::make_unique<query_stats::FindKey>( + expCtxWithComment, *parsedFindWithComment, collectionType); + + ASSERT_LT(keyWithoutComment->size(), keyWithComment->size()); +} + +// Testing item from command request that should impact key size. +TEST_F(FindKeyTest, SizeOfFindKeyWithAndWithoutReadConcern) { + auto query = BSON("query" << 1 << "xEquals" << 42); + + auto keyWithoutReadConcern = makeFindKeyFromQuery(query); + + auto expCtxWithReadConcern = make_intrusive<ExpressionContextForTest>(); + auto fcrWithReadConcern = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcrWithReadConcern->setFilter(query.getOwned()); + fcrWithReadConcern->setReadConcern(fromjson(R"({level: "local"})")); + auto parsedFindWithReadConcern = uassertStatusOK( + parsed_find_command::parse(expCtxWithReadConcern, {std::move(fcrWithReadConcern)})); + auto keyWithReadConcern = std::make_unique<query_stats::FindKey>( + expCtxWithReadConcern, *parsedFindWithReadConcern, collectionType); + + ASSERT_LT(keyWithoutReadConcern->size(), keyWithReadConcern->size()); +} + + +} // namespace +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/key.cpp b/src/mongo/db/query/query_stats/key.cpp new file mode 100644 index 00000000000..f282ef21a2a --- /dev/null +++ b/src/mongo/db/query/query_stats/key.cpp @@ -0,0 +1,223 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_stats/key.h" + +#include "mongo/db/query/query_stats/query_stats_helpers.h" +#include "mongo/rpc/metadata/client_metadata.h" + +namespace mongo::query_stats { + +namespace { + +BSONObj scrubHighCardinalityFields(const ClientMetadata* clientMetadata) { + if (!clientMetadata) { + return BSONObj(); + } + return clientMetadata->documentWithoutMongosInfo(); +} + +BSONObj shapifyReadPreference(boost::optional<BSONObj> readPreference) { + if (!readPreference) { + return BSONObj(); + } + + BSONObjBuilder builder; + for (const auto& elem : *readPreference) { + if (elem.fieldNameStringData() != "tags"_sd) { + builder.append(elem); + continue; + } + + // Sort the $readPreference tags so that different orderings still map to one query stats + // store key. + BSONObjSet sortedTags = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); + for (const auto& tag : elem.Array()) { + sortedTags.insert(tag.Obj()); + } + + BSONArrayBuilder arrBuilder(builder.subarrayStart("tags"_sd)); + for (const auto& tag : sortedTags) { + arrBuilder.append(tag); + } + } + return builder.obj(); +} + +} // namespace + +UniversalKeyComponents::UniversalKeyComponents(std::unique_ptr<query_shape::Shape> queryShape, + const ClientMetadata* clientMetadata, + boost::optional<BSONObj> commentObj, + boost::optional<BSONObj> hint, + boost::optional<BSONObj> readPreference, + boost::optional<BSONObj> writeConcern, + boost::optional<BSONObj> readConcern, + std::unique_ptr<APIParameters> apiParams, + query_shape::CollectionType collectionType, + bool maxTimeMS) + : _clientMetaData(scrubHighCardinalityFields(clientMetadata)), + _commentObj(commentObj.value_or(BSONObj()).getOwned()), + _hintObj(hint.value_or(BSONObj()).getOwned()), + _writeConcern(writeConcern.value_or(BSONObj()).getOwned()), + _shapifiedReadPreference(shapifyReadPreference(readPreference)), + _shapifiedReadConcern(shapifyReadConcern(readConcern.value_or(BSONObj()))), + _comment(commentObj ? _commentObj.firstElement() : BSONElement()), + _queryShape(std::move(queryShape)), + _apiParams(std::move(apiParams)), + _clientMetaDataHash(clientMetadata ? clientMetadata->hashWithoutMongosInfo() + : simpleHash(BSONObj())), + _collectionType(collectionType), + _hasField() { + _hasField.clientMetaData = bool(clientMetadata); + _hasField.comment = bool(commentObj); + _hasField.hint = bool(hint); + _hasField.readPreference = bool(readPreference); + _hasField.writeConcern = bool(writeConcern); + _hasField.readConcern = bool(readConcern); + _hasField.maxTimeMS = maxTimeMS; + tassert(7973600, "shape must not be null", _queryShape); +} + +BSONObj UniversalKeyComponents::shapifyReadConcern(const BSONObj& readConcern, + const SerializationOptions& opts) { + // Read concern should not be considered a literal. + // afterClusterTime is distinct for every operation with causal consistency enabled. We + // normalize it in order not to blow out the queryStats store cache. + if (readConcern["afterClusterTime"].eoo() && readConcern["atClusterTime"].eoo()) { + return readConcern.copy(); + } else { + BSONObjBuilder bob; + + if (auto levelElem = readConcern["level"]) { + bob.append(levelElem); + } + if (auto afterClusterTime = readConcern["afterClusterTime"]) { + opts.appendLiteral(&bob, "afterClusterTime", afterClusterTime); + } + if (auto atClusterTime = readConcern["atClusterTime"]) { + opts.appendLiteral(&bob, "atClusterTime", atClusterTime); + } + return bob.obj(); + } +} + +size_t UniversalKeyComponents::size() const { + return sizeof(*this) + _queryShape->size() + + (_apiParams ? sizeof(*_apiParams) + shape_helpers::optionalSize(_apiParams->getAPIVersion()) + : 0) + + _hintObj.objsize() + (_hasField.clientMetaData ? _clientMetaData.objsize() : 0) + + _commentObj.objsize() + + (_hasField.readPreference ? _shapifiedReadPreference.objsize() : 0) + + (_hasField.readConcern ? _shapifiedReadConcern.objsize() : 0) + + (_hasField.writeConcern ? _writeConcern.objsize() : 0); +} + +void UniversalKeyComponents::appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const { + if (_hasField.comment) { + opts.appendLiteral(&bob, "comment", _comment); + } + + if (_hasField.readConcern) { + auto readConcernToAppend = _shapifiedReadConcern; + if (opts != SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + // The options aren't the same as the first time we shapified, so re-computation is + // necessary (e.g. use "?timestamp" instead of the representative Timestamp(0, 0)). + readConcernToAppend = shapifyReadConcern(_shapifiedReadConcern, opts); + } + bob.append("readConcern", readConcernToAppend); + } + + if (const auto& apiVersion = _apiParams->getAPIVersion()) { + bob.append("apiVersion", apiVersion.value()); + } + + if (const auto& apiStrict = _apiParams->getAPIStrict()) { + bob.append("apiStrict", apiStrict.value()); + } + + if (const auto& apiDeprecationErrors = _apiParams->getAPIDeprecationErrors()) { + bob.append("apiDeprecationErrors", apiDeprecationErrors.value()); + } + + if (_hasField.readPreference) { + bob.append("$readPreference", _shapifiedReadPreference); + } + + if (_hasField.writeConcern) { + bob.append("writeConcern", _writeConcern); + } + + if (_hasField.clientMetaData) { + bob.append("client", _clientMetaData); + } + if (_collectionType > query_shape::CollectionType::kUnknown) { + bob.append("collectionType", toStringData(_collectionType)); + } + if (!_hintObj.isEmpty()) { + bob.append("hint", shape_helpers::extractHintShape(_hintObj, opts)); + } + if (_hasField.maxTimeMS) { + opts.appendLiteral(&bob, "maxTimeMS", 0ll); + } +} +Key::Key(OperationContext* opCtx, + std::unique_ptr<query_shape::Shape> queryShape, + boost::optional<BSONObj> hint, + boost::optional<BSONObj> readConcern, + bool maxTimeMS, + query_shape::CollectionType collectionType) + : _universalComponents( + std::move(queryShape), + ClientMetadata::get(opCtx->getClient()), + opCtx->getCommentOwnedCopy(), + hint, + ReadPreferenceSetting::get(opCtx).usedDefaultReadPrefValue() + ? boost::none + : boost::make_optional(ReadPreferenceSetting::get(opCtx).toInnerBSON()), + opCtx->getWriteConcern().isImplicitDefaultWriteConcern() + ? boost::none + : boost::make_optional(opCtx->getWriteConcern().toBSON()), + readConcern, + std::make_unique<APIParameters>(APIParameters::get(opCtx)), + collectionType, + maxTimeMS) {} + +BSONObj Key::toBson(OperationContext* opCtx, const SerializationOptions& opts) const { + BSONObjBuilder bob; + + // We'll take care of appending this one outside of the appendTo() call below since it needs + // an OperationContext in some re-parsing cases. The rest is simpler. + bob.append("queryShape", _universalComponents._queryShape->toBson(opCtx, opts)); + + _universalComponents.appendTo(bob, opts); + appendCommandSpecificComponents(bob, opts); + return bob.obj(); +} +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/key.h b/src/mongo/db/query/query_stats/key.h new file mode 100644 index 00000000000..bb83fca82ed --- /dev/null +++ b/src/mongo/db/query/query_stats/key.h @@ -0,0 +1,304 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <memory> + +#include "mongo/bson/bsonobj.h" +#include "mongo/db/api_parameters.h" +#include "mongo/db/collection_type.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/db/query/query_shape/shape_helpers.h" +#include "mongo/db/query/query_stats/transform_algorithm_gen.h" +#include "mongo/rpc/metadata/client_metadata.h" + +namespace mongo::query_stats { + +/** + * A struct holding pieces of the command request that are a component of the query stats store key + * and are options/arguments to all supported query stats commands. + * + * This struct (and the SpecificKeyComponents) are split out as a separate inheritence hierarchy to + * make it easier to ensure each piece is hashed without sub-classes needing to enumerate the parent + * class's member variables. + */ +struct UniversalKeyComponents { + UniversalKeyComponents(std::unique_ptr<query_shape::Shape> queryShape, + const ClientMetadata* clientMetadata, + boost::optional<BSONObj> commentObj, + boost::optional<BSONObj> hint, + boost::optional<BSONObj> readPreference, + boost::optional<BSONObj> writeConcern, + boost::optional<BSONObj> readConcern, + std::unique_ptr<APIParameters> apiParams, + query_shape::CollectionType collectionType, + bool maxTimeMS); + /** + * Returns a copy of the read concern object. If there is an "afterClusterTime" or + * "atClusterTime" component, the timestamp is shapified according to 'opts'. + */ + static BSONObj shapifyReadConcern( + const BSONObj& readConcern, + const SerializationOptions& opts = + SerializationOptions::kRepresentativeQueryShapeSerializeOptions); + + size_t size() const; + + void appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const; + + // Avoid using boost::optional here because it creates extra padding at the beginning of the + // struct. Since each QueryStatsEntry has its own Key subclass, it's better to minimize + // the struct's size as much as possible. + + BSONObj _clientMetaData; // Preserve this value. + BSONObj _commentObj; // Shapify this value. + BSONObj _hintObj; // Preserve this value. + BSONObj _writeConcern; // Preserve this value. + + // Preserved literal except value of 'tags' field is sorted. + BSONObj _shapifiedReadPreference; + // Preserved literal except 'afterClusterTime' and 'atClusterTime' are shapified. + BSONObj _shapifiedReadConcern; + + // Separate the possibly-enormous BSONObj from the remaining members + + BSONElement _comment; + + std::unique_ptr<query_shape::Shape> _queryShape; + std::unique_ptr<APIParameters> _apiParams; // Preserve this value in the query shape. + + // Simple hash of the client metadata object. This value is stored separately because it is + // cached on the client to avoid re-computing on every operation. If no client metadata is + // present, this will be the hash of an empty BSON object (otherwise known as 0). + const unsigned long _clientMetaDataHash; + + // This value is not known when run a query is run on mongos over an unsharded collection, so it + // is not set through that code path. + query_shape::CollectionType _collectionType; + + // This anonymous struct represents the presence of the member variables as C++ bit fields. + // In doing so, each of these boolean values takes up 1 bit instead of 1 byte. + struct HasField { + HasField() + : clientMetaData(false), + comment(false), + hint(false), + readPreference(false), + writeConcern(false), + readConcern(false), + maxTimeMS(false) {} + + bool clientMetaData : 1; + bool comment : 1; + bool hint : 1; + bool readPreference : 1; + bool writeConcern : 1; + bool readConcern : 1; + bool maxTimeMS : 1; + } _hasField; +}; + +/** + * A base class for sub-classes to derive from to expose the hashing ability for all of their + * sub-components. + * + * This struct (and the UniversalKeyComponents) are split out as a separate inheritence hierarchy to + * make it easier to ensure each piece is hashed without sub-classes needing to enumerate the parent + * class's member variables. + */ +struct SpecificKeyComponents { + virtual ~SpecificKeyComponents() {} + + virtual void HashValue(absl::HashState state) const = 0; + + /** + * Sub-classes should implement this to report how much memory is used. This is important to do + * carefully since we are under a budget in the query stats store and use this to do the + * accounting. Implementers should include sizeof(*derivedThis) and be sure to also include the + * size of any owned pointer-like objects such as BSONObj or NamespaceString which are + * indirectly using memory elsehwhere. + * + * We cannot just use sizeof() because there are some variable size data members (like BSON + * objects) which depend on the particular instance. + */ + virtual size_t size() const = 0; +}; + +template <typename H> +H AbslHashValue(H state, const SpecificKeyComponents& value) { + value.HashValue(absl::HashState::Create(&state)); + return std::move(state); +} + +template <typename H> +H AbslHashValue(H h, const UniversalKeyComponents& components) { + return H::combine(std::move(h), + *components._queryShape, + components._clientMetaDataHash, + // Note we use the comment's type in the hash function. + components._comment.type(), + simpleHash(components._hintObj), + simpleHash(components._shapifiedReadPreference), + simpleHash(components._writeConcern), + simpleHash(components._shapifiedReadConcern), + components._apiParams ? APIParameters::Hash{}(*components._apiParams) : 0, + components._collectionType, + components._hasField); +} + +template <typename H> +H AbslHashValue(H h, const UniversalKeyComponents::HasField& hasField) { + return H::combine(std::move(h), + hasField.clientMetaData, + hasField.comment, + hasField.hint, + hasField.readPreference, + hasField.writeConcern, + hasField.readConcern, + hasField.maxTimeMS); +} + + +// This static assert checks to ensure that the struct's size is changed thoughtfully. If adding +// or otherwise changing the members, this assert may be updated with care. +static_assert( + sizeof(UniversalKeyComponents) <= sizeof(query_shape::Shape) + 6 * sizeof(BSONObj) + + sizeof(BSONElement) + sizeof(std::unique_ptr<APIParameters>) + + sizeof(query_shape::CollectionType) + sizeof(query_shape::QueryShapeHash) + + sizeof(int64_t), + "Size of Key is too large! " + "Make sure that the struct has been align- and padding-optimized. " + "If the struct's members have changed, this assert may need to be updated with a new value."); + +/** + * An abstract base class representing a query stats store key for a given request. All query stats + * store entries should include some common elements, tracked in `_universalComponents`. For + * example, everything tracked must have a `query_shape::Shape`. + * + * Subclasses can add more components to include as discriminating factors in which entries should + * be tracked separately. For example, two find commands which are identical except in their read + * concern should be tracked differently. Maybe they will have quite different performance + * characteristics or help us determine when the read concern was changed by the client. + * + * The interface to do this is to split out the state/memory for these components as a separate + * struct which can indpendently hash itself and compute its size (both of which are important for + * the query stats store). Subclasses of Key itself should not have any meaningfully sized + * state other than the 'specificComponents().' + */ +class Key { +public: + virtual ~Key() = default; + + /** + * All Keys will share these characteristics as part of their query stats store key. + * Returns an unowned reference so the caller must ensure the result does not outlive this + * Key instance. + */ + const auto& universalComponents() const { + return _universalComponents; + } + + /** + * Different commands will have different components they want to be included in the query stats + * store key. This interface allows them to do so and easily have those components incorporated + * into this key generation and hashing. + */ + virtual const SpecificKeyComponents& specificComponents() const = 0; + + /** + * Materializes the query stats store key. Not expected to be used on ingestion, since we should + * store this object and its components directly in their native C++ data structures - we can + * use the absl::Hash<query_stats::Key>{}() API to look them up. Instead, this may be useful to + * display the key (as it is used for $queryStats) or perhaps one day persist it to storage. + */ + BSONObj toBson(OperationContext* opCtx, const SerializationOptions& opts) const; + + /** + * Convenience function. + */ + query_shape::QueryShapeHash getQueryShapeHash(OperationContext* opCtx) const { + // TODO (future ticket?) should we cache this somewhere else? + return _universalComponents._queryShape->sha256Hash(opCtx); + } + + size_t size() const { + return sizeof(Key) + specificComponents().size() + _universalComponents.size(); + } + + template <typename H> + friend H AbslHashValue(H h, const Key& key) { + return H::combine(std::move(h), key._universalComponents, key.specificComponents()); + } + + // The default implementation of hashing for smart pointers is not a good one for our purposes. + // Here we overload them to actually take the hash of the object, rather than hashing the + // pointer itself. + template <typename H> + friend H AbslHashValue(H h, const std::unique_ptr<const Key>& key) { + return H::combine(std::move(h), *key); + } + template <typename H> + friend H AbslHashValue(H h, const std::shared_ptr<const Key>& key) { + return H::combine(std::move(h), *key); + } + +protected: + /** + * Sub-classes can use this to instantiate a 'real' Key. 'queryShape' must not be null, + * but is tracked as a pointer since it is a virtual class and we want to own it here. + */ + Key(OperationContext* opCtx, + std::unique_ptr<query_shape::Shape> queryShape, + boost::optional<BSONObj> hint, + boost::optional<BSONObj> readConcern, + bool maxTimeMS, + query_shape::CollectionType collectionType = query_shape::CollectionType::kUnknown); + + /** + * With a given BSONObjBuilder, append the command-specific components of the query stats key. + * + * You may be wondering why this API is here rather than as a virtual method on + * CmdSpecificComponents - and that would be because many implementations can involve a re-parse + * of the request if it needs to serialize with different serialization options. This re-parsing + * process often needs the context of things tracked in _universalComponents, which is hard to + * access from the specific components. + */ + virtual void appendCommandSpecificComponents(BSONObjBuilder& bob, + const SerializationOptions& opts) const = 0; + +private: + UniversalKeyComponents _universalComponents; +}; +static_assert( + sizeof(Key) == sizeof(void*) /*vtable ptr*/ + sizeof(UniversalKeyComponents), + "If the class' members have changed, this assert may need to be updated with a new value."); +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/key_test.cpp b/src/mongo/db/query/query_stats/key_test.cpp new file mode 100644 index 00000000000..69359d08bda --- /dev/null +++ b/src/mongo/db/query/query_stats/key_test.cpp @@ -0,0 +1,177 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/bson/bsonelement.h" +#include "mongo/db/collection_type.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/parsed_find_command.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_stats/find_key.h" +#include "mongo/db/query/query_stats/key.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_stats { + +namespace { +static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl"); + + +struct DummyShapeSpecificComponents : public query_shape::CmdSpecificShapeComponents { + DummyShapeSpecificComponents(){}; + void HashValue(absl::HashState state) const {} + size_t size() const final { + return sizeof(DummyShapeSpecificComponents); + } +}; + +class DummyShape : public query_shape::Shape { +public: + DummyShape(NamespaceStringOrUUID nssOrUUID, + BSONObj collation, + DummyShapeSpecificComponents dummyComponents) + : Shape(nssOrUUID, collation) { + components = dummyComponents; + } + + const query_shape::CmdSpecificShapeComponents& specificComponents() const final { + return components; + } + + void appendCmdSpecificShapeComponents(BSONObjBuilder&, + OperationContext*, + const SerializationOptions& opts) const final {} + DummyShapeSpecificComponents components; +}; + +struct DummyKeyComponents : public SpecificKeyComponents { + DummyKeyComponents(){}; + + void HashValue(absl::HashState state) const {} + size_t size() const { + return sizeof(DummyKeyComponents); + } +}; + +class DummyKey : public Key { +public: + DummyKey(OperationContext* opCtx, + std::unique_ptr<query_shape::Shape> queryShape, + boost::optional<BSONObj> hint, + boost::optional<BSONObj> readConcern, + bool maxTimeMS, + query_shape::CollectionType collectionType, + DummyKeyComponents dummyComponents) + : Key(opCtx, std::move(queryShape), hint, readConcern, maxTimeMS, collectionType) { + components = dummyComponents; + } + const SpecificKeyComponents& specificComponents() const { + return components; + }; + void appendCommandSpecificComponents(BSONObjBuilder& bob, + const SerializationOptions& opts) const {}; + DummyKeyComponents components; +}; +class UniversalKeyTest : public ServiceContextTest {}; + +TEST_F(UniversalKeyTest, SizeOfUniversalComponents) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + // Make shape for testing. + auto collation = BSONObj{}; + auto innerComponents = std::make_unique<DummyShapeSpecificComponents>(); + auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents); + + // Gather sizes and create universalComponents. + const auto shapeSize = shape->size(); + auto clientMetadata = ClientMetadata::get(expCtx->opCtx->getClient()); + + auto clientMetadataSize = clientMetadata ? clientMetadata->documentWithoutMongosInfo().objsize() + : BSONObj().objsize(); + + auto apiParams = std::make_unique<APIParameters>(APIParameters::get(expCtx->opCtx)); + const auto apiParamsSize = static_cast<size_t>( + apiParams ? sizeof(*apiParams) + shape_helpers::optionalSize(apiParams->getAPIVersion()) + : 0); + auto universalComponents = + std::make_unique<UniversalKeyComponents>(std::move(shape), + clientMetadata, + BSONObj(), + BSONObj(), + BSONObj(), + BSONObj(), + BSONObj(), + std::move(apiParams), + query_shape::CollectionType::kUnknown, + true); + + const auto minimumUniversalKeyComponentSize = sizeof(std::unique_ptr<query_shape::Shape>) + + (6 * sizeof(BSONObj)) + sizeof(std::unique_ptr<APIParameters>) + sizeof(BSONElement) + + sizeof(query_shape::CollectionType) + sizeof(unsigned long) + 1 /*HasField*/; + ASSERT_GTE(sizeof(UniversalKeyComponents), minimumUniversalKeyComponentSize); + ASSERT_LTE(sizeof(UniversalKeyComponents), minimumUniversalKeyComponentSize + 8 /*padding*/); + + ASSERT_GT(universalComponents->size(), + sizeof(UniversalKeyComponents) + shapeSize + clientMetadataSize + apiParamsSize); + ASSERT_LTE(universalComponents->size(), + sizeof(UniversalKeyComponents) + shapeSize + clientMetadataSize + + (5 * static_cast<size_t>(BSONObj().objsize())) + apiParamsSize); +} + +TEST_F(UniversalKeyTest, SizeOfSpecificComponents) { + auto innerComponents = std::make_unique<DummyShapeSpecificComponents>(); + auto keyComponents = std::make_unique<DummyKeyComponents>(); + + ASSERT_EQ(keyComponents->size(), sizeof(SpecificKeyComponents)); + ASSERT_EQ(sizeof(SpecificKeyComponents), sizeof(void*) /*vtable ptr*/); +} + +TEST_F(UniversalKeyTest, SizeOfKey) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + auto collation = BSONObj{}; + auto innerComponents = std::make_unique<DummyShapeSpecificComponents>(); + auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents); + + auto keyComponents = std::make_unique<DummyKeyComponents>(); + + auto key = std::make_unique<DummyKey>(expCtx->opCtx, + std::move(shape), + BSONObj(), + BSONObj(), + false, + query_shape::CollectionType::kUnknown, + *keyComponents); + ASSERT_EQ(innerComponents->size(), key->specificComponents().size()); + ASSERT_EQ(sizeof(Key), sizeof(UniversalKeyComponents) + sizeof(void*)); + ASSERT_EQ(key->size(), + sizeof(Key) + key->universalComponents().size() + key->specificComponents().size()); +} +} // namespace +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/query_stats.cpp b/src/mongo/db/query/query_stats/query_stats.cpp new file mode 100644 index 00000000000..a8bd49e0533 --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats.cpp @@ -0,0 +1,467 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQueryStats + +#include "mongo/db/query/query_stats/query_stats.h" + +#include "mongo/crypto/hash_block.h" +#include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/locker.h" +#include "mongo/db/curop.h" +#include "mongo/db/exec/projection_executor_builder.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/pipeline/aggregate_command_gen.h" +#include "mongo/db/pipeline/process_interface/stub_mongo_process_interface.h" +#include "mongo/db/query/find_command_gen.h" +#include "mongo/db/query/plan_explainer.h" +#include "mongo/db/query/projection_ast_util.h" +#include "mongo/db/query/projection_parser.h" +#include "mongo/db/query/query_feature_flags_gen.h" +#include "mongo/db/query/query_planner_params.h" +#include "mongo/db/query/query_request_helper.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/db/query/query_stats/query_stats_on_parameter_change.h" +#include "mongo/db/query/sort_pattern.h" +#include "mongo/logv2/log.h" +#include "mongo/rpc/metadata/client_metadata.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/debug_util.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/system_clock_source.h" +#include <optional> + +namespace mongo::query_stats { + +Counter64 queryStatsStoreSizeEstimateBytesMetric; +ServerStatusMetricField<Counter64> displaySizeEstimateMetric( + "queryStats.queryStatsStoreSizeEstimateBytes", &queryStatsStoreSizeEstimateBytesMetric); + + +const Decorable<ServiceContext>::Decoration<std::unique_ptr<QueryStatsStoreManager>> + QueryStatsStoreManager::get = + ServiceContext::declareDecoration<std::unique_ptr<QueryStatsStoreManager>>(); + +const Decorable<ServiceContext>::Decoration<std::unique_ptr<RateLimiting>> + QueryStatsStoreManager::getRateLimiter = + ServiceContext::declareDecoration<std::unique_ptr<RateLimiting>>(); + + +namespace { + +Counter64 queryStatsEvictedMetric; +ServerStatusMetricField<Counter64> displayEvictedMetric("queryStats.numEvicted", + &queryStatsEvictedMetric); +Counter64 queryStatsRateLimitedRequestsMetric; +ServerStatusMetricField<Counter64> displayRateLimitMetric("queryStats.numRateLimitedRequests", + &queryStatsRateLimitedRequestsMetric); +Counter64 queryStatsStoreWriteErrorsMetric; +ServerStatusMetricField<Counter64> displayWriteErrorsMetric( + "queryStats.numQueryStatsStoreWriteErrors", &queryStatsStoreWriteErrorsMetric); + +/** + * Indicates whether or not query stats is enabled via the feature flag. + */ +bool isQueryStatsFeatureEnabled() { + // We need to call isVersionInitialized() first because this could run during startup while the + // FCV is still uninitialized. + if (serverGlobalParams.featureCompatibility.isVersionInitialized()) { + return feature_flags::gFeatureFlagQueryStats.isEnabled( + serverGlobalParams.featureCompatibility); + } + // (Generic FCV reference): This reference is needed to ensure we correctly initialize query + // stats during startup. + return feature_flags::gFeatureFlagQueryStats.isEnabledOnVersion( + multiversion::GenericFCV::kLatest); +} + +/** + * Cap the queryStats store size. + */ +size_t capQueryStatsStoreSize(size_t requestedSize) { + size_t cappedStoreSize = memory_util::capMemorySize( + requestedSize /*requestedSizeBytes*/, 1 /*maximumSizeGB*/, 25 /*percentTotalSystemMemory*/); + // If capped size is less than requested size, the queryStats store has been capped at its + // upper limit. + if (cappedStoreSize < requestedSize) { + LOGV2_DEBUG(7106502, + 1, + "The queryStats store size has been capped", + "cappedSize"_attr = cappedStoreSize); + } + return cappedStoreSize; +} + +/** + * Get the queryStats store size based on the query job's value. + */ +size_t getQueryStatsStoreSize() { + auto status = memory_util::MemorySize::parse(internalQueryStatsCacheSize.get()); + uassertStatusOK(status); + size_t requestedSize = memory_util::convertToSizeInBytes(status.getValue()); + return capQueryStatsStoreSize(requestedSize); +} + +void assertConfigurationAllowed() { + uassert(ErrorCodes::QueryFeatureNotAllowed, + "Cannot configure queryStats store. The feature flag is not enabled. Please restart " + "and specify the feature flag, or upgrade the feature compatibility version to one " + "where it is enabled by default.", + isQueryStatsFeatureEnabled()); +} + +class QueryStatsOnParamChangeUpdaterImpl final : public query_stats_util::OnParamChangeUpdater { +public: + void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) final { + assertConfigurationAllowed(); + auto requestedSize = memory_util::convertToSizeInBytes(memSize); + auto cappedSize = capQueryStatsStoreSize(requestedSize); + auto& queryStatsStoreManager = QueryStatsStoreManager::get(serviceCtx); + size_t numEvicted = queryStatsStoreManager->resetSize(cappedSize); + queryStatsEvictedMetric.increment(numEvicted); + } + + void updateSamplingRate(ServiceContext* serviceCtx, int samplingRate) { + assertConfigurationAllowed(); + QueryStatsStoreManager::getRateLimiter(serviceCtx).get()->setSamplingRate(samplingRate); + } +}; + +ServiceContext::ConstructorActionRegisterer queryStatsStoreManagerRegisterer{ + "QueryStatsStoreManagerRegisterer", [](ServiceContext* serviceCtx) { + // Note: it is possible that this is called before FCV is properly set up. The feature flags + // can only be specified at startup, but the feature compatibility version may change at + // runtime. If the feature compatibility version upgrades at runtime, the feature may now be + // enabled by default, even if the flag was not specified. To allow for this possibility, we + // will always configure a query stats store of the size currently specified by + // 'internalQueryStatsCacheSize', but we will prevent changing its shape or rate limit at + // runtime unless the feature flag is enabled (at whatever current FCV when the + // configuration setParameter command is run). + + query_stats_util::queryStatsStoreOnParamChangeUpdater(serviceCtx) = + std::make_unique<QueryStatsOnParamChangeUpdaterImpl>(); + size_t size = getQueryStatsStoreSize(); + auto&& globalQueryStatsStoreManager = QueryStatsStoreManager::get(serviceCtx); + // Initially the queryStats store used the same number of partitions as the plan cache, that + // is the number of cpu cores. However, with performance investigation we found that when + // the size of the partitions was too large, it took too long to copy out and read one + // partition. We are now capping each partition at 16MB (the largest size a query shape can + // be. If that gives us fewer partitions than we have cores, we set it to match the + // number of cores. The size needs to be cast to a double since we want to round up the + // number of partitions, and therefore need to avoid int division. + size_t numPartitions = std::ceil(double(size) / (16 * 1024 * 1024)); + auto numLogicalCores = ProcessInfo::getNumCores(); + if (numPartitions < numLogicalCores) { + numPartitions = numLogicalCores; + } + + globalQueryStatsStoreManager = + std::make_unique<QueryStatsStoreManager>(size, numPartitions); + auto configuredSamplingRate = internalQueryStatsRateLimit.load(); + QueryStatsStoreManager::getRateLimiter(serviceCtx) = std::make_unique<RateLimiting>( + configuredSamplingRate < 0 ? INT_MAX : configuredSamplingRate, Seconds{1}); + }}; + +/** + * Top-level checks for whether queryStats collection is enabled. If this returns false, we must + * go no further. + */ +bool isQueryStatsEnabled(const ServiceContext* serviceCtx) { + // During initialization, FCV may not yet be setup but queries could be run. We can't + // check whether queryStats should be enabled without FCV, so default to not recording + // those queries. + return isQueryStatsFeatureEnabled() && + QueryStatsStoreManager::get(serviceCtx)->getMaxSize() > 0; +} + +/** + * Internal check for whether we should collect metrics. This checks the rate limiting + * configuration for a global on/off decision and, if enabled, delegates to the rate limiter. + */ +bool shouldCollect(const ServiceContext* serviceCtx) { + // Cannot collect queryStats if sampling rate is not greater than 0. Note that we do not + // increment queryStatsRateLimitedRequestsMetric here since queryStats is entirely disabled. + auto samplingRate = QueryStatsStoreManager::getRateLimiter(serviceCtx)->getSamplingRate(); + if (samplingRate <= 0) { + LOGV2_DEBUG(8473001, + 5, + "sampling rate is <= 0, skipping this request", + "samplingRate"_attr = samplingRate); + return false; + } + // Check if rate limiting allows us to collect queryStats for this request. + if (samplingRate < INT_MAX && + !QueryStatsStoreManager::getRateLimiter(serviceCtx)->handleRequestSlidingWindow()) { + queryStatsRateLimitedRequestsMetric.increment(); + LOGV2_DEBUG(8473002, + 5, + "rate limited this request", + "samplingRate"_attr = samplingRate, + "totalLimited"_attr = queryStatsRateLimitedRequestsMetric.get()); + return false; + } + return true; +} + +void updateStatistics(const QueryStatsStore::Partition& proofOfLock, + QueryStatsEntry& toUpdate, + const uint64_t queryExecMicros, + const uint64_t firstResponseExecMicros, + const uint64_t docsReturned) { + toUpdate.latestSeenTimestamp = Date_t::now(); + toUpdate.lastExecutionMicros = queryExecMicros; + toUpdate.execCount++; + toUpdate.totalExecMicros.aggregate(queryExecMicros); + toUpdate.firstResponseExecMicros.aggregate(firstResponseExecMicros); + toUpdate.docsReturned.aggregate(docsReturned); +} + +} // namespace + +void registerRequest(OperationContext* opCtx, + const NamespaceString& collection, + std::function<std::unique_ptr<Key>(void)> makeKey, + bool willNeverExhaust) { + if (!isQueryStatsEnabled(opCtx->getServiceContext())) { + LOGV2_DEBUG(8473000, + 5, + "not collecting query stats for this request since it is disabled", + "featureEnabled"_attr = isQueryStatsFeatureEnabled()); + return; + } + + // Queries against metadata collections should never appear in queryStats data. + if (collection.isFLE2StateCollection()) { + return; + } + + // Don't record queries from internal clients. + if (opCtx->getClient()->session() && + (opCtx->getClient()->session()->getTags() & transport::Session::kInternalClient)) { + return; + } + + auto& opDebug = CurOp::get(opCtx)->debug(); + + if (opDebug.queryStatsInfo.wasRateLimited) { + LOGV2_DEBUG( + 8288900, + 4, + "Query stats request was previously rate limited. We expect this is a query on a view"); + return; + } + + if (!shouldCollect(opCtx->getServiceContext())) { + opDebug.queryStatsInfo.wasRateLimited = true; + return; + } + + if (opDebug.queryStatsInfo.key) { + // A find() request may have already registered the shapifier. Ie, it's a find command over + // a non-physical collection, eg view, which is implemented by generating an agg pipeline. + LOGV2_DEBUG(7198700, + 2, + "Query stats request shapifier already registered", + "collection"_attr = collection); + return; + } + + opDebug.queryStatsInfo.willNeverExhaust = willNeverExhaust; + // There are a few cases where a query shape can be larger than the original query. For example, + // {$exists: false} in the input query serializes to {$not: {$exists: true}. In rare cases where + // an input query has thousands of clauses, the cumulative bloat that shapification adds results + // in a BSON object that exceeds the 16 MB memory limit. In these cases, we want to exclude the + // original query from queryStats metrics collection and let it execute normally. + try { + opDebug.queryStatsInfo.key = makeKey(); + } catch (const DBException& ex) { + queryStatsStoreWriteErrorsMetric.increment(); + + const auto status = ex.toStatus(); + if (status.code() == ErrorCodes::BSONObjectTooLarge) { + LOGV2_DEBUG(7979400, + 2, + "Query Stats shapification has exceeded the 16 MB memory limit. Metrics " + "will not be collected"); + return; + } + + const auto& cmdObj = CurOp::get(opCtx)->opDescription(); + LOGV2_DEBUG(9423100, + 2, + "Error encountered when creating the Query Stats store key. Metrics will not " + "be collected for this command", + "status"_attr = status, + "command"_attr = cmdObj); + if (kDebugBuild || internalQueryStatsErrorsAreCommandFatal.load()) { + // uassert rather than tassert so that we avoid creating fatal failures on queries that + // were going to fail anyway, but trigger the error here first. A query that ONLY fails + // when query stats is enabled will still be surfaced by the uassert. + // Note that in the former case, these queries will fail with a different error code + // than they would have otherwise. Since this block is only applicable in test + // environments, this is fine. We make this tradeoff because it is desirable to have + // real bugs clearly surfaced as query stats issues. + uasserted(9423101, + str::stream() << "Failed to create query stats store key. Status: " << status + << " Command: " << cmdObj); + } + + return; + } + opDebug.queryStatsInfo.keyHash = absl::Hash<query_stats::Key>{}(*opDebug.queryStatsInfo.key); + // TODO look up this query shape (sub-component of query stats store key) in some new shared + // data structure that the query settings component could share. See if the query SHAPE hash has + // been computed before. If so, record the query shape hash on the opDebug. If not, compute the + // hash and store it there so we can avoid re-doing this for each request. +} + +QueryStatsStore& getQueryStatsStore(OperationContext* opCtx) { + uassert(ErrorCodes::QueryFeatureNotAllowed, + "Query stats is not enabled without the feature flag on and a cache size greater than " + "0 bytes", + isQueryStatsEnabled(opCtx->getServiceContext())); + return QueryStatsStoreManager::get(opCtx->getServiceContext())->getQueryStatsStore(); +} + +void writeQueryStats(OperationContext* opCtx, + boost::optional<size_t> queryStatsKeyHash, + std::unique_ptr<Key> key, + const uint64_t queryExecMicros, + const uint64_t firstResponseExecMicros, + const uint64_t docsReturned, + bool willNeverExhaust) { + // Generally we expect a 'key' to write query stats. However, for a change stream query, we + // expect it has no 'key' after its first writeQueryStats(), but it must have a + // 'queryStatsKeyHash' for its entry to be updated. + // TODO SERVER-89058 Modify comment to include tailable cursors. + if (!key && !(willNeverExhaust && queryStatsKeyHash)) { + return; + } + + // It's possible that query stats was enabled in registerRequest but has been disabled since + // (e.g., by FCV downgrade or setting the store size to 0). Rather than calling + // getQueryStatsStore (which would trigger a uassert if queryStats is disabled), we return and + // log a message if query stats is disabled, and otherwise grab the query stats store directly. + if (!isQueryStatsEnabled(opCtx->getServiceContext())) { + LOGV2_DEBUG(8456700, + 2, + "Query stats was enabled when the command started but is now disabled. " + "Metrics will not be collected.", + "queryStatsKeyHash"_attr = queryStatsKeyHash); + return; + } + auto&& queryStatsStore = + QueryStatsStoreManager::get(opCtx->getServiceContext())->getQueryStatsStore(); + if (key) { + dassert(absl::Hash<query_stats::Key>{}(*key) == queryStatsKeyHash, + "Expecting query stats key to hash to the given hash. Is the OpCtx state being " + "incorrectly re-used?"); + } + auto&& [statusWithMetrics, partitionLock] = + queryStatsStore.getWithPartitionLock(*queryStatsKeyHash); + if (statusWithMetrics.isOK()) { + // Found an existing entry! Just update the metrics and we're done. + return updateStatistics(partitionLock, + *statusWithMetrics.getValue(), + queryExecMicros, + firstResponseExecMicros, + docsReturned); + } + + // It is possible a cursor that lives forever has no key associated with it and its entry may + // have been evicted. + if (willNeverExhaust && !key) { + return; + } + + // Otherwise we didn't find an existing entry. Try to create one. + tassert(7315200, + "key cannot be null when writing a new entry to the queryStats store", + key != nullptr); + size_t numEvicted = + queryStatsStore.put(*queryStatsKeyHash, QueryStatsEntry(std::move(key)), partitionLock); + queryStatsEvictedMetric.increment(numEvicted); + auto newMetrics = partitionLock->get(*queryStatsKeyHash); + if (!newMetrics.isOK()) { + // This can happen if the budget is immediately exceeded. Specifically if the there is + // not enough room for a single new entry if the number of partitions is too high + // relative to the size. + queryStatsStoreWriteErrorsMetric.increment(); + LOGV2_DEBUG(7560900, + 0, + "Failed to store queryStats entry.", + "status"_attr = newMetrics.getStatus(), + "queryStatsKeyHash"_attr = queryStatsKeyHash); + return; + } + + return updateStatistics(partitionLock, + newMetrics.getValue()->second, + queryExecMicros, + firstResponseExecMicros, + docsReturned); +} + +void writeQueryStatsOnCursorDisposeOrKill(OperationContext* opCtx, + boost::optional<size_t> queryStatsKeyHash, + std::unique_ptr<Key> key, + bool willNeverExhaust, + const uint64_t queryExecMicros, + const uint64_t firstResponseExecMicros, + const uint64_t docsReturned) { + // It is discouraged but technically possible for a user to enable queryStats on the mongods of + // a replica set. In this case, a cursor will be created for each mongod. However, the + // queryStatsKey is behind a unique_ptr on CurOp. The ClientCursor constructor std::moves the + // queryStatsKey so it uniquely owns it (and also makes the queryStatsKey on CurOp now a + // nullptr) and copies over the queryStatsKeyHash as the latter is a cheap copy. + // In the case of sharded $search, two cursors will be created per mongod. In this way, + // two cursors are part of the same thread/operation, and therefore share a OpCtx/CurOp/OpDebug. + // The first cursor that is created will own the queryStatsKey and have a copy of the + // queryStatsKeyHash. On the other hand, the second one will only have a copy of the hash since + // the queryStatsKey will be null on CurOp from being std::move'd in the first cursor + // construction call. To not trip the tassert in writeQueryStats and because all cursors are + // guaranteed to have a copy of the hash, we check that the cursor has a key + if (key && opCtx) { + query_stats::writeQueryStats(opCtx, + queryStatsKeyHash, + std::move(key), + queryExecMicros, + firstResponseExecMicros, + docsReturned, + willNeverExhaust); + } else if (willNeverExhaust && opCtx) { + // Since we already recorded information about the possible getMores associated with a + // cursor that never ends, the only information left to record is about the kill/dispose + // cursor operation. This operation is not timed and does not have any metrics associated + // with it. + query_stats::writeQueryStats(opCtx, queryStatsKeyHash, nullptr, 0, 0, 0, willNeverExhaust); + } +} + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/query_stats.h b/src/mongo/db/query/query_stats/query_stats.h new file mode 100644 index 00000000000..fc96a8be179 --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats.h @@ -0,0 +1,211 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/curop.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/query/partitioned_cache.h" +#include "mongo/db/query/plan_explainer.h" +#include "mongo/db/query/query_stats/key.h" +#include "mongo/db/query/query_stats/query_stats_entry.h" +#include "mongo/db/query/query_stats/rate_limiting.h" +#include "mongo/db/service_context.h" +#include "mongo/db/views/view.h" +#include <cstdint> +#include <memory> + +namespace mongo::query_stats { + +extern Counter64 queryStatsStoreSizeEstimateBytesMetric; + +struct QueryStatsPartitioner { + // The partitioning function for use with the 'Partitioned' utility. + std::size_t operator()(const std::size_t hash, const std::size_t nPartitions) const { + return hash % nPartitions; + } +}; + +struct QueryStatsStoreEntryBudgetor { + size_t operator()(const std::size_t hash, const QueryStatsEntry& value) { + return sizeof(decltype(value)) + sizeof(decltype(hash)) + value.key->size(); + } +}; + +/* + * 'QueryStatsStore insertion and eviction listener implementation. This class adjusts the + * 'queryStatsStoreSize' serverStatus metric when entries are inserted or evicted. + */ +struct QueryStatsStoreInsertionEvictionListener { + void onInsert(const std::size_t&, const QueryStatsEntry&, size_t estimatedSize) { + queryStatsStoreSizeEstimateBytesMetric.increment(estimatedSize); + } + + void onEvict(const std::size_t&, const QueryStatsEntry&, size_t estimatedSize) { + queryStatsStoreSizeEstimateBytesMetric.decrement(estimatedSize); + } + + void onClear(size_t estimatedSize) { + queryStatsStoreSizeEstimateBytesMetric.decrement(estimatedSize); + } +}; +using QueryStatsStore = PartitionedCache<std::size_t, + QueryStatsEntry, + QueryStatsStoreEntryBudgetor, + QueryStatsPartitioner, + QueryStatsStoreInsertionEvictionListener>; + +/** + * A manager for the queryStats store allows a "pointer swap" on the queryStats store itself. The + * usage patterns are as follows: + * + * - Updating the queryStats store uses the `getQueryStatsStore()` method. The queryStats store + * instance is obtained, entries are looked up and mutated, or created anew. + * - The queryStats store is "reset". This involves atomically allocating a new instance, once + * there are no more updaters (readers of the store "pointer"), and returning the existing + * instance. + */ +class QueryStatsStoreManager { +public: + // The query stats store can be configured using these objects on a per-ServiceContext level. + // This is essentially global, but can be manipulated by unit tests. + static const ServiceContext::Decoration<std::unique_ptr<QueryStatsStoreManager>> get; + static const ServiceContext::Decoration<std::unique_ptr<RateLimiting>> getRateLimiter; + + template <typename... QueryStatsStoreArgs> + QueryStatsStoreManager(size_t cacheSize, size_t numPartitions) + : _queryStatsStore(std::make_unique<QueryStatsStore>(cacheSize, numPartitions)), + _maxSize(cacheSize) {} + + /** + * Acquire the instance of the queryStats store. + */ + QueryStatsStore& getQueryStatsStore() { + return *_queryStatsStore; + } + + size_t getMaxSize() { + return _maxSize.load(); + } + + /** + * Resize the queryStats store and return the number of evicted + * entries. + */ + size_t resetSize(size_t cacheSize) { + _maxSize.store(cacheSize); + return _queryStatsStore->reset(cacheSize); + } + +private: + std::unique_ptr<QueryStatsStore> _queryStatsStore; + + /** + * Max size of the queryStats store. Tracked here to avoid having to recompute after it's + * divided up into partitions. + */ + AtomicWord<size_t> _maxSize; +}; + +/** + * Acquire a reference to the global queryStats store. + */ +QueryStatsStore& getQueryStatsStore(OperationContext* opCtx); + +/** + * Registers a request for query stats collection. The function may decide not to collect anything, + * so this should be called for all requests. The decision is made based on the feature flag and + * query stats rate limiting. + * + * The originating command/query does not persist through the end of query execution due to + * optimizations made to the original query and the expiration of OpCtx across getMores. In order + * to pair the query stats metrics that are collected at the end of execution with the original + * query, it is necessary to store the original query during planning and persist it through + * getMores. + * + * During planning, registerRequest is called to serialize the query stats key and save it to + * OpDebug. If a query's execution is complete within the original operation, + * collectQueryStatsMongod/collectQueryStatsMongos will call writeQueryStats() and pass along the + * query stats key to be saved in the query stats store alongside metrics collected. + * + * However, OpDebug does not persist through cursor iteration, so if a query's execution will span + * more than one request/operation, it's necessary to save the query stats context to the cursor + * upon cursor registration. In these cases, collectQueryStatsMongod/collectQueryStatsMongos will + * aggregate each operation's metrics within the cursor. Once the request is eventually complete, + * the cursor calls writeQueryStats() on its destruction. + * + * Notes: + * - It's important to call registerRequest with the original request, before canonicalizing or + * optimizing it, in order to preserve the user's input for the query shape. + * - Calling this affects internal state. It should be called exactly once for each request for + * which query stats may be collected. + * - The std::function argument to construct an abstracted Key is provided to break + * library cycles so this library does not need to know how to parse everything. It is done as a + * deferred construction callback to ensure that this feature does not impact performance if + * collecting stats is not needed due to the feature being disabled or the request being rate + * limited. + */ +void registerRequest(OperationContext* opCtx, + const NamespaceString& collection, + std::function<std::unique_ptr<Key>(void)> makeKey, + bool willNeverExhaust = false); + +/** + * Writes query stats to the query stats store for the operation identified by `queryStatsKeyHash`. + * + * Direct calls to writeQueryStats in new code should be avoided in favor of calling existing + * functions: + * - collectQueryStatsMongod/collectQueryStatsMongos in the case of requests that span one + * operation + * - writeQueryStatsOnCursorDisposeOrKill() in the case of requests that span + * multiple operations (via getMore) + */ +void writeQueryStats(OperationContext* opCtx, + boost::optional<size_t> queryStatsKeyHash, + std::unique_ptr<Key> key, + uint64_t queryExecMicros, + uint64_t firstResponseExecMicros, + uint64_t docsReturned, + bool willNeverExhaust = false); + +/** + * Called from ClientCursor::dispose/ClusterClientCursorImpl::kill to set up and writeQueryStats() + * at the end of life of a cursor. + */ +void writeQueryStatsOnCursorDisposeOrKill(OperationContext* opCtx, + boost::optional<size_t> queryStatsKeyHash, + std::unique_ptr<Key> key, + bool willNeverExhaust, + uint64_t queryExecMicros, + uint64_t firstResponseExecMicros, + uint64_t docsReturned); +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/query_stats_entry.cpp b/src/mongo/db/query/query_stats/query_stats_entry.cpp new file mode 100644 index 00000000000..f69f0a6ee2a --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats_entry.cpp @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_stats/query_stats_entry.h" + +#include <boost/optional.hpp> + +#include "mongo/crypto/hash_block.h" +#include "mongo/crypto/sha256_block.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + +namespace mongo::query_stats { + +BSONObj QueryStatsEntry::toBSON() const { + BSONObjBuilder builder{sizeof(QueryStatsEntry) + 100}; + builder.append("lastExecutionMicros", (long long)lastExecutionMicros); + builder.append("execCount", (long long)execCount); + totalExecMicros.appendTo(builder, "totalExecMicros"); + firstResponseExecMicros.appendTo(builder, "firstResponseExecMicros"); + docsReturned.appendTo(builder, "docsReturned"); + builder.append("firstSeenTimestamp", firstSeenTimestamp); + builder.append("latestSeenTimestamp", latestSeenTimestamp); + return builder.obj(); +} + + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/query_stats_entry.h b/src/mongo/db/query/query_stats/query_stats_entry.h new file mode 100644 index 00000000000..6b61a6a6dcf --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats_entry.h @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <algorithm> +#include <cstdint> +#include <memory> + +#include "mongo/db/commands/server_status_metric.h" +#include "mongo/db/query/query_stats/aggregated_metric.h" +#include "mongo/db/query/query_stats/key.h" +#include "mongo/db/query/query_stats/transform_algorithm_gen.h" +#include "mongo/util/time_support.h" + +namespace mongo::query_stats { + +/** + * The value stored in the query stats store. It contains a Key representing this "kind" of + * query, and some metrics about that shape. This class is responsible for knowing its size and + * updating our server status metrics about the size of the query stats store accordingly. At the + * time of this writing, the LRUCache utility does not easily expose its size in a way we could use + * as server status metrics. + */ +struct QueryStatsEntry { + QueryStatsEntry(std::unique_ptr<const Key> key_) + : firstSeenTimestamp(Date_t::now()), key(std::move(key_)) {} + + BSONObj toBSON() const; + + /** + * Timestamp for when this query shape was added to the store. Set on construction. + */ + const Date_t firstSeenTimestamp; + + /** + * Timestamp for when the latest time this query shape was seen. + */ + Date_t latestSeenTimestamp; + + /** + * Last execution time in microseconds. + */ + uint64_t lastExecutionMicros = 0; + + /** + * Number of query executions. + */ + uint64_t execCount = 0; + + /** + * Aggregates the total time for execution including getMore requests. + */ + AggregatedMetric totalExecMicros; + + /** + * Aggregates the time for execution for first batch only. + */ + AggregatedMetric firstResponseExecMicros; + + AggregatedMetric docsReturned; + + /** + * The Key that can generate the query stats key for this request. + */ + std::shared_ptr<const Key> key; +}; + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/query_stats_helpers.h b/src/mongo/db/query/query_stats/query_stats_helpers.h new file mode 100644 index 00000000000..6d53cc8d4ce --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats_helpers.h @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <absl/hash/hash.h> +#include <boost/optional.hpp> + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/simple_bsonobj_comparator.h" +#include "mongo/db/query/query_shape/shape_helpers.h" + +namespace mongo::query_stats { + +/** + * An abseil compatible hash function for BSONObjects. Note that this hasher ignores any collation + * and uses the "simple" comparisons. This is fine and correct for query stats, but this is + * intentionally placed within the 'query_stats' namespace to avoid polluting the whole codebase + * with this helper which could cause an accidental bug where we ignore the request's collation. + */ +template <typename H> +H AbslHashValue(H h, const BSONObj& obj) { + return H::combine(std::move(h), simpleHash(obj)); +} + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/query_stats_on_parameter_change.cpp b/src/mongo/db/query/query_stats/query_stats_on_parameter_change.cpp new file mode 100644 index 00000000000..a8b7df9fccb --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats_on_parameter_change.cpp @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + +#include "mongo/db/query/query_stats/query_stats_on_parameter_change.h" + +#include "mongo/base/status.h" +#include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/query/partitioned_cache.h" +#include "mongo/db/query/query_knobs_gen.h" +#include "mongo/db/query/util/memory_util.h" +#include "mongo/db/service_context.h" +#include "mongo/logv2/log.h" + +namespace mongo::query_stats_util { + +namespace { +/** + * Given the current 'Client', returns a pointer to the 'ServiceContext' and an interface for + * updating the queryStats store. + */ +std::pair<ServiceContext*, OnParamChangeUpdater*> getUpdater(const Client& client) { + auto serviceCtx = client.getServiceContext(); + tassert(7106500, "ServiceContext must be non null", serviceCtx); + + auto updater = queryStatsStoreOnParamChangeUpdater(serviceCtx).get(); + tassert(7106501, "queryStats store size updater must be non null", updater); + return {serviceCtx, updater}; +} +} // namespace + + +Status onQueryStatsStoreSizeUpdate(const std::string& str) { + auto newSize = memory_util::MemorySize::parse(str); + if (!newSize.isOK()) { + return newSize.getStatus(); + } + + // The client is nullptr if the parameter is supplied from the command line. In this case, we + // ignore the update event, the parameter will be processed when initializing the service + // context. + if (auto client = Client::getCurrent()) { + auto&& [serviceCtx, updater] = getUpdater(*client); + updater->updateCacheSize(serviceCtx, newSize.getValue()); + } + + return Status::OK(); +} + +Status validateQueryStatsStoreSize(const std::string& str) { + return memory_util::MemorySize::parse(str).getStatus(); +} + +Status onQueryStatsSamplingRateUpdate(int samplingRate) { + // The client is nullptr if the parameter is supplied from the command line. In this case, we + // ignore the update event, the parameter will be processed when initializing the service + // context. + if (auto client = Client::getCurrent()) { + auto&& [serviceCtx, updater] = getUpdater(*client); + updater->updateSamplingRate(serviceCtx, samplingRate < 0 ? INT_MAX : samplingRate); + } + + return Status::OK(); +} + +const Decorable<ServiceContext>::Decoration<std::unique_ptr<OnParamChangeUpdater>> + queryStatsStoreOnParamChangeUpdater = + ServiceContext::declareDecoration<std::unique_ptr<OnParamChangeUpdater>>(); +} // namespace mongo::query_stats_util diff --git a/src/mongo/db/query/query_stats/query_stats_on_parameter_change.h b/src/mongo/db/query/query_stats/query_stats_on_parameter_change.h new file mode 100644 index 00000000000..2a824961b34 --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats_on_parameter_change.h @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/query/partitioned_cache.h" +#include "mongo/db/query/util/memory_util.h" + + +namespace mongo::query_stats_util { + +Status onQueryStatsStoreSizeUpdate(const std::string& str); + + +Status validateQueryStatsStoreSize(const std::string& str); + +Status onQueryStatsSamplingRateUpdate(int samplingRate); + +/** + * An interface used to modify the queryStats store when query setParameters are modified. This is + * done via an interface decorating the 'ServiceContext' in order to avoid a link-time dependency of + * the query knobs library on the queryStats code. + */ +class OnParamChangeUpdater { +public: + virtual ~OnParamChangeUpdater() = default; + + /** + * Resizes the queryStats store decorating 'serviceCtx' to the new size given by 'memSize'. If + * the new size is smaller than the old, cache entries are evicted in order to ensure the + * cache fits within the new size bound. + */ + virtual void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) = 0; + + /** + * Updates the sampling rate for the queryStats rate limiter. + */ + virtual void updateSamplingRate(ServiceContext* serviceCtx, int samplingRate) = 0; +}; + +/** + * Decorated accessor to the 'OnParamChangeUpdater' stored in 'ServiceContext'. Again, this is done + * via a decoration and interface to avoid a link-time dependency from the query knobs library on + * the queryStats code. + */ +extern const Decorable<ServiceContext>::Decoration<std::unique_ptr<OnParamChangeUpdater>> + queryStatsStoreOnParamChangeUpdater; +} // namespace mongo::query_stats_util diff --git a/src/mongo/db/query/query_stats/query_stats_store_test.cpp b/src/mongo/db/query/query_stats/query_stats_store_test.cpp new file mode 100644 index 00000000000..74965c4ece9 --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats_store_test.cpp @@ -0,0 +1,1427 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/bson/simple_bsonobj_comparator.h" +#include "mongo/db/catalog/rename_collection.h" +#include "mongo/db/collection_type.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_feature_flags_gen.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/db/query/query_stats/agg_key.h" +#include "mongo/db/query/query_stats/find_key.h" +#include "mongo/db/query/query_stats/key.h" +#include "mongo/db/query/query_stats/query_stats.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/idl/server_parameter_test_util.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_stats { + +int countAllEntries(const QueryStatsStore& store) { + int numKeys = 0; + store.forEach([&](auto&& key, auto&& entry) { numKeys++; }); + return numKeys; +} + +static const NamespaceStringOrUUID kDefaultTestNss = NamespaceString("testDB.testColl"); +class QueryStatsStoreTest : public ServiceContextTest { +public: + static std::unique_ptr<const Key> makeFindKeyFromQuery(BSONObj filter) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcr))); + return std::make_unique<FindKey>(expCtx, *parsedFind, collectionType); + } + + static constexpr auto collectionType = query_shape::CollectionType::kCollection; + BSONObj makeQueryStatsKeyFindRequest(const FindCommandRequest& fcr, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + bool applyHmac) { + auto fcrCopy = std::make_unique<FindCommandRequest>(fcr); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcrCopy))); + FindKey findKey(expCtx, *parsedFind, collectionType); + SerializationOptions opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + if (!applyHmac) { + opts.transformIdentifiers = false; + opts.transformIdentifiersCallback = defaultHmacStrategy; + } + return findKey.toBson(expCtx->opCtx, opts); + } + + BSONObj makeQueryStatsKeyAggregateRequest(AggregateCommandRequest acr, + const Pipeline& pipeline, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + LiteralSerializationPolicy literalPolicy, + bool applyHmac = false) { + auto aggKey = std::make_unique<AggKey>(acr, + pipeline, + expCtx, + pipeline.getInvolvedCollections(), + acr.getNamespace(), + collectionType); + + // SerializationOptions opts{.literalPolicy = literalPolicy}; + SerializationOptions opts = SerializationOptions::kMarkIdentifiers_FOR_TEST; + opts.literalPolicy = literalPolicy; + if (!applyHmac) { + opts.transformIdentifiers = false; + opts.transformIdentifiersCallback = defaultHmacStrategy; + } + return aggKey->toBson(expCtx->opCtx, opts); + } +}; + +TEST_F(QueryStatsStoreTest, BasicUsage) { + QueryStatsStore queryStatsStore{5000000, 1000}; + + auto getMetrics = [&](BSONObj query) { + auto key = makeFindKeyFromQuery(query); + auto lookupResult = queryStatsStore.lookup(absl::Hash<query_stats::Key>{}(*key)); + ASSERT_OK(lookupResult); + return *lookupResult.getValue(); + }; + + auto collectMetrics = [&](BSONObj query) { + auto key = makeFindKeyFromQuery(query); + auto lookupHash = absl::Hash<query_stats::Key>{}(*key); + auto lookupResult = queryStatsStore.lookup(lookupHash); + if (!lookupResult.isOK()) { + queryStatsStore.put(lookupHash, QueryStatsEntry{std::move(key)}); + lookupResult = queryStatsStore.lookup(lookupHash); + } + auto metrics = lookupResult.getValue(); + metrics->execCount += 1; + metrics->lastExecutionMicros += 123456; + }; + + auto query1 = BSON("query" << 1 << "xEquals" << 42); + // same value, different instance (tests hashing & equality) + auto query1x = BSON("query" << 1 << "xEquals" << 42); + auto query2 = BSON("query" << 2 << "yEquals" << 43); + + collectMetrics(query1); + collectMetrics(query1); + collectMetrics(query1x); + collectMetrics(query2); + + ASSERT_EQ(getMetrics(query1).execCount, 3); + ASSERT_EQ(getMetrics(query1x).execCount, 3); + ASSERT_EQ(getMetrics(query2).execCount, 1); + + auto collectMetricsWithLock = [&](BSONObj& filter) { + auto key = makeFindKeyFromQuery(filter); + auto [lookupResult, lock] = + queryStatsStore.getWithPartitionLock(absl::Hash<query_stats::Key>{}(*key)); + ASSERT_OK(lookupResult); + auto& metrics = *lookupResult.getValue(); + metrics.execCount += 1; + metrics.lastExecutionMicros += 123456; + }; + + collectMetricsWithLock(query1x); + collectMetricsWithLock(query2); + + ASSERT_EQ(getMetrics(query1).execCount, 4); + ASSERT_EQ(getMetrics(query1x).execCount, 4); + ASSERT_EQ(getMetrics(query2).execCount, 2); + + ASSERT_EQ(2, countAllEntries(queryStatsStore)); +} + +TEST_F(QueryStatsStoreTest, EvictionTest) { + // This creates a queryStats store with a single partition to specifically test the eviction + // behavior with very large queries. + // Add an entry that is smaller than the max partition size. + auto query = BSON("query" << 1 << "xEquals" << 42); + auto key = makeFindKeyFromQuery(query); + + const size_t cacheSize = key->size() + sizeof(QueryStatsEntry) + 100; + const auto numPartitions = 1; + QueryStatsStore queryStatsStore{cacheSize, numPartitions}; + + auto hash = absl::Hash<query_stats::Key>{}(*key); + queryStatsStore.put(hash, QueryStatsEntry{std::move(key)}); + ASSERT_EQ(countAllEntries(queryStatsStore), 1); + + // We'll do this again later so save this as a helper function. + auto addLargeEntry = [&](auto& queryStatsStore) { + // Add an entry that is larger than the max partition size to the non-empty partition. This + // should evict both entries, the first small entry written to the partition and the current + // too large entry we wish to write to the partition. The reason is because entries are + // evicted from the partition in order of least recently used. Thus, the small entry will be + // evicted first but the partition will still be over budget so the final, too large entry + // will also be evicted. + auto opCtx = makeOperationContext(); + auto fcr = std::make_unique<FindCommandRequest>( + NamespaceStringOrUUID(NamespaceString("testDB.testColl"))); + fcr->setLet(BSON("var" << 2)); + fcr->setFilter(fromjson("{$expr: [{$eq: ['$a', '$$var']}]}")); + fcr->setProjection(fromjson("{varIs: '$$var'}")); + fcr->setLimit(5); + fcr->setSkip(2); + fcr->setBatchSize(25); + fcr->setMaxTimeMS(1000); + fcr->setNoCursorTimeout(false); + opCtx->setComment(BSON("comment" + << " foo bar baz")); + fcr->setSingleBatch(false); + fcr->setAllowDiskUse(false); + fcr->setAllowPartialResults(true); + fcr->setAllowDiskUse(false); + fcr->setShowRecordId(true); + fcr->setHint(BSON("z" << 1 << "c" << 1)); + fcr->setMax(BSON("z" << 25)); + fcr->setMin(BSON("z" << 80)); + fcr->setSort(BSON("sortVal" << 1 << "otherSort" << -1)); + auto&& [expCtx, parsedFind] = + uassertStatusOK(parsed_find_command::parse(opCtx.get(), std::move(fcr))); + + key = std::make_unique<query_stats::FindKey>(expCtx, *parsedFind, collectionType); + auto lookupHash = absl::Hash<query_stats::Key>{}(*key); + QueryStatsEntry testMetrics{std::move(key)}; + queryStatsStore.put(lookupHash, testMetrics); + }; + + addLargeEntry(queryStatsStore); + ASSERT_EQ(countAllEntries(queryStatsStore), 0); + + // This creates a queryStats store where each partition has a max size of 500 bytes. + QueryStatsStore queryStatsStoreTwo{/*cacheSize*/ cacheSize * 3, /*numPartitions*/ 3}; + // Adding a queryStats store entry that is smaller than the overal cache size but larger + // than a single partition max size, will cause an eviction. testMetrics is larger than 500 + // bytes and thus over budget for the partitions of this cache. + addLargeEntry(queryStatsStoreTwo); + ASSERT_EQ(countAllEntries(queryStatsStoreTwo), 0); +} + +TEST_F(QueryStatsStoreTest, GenerateMaxBsonSizeQueryShape) { + const NamespaceString nss = NamespaceString("testDB.testColl"); + FindCommandRequest fcr((NamespaceStringOrUUID(nss))); + // This creates a query that is just below the 16 MB memory limit. + int limit = 225500; + BSONObjBuilder bob; + BSONArrayBuilder andBob(bob.subarrayStart("$and")); + for (int i = 1; i <= limit; i++) { + BSONObjBuilder childrenBob; + childrenBob.append("x", BSON("$lt" << i << "$gte" << i)); + andBob.append(childrenBob.obj()); + } + andBob.doneFast(); + fcr.setFilter(bob.obj()); + auto fcrCopy = std::make_unique<FindCommandRequest>(fcr); + auto opCtx = makeOperationContext(); + auto parsedFindPair = + uassertStatusOK(parsed_find_command::parse(opCtx.get(), std::move(fcrCopy))); + + auto&& globalQueryStatsStoreManager = QueryStatsStoreManager::get(opCtx->getServiceContext()); + globalQueryStatsStoreManager = std::make_unique<QueryStatsStoreManager>(500000, 1000); + + // The shapification process will bloat the input query over the 16 MB memory limit. Assert that + // calling registerRequest() doesn't throw and that the opDebug isn't registered with a key hash + // (thus metrics won't be tracked for this query). + ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() { + return std::make_unique<query_stats::FindKey>( + parsedFindPair.first, *parsedFindPair.second, query_shape::CollectionType::kCollection); + })); + auto& opDebug = CurOp::get(*opCtx)->debug(); + ASSERT_FALSE(opDebug.queryStatsInfo.keyHash.has_value()); +} + +TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + FindCommandRequest fcr(kDefaultTestNss); + + fcr.setFilter(BSON("a" << 1)); + + auto key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "find", + "filter": { + "HASH<a>": { + "$eq": "?number" + } + } + }, + "collectionType": "collection" + })", + key); + + // Add sort. + fcr.setSort(BSON("sortVal" << 1 << "otherSort" << -1)); + key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "find", + "filter": { + "HASH<a>": { + "$eq": "?number" + } + }, + "sort": { + "HASH<sortVal>": 1, + "HASH<otherSort>": -1 + } + }, + "collectionType": "collection" + })", + key); + + // Add inclusion projection. + fcr.setProjection(BSON("e" << true << "f" << true)); + key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "find", + "filter": { + "HASH<a>": { + "$eq": "?number" + } + }, + "projection": { + "HASH<e>": true, + "HASH<f>": true, + "HASH<_id>": true + }, + "sort": { + "HASH<sortVal>": 1, + "HASH<otherSort>": -1 + } + }, + "collectionType": "collection" + })", + key); + + // Add let. + fcr.setLet(BSON("var1" << 1 << "var2" + << "const1")); + key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "let": { + "HASH<var1>": "?number", + "HASH<var2>": "?string" + }, + "command": "find", + "filter": { + "HASH<a>": { + "$eq": "?number" + } + }, + "projection": { + "HASH<e>": true, + "HASH<f>": true, + "HASH<_id>": true + }, + "sort": { + "HASH<sortVal>": 1, + "HASH<otherSort>": -1 + } + }, + "collectionType": "collection" + })", + key); + + // Add hinting fields. + fcr.setHint(BSON("z" << 1 << "c" << 1)); + fcr.setMax(BSON("z" << 25)); + fcr.setMin(BSON("z" << 80)); + key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "let": { + "HASH<var1>": "?number", + "HASH<var2>": "?string" + }, + "command": "find", + "filter": { + "HASH<a>": { + "$eq": "?number" + } + }, + "projection": { + "HASH<e>": true, + "HASH<f>": true, + "HASH<_id>": true + }, + "max": { + "HASH<z>": "?number" + }, + "min": { + "HASH<z>": "?number" + }, + "sort": { + "HASH<sortVal>": 1, + "HASH<otherSort>": -1 + } + }, + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } + })", + key); + + // Add the literal redaction fields. + fcr.setLimit(5); + fcr.setSkip(2); + fcr.setBatchSize(25); + fcr.setMaxTimeMS(1000); + fcr.setNoCursorTimeout(false); + + key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "let": { + "HASH<var1>": "?number", + "HASH<var2>": "?string" + }, + "command": "find", + "filter": { + "HASH<a>": { + "$eq": "?number" + } + }, + "projection": { + "HASH<e>": true, + "HASH<f>": true, + "HASH<_id>": true + }, + "max": { + "HASH<z>": "?number" + }, + "min": { + "HASH<z>": "?number" + }, + "sort": { + "HASH<sortVal>": 1, + "HASH<otherSort>": -1 + }, + "limit": "?number", + "skip": "?number" + }, + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + }, + "maxTimeMS": "?number", + "noCursorTimeout": false, + "batchSize": "?number" + })", + key); + + // Add the fields that shouldn't be hmacApplied. + fcr.setSingleBatch(true); + fcr.setAllowDiskUse(false); + fcr.setAllowPartialResults(true); + fcr.setAllowDiskUse(false); + fcr.setShowRecordId(true); + auto readPreference = BSON("mode" + << "nearest" + << "tags" + << BSON_ARRAY(BSON("some" + << "tag") + << BSON("some" + << "other tag"))); + ReadPreferenceSetting::get(expCtx->opCtx) = + uassertStatusOK(ReadPreferenceSetting::fromInnerBSON(readPreference)); + key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "let": { + "HASH<var1>": "?number", + "HASH<var2>": "?string" + }, + "command": "find", + "filter": { + "HASH<a>": { + "$eq": "?number" + } + }, + "projection": { + "HASH<e>": true, + "HASH<f>": true, + "HASH<_id>": true + }, + "max": { + "HASH<z>": "?number" + }, + "min": { + "HASH<z>": "?number" + }, + "sort": { + "HASH<sortVal>": 1, + "HASH<otherSort>": -1 + }, + "limit": "?number", + "skip": "?number", + "singleBatch": true, + "allowDiskUse": false, + "showRecordId": true + }, + "$readPreference": { + "mode": "nearest", + "tags": [ { "some": "other tag" }, { "some": "tag" } ], + "hedge": { "enabled": true } + }, + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + }, + "maxTimeMS": "?number", + "allowPartialResults": true, + "noCursorTimeout": false, + "batchSize": "?number" + })", + key); + + fcr.setAllowPartialResults(false); + key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + // Make sure that a false allowPartialResults is also accurately captured. + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "let": { + "HASH<var1>": "?number", + "HASH<var2>": "?string" + }, + "command": "find", + "filter": { + "HASH<a>": { + "$eq": "?number" + } + }, + "projection": { + "HASH<e>": true, + "HASH<f>": true, + "HASH<_id>": true + }, + "max": { + "HASH<z>": "?number" + }, + "min": { + "HASH<z>": "?number" + }, + "sort": { + "HASH<sortVal>": 1, + "HASH<otherSort>": -1 + }, + "limit": "?number", + "skip": "?number", + "singleBatch": true, + "allowDiskUse": false, + "showRecordId": true + }, + "$readPreference": { + "mode": "nearest", + "tags": [ { "some": "other tag" }, { "some": "tag" } ], + "hedge": { "enabled": true } + }, + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + }, + "maxTimeMS": "?number", + "allowPartialResults": false, + "noCursorTimeout": false, + "batchSize": "?number" + })", + key); +} + +TEST_F(QueryStatsStoreTest, CorrectlyRedactsTailableFindCommandRequest) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + FindCommandRequest fcr(NamespaceStringOrUUID(NamespaceString("testDB.testColl"))); + fcr.setAwaitData(true); + fcr.setTailable(true); + fcr.setSort(BSON("$natural" << 1)); + auto key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "find", + "filter": {}, + "tailable": true, + "awaitData": true + }, + "collectionType": "collection", + "hint": { + "$natural": 1 + } + })", + key); +} + +TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestEmptyFields) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + FindCommandRequest fcr(NamespaceStringOrUUID(NamespaceString("testDB.testColl"))); + fcr.setFilter(BSONObj()); + fcr.setSort(BSONObj()); + fcr.setProjection(BSONObj()); + + auto hmacApplied = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "find", + "filter": {} + }, + "collectionType": "collection" + })", + hmacApplied); // NOLINT (test auto-update) +} + +TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + FindCommandRequest fcr(NamespaceStringOrUUID(NamespaceString("testDB.testColl"))); + + fcr.setFilter(BSON("b" << 1)); + fcr.setHint(BSON("z" << 1 << "c" << 1)); + fcr.setMax(BSON("z" << 25)); + fcr.setMin(BSON("z" << 80)); + + auto key = makeQueryStatsKeyFindRequest(fcr, expCtx, false); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "testDB", + "coll": "testColl" + }, + "command": "find", + "filter": { + "b": { + "$eq": "?number" + } + }, + "max": { + "z": "?number" + }, + "min": { + "z": "?number" + } + }, + "collectionType": "collection", + "hint": { + "z": 1, + "c": 1 + } + })", + key); + // Test with a string hint. Note that this is the internal representation of the string hint + // generated at parse time. + fcr.setHint(BSON("$hint" + << "z")); + + key = makeQueryStatsKeyFindRequest(fcr, expCtx, false); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "testDB", + "coll": "testColl" + }, + "command": "find", + "filter": { + "b": { + "$eq": "?number" + } + }, + "max": { + "z": "?number" + }, + "min": { + "z": "?number" + } + }, + "collectionType": "collection", + "hint": { + "$hint": "z" + } + })", + key); + + fcr.setHint(BSON("z" << 1 << "c" << 1)); + key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "find", + "filter": { + "HASH<b>": { + "$eq": "?number" + } + }, + "max": { + "HASH<z>": "?number" + }, + "min": { + "HASH<z>": "?number" + } + }, + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } + })", + key); + + // Test that $natural comes through unmodified. + fcr.setHint(BSON("$natural" << -1)); + key = makeQueryStatsKeyFindRequest(fcr, expCtx, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "find", + "filter": { + "HASH<b>": { + "$eq": "?number" + } + }, + "max": { + "HASH<z>": "?number" + }, + "min": { + "HASH<z>": "?number" + } + }, + "collectionType": "collection", + "hint": { + "$natural": -1 + } + })", + key); +} + +TEST_F(QueryStatsStoreTest, DefinesLetVariables) { + // Test that the expression context we use to apply hmac will understand the 'let' part of + // the find command while parsing the other pieces of the command. + + // Note that this ExpressionContext will not have the let variables defined - we expect the + // 'makeQueryStatsKey' call to do that. + auto opCtx = makeOperationContext(); + auto fcr = std::make_unique<FindCommandRequest>(NamespaceString("testDB.testColl")); + fcr->setLet(BSON("var" << 2)); + fcr->setFilter(fromjson("{$expr: [{$eq: ['$a', '$$var']}]}")); + fcr->setProjection(fromjson("{varIs: '$$var'}")); + + auto expCtx = make_intrusive<ExpressionContextForTest>(opCtx.get()); + expCtx->variables.seedVariablesWithLetParameters(expCtx.get(), *fcr->getLet()); + auto hmacApplied = makeQueryStatsKeyFindRequest(*fcr, expCtx, false); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "testDB", + "coll": "testColl" + }, + "let": { + "var": "?number" + }, + "command": "find", + "filter": { + "$expr": [ + { + "$eq": [ + "$a", + "$$var" + ] + } + ] + }, + "projection": { + "varIs": "$$var", + "_id": true + } + }, + "collectionType": "collection" + })", + hmacApplied); + + hmacApplied = makeQueryStatsKeyFindRequest(*fcr, expCtx, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "let": { + "HASH<var>": "?number" + }, + "command": "find", + "filter": { + "$expr": [ + { + "$eq": [ + "$HASH<a>", + "$$HASH<var>" + ] + } + ] + }, + "projection": { + "HASH<varIs>": "$$HASH<var>", + "HASH<_id>": true + } + }, + "collectionType": "collection" + })", + hmacApplied); +} + +TEST_F(QueryStatsStoreTest, CorrectlyTokenizesAggregateCommandRequestAllFieldsSimplePipeline) { + auto expCtx = make_intrusive<ExpressionContextForTest>(*kDefaultTestNss.nss()); + AggregateCommandRequest acr(*kDefaultTestNss.nss()); + auto matchStage = fromjson(R"({ + $match: { + foo: { $in: ["a", "b"] }, + bar: { $gte: { $date: "2022-01-01T00:00:00Z" } } + } + })"); + auto unwindStage = fromjson("{$unwind: '$x'}"); + auto groupStage = fromjson(R"({ + $group: { + _id: "$_id", + c: { $first: "$d.e" }, + f: { $sum: 1 } + } + })"); + auto limitStage = fromjson("{$limit: 10}"); + auto outStage = fromjson(R"({$out: 'outColl'})"); + auto rawPipeline = {matchStage, unwindStage, groupStage, limitStage, outStage}; + acr.setPipeline(rawPipeline); + auto pipeline = Pipeline::parse(rawPipeline, expCtx); + + auto shapified = makeQueryStatsKeyAggregateRequest( + acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "$and": [ + { + "HASH<foo>": { + "$in": "?array<?string>" + } + }, + { + "HASH<bar>": { + "$gte": "?date" + } + } + ] + } + }, + { + "$unwind": { + "path": "$HASH<x>" + } + }, + { + "$group": { + "_id": "$HASH<_id>", + "HASH<c>": { + "$first": "$HASH<d>.HASH<e>" + }, + "HASH<f>": { + "$sum": "?number" + } + } + }, + { + "$limit": "?number" + }, + { + "$out": { + "coll": "HASH<outColl>", + "db": "HASH<testDB>" + } + } + ] + }, + "collectionType": "collection" + })", + shapified); + + // Add the fields that shouldn't be abstracted. + acr.setAllowDiskUse(false); + acr.setHint(BSON("z" << 1 << "c" << 1)); + acr.setCollation(BSON("locale" + << "simple")); + shapified = makeQueryStatsKeyAggregateRequest( + acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "collation": { + "locale": "simple" + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "$and": [ + { + "HASH<foo>": { + "$in": "?array<?string>" + } + }, + { + "HASH<bar>": { + "$gte": "?date" + } + } + ] + } + }, + { + "$unwind": { + "path": "$HASH<x>" + } + }, + { + "$group": { + "_id": "$HASH<_id>", + "HASH<c>": { + "$first": "$HASH<d>.HASH<e>" + }, + "HASH<f>": { + "$sum": "?number" + } + } + }, + { + "$limit": "?number" + }, + { + "$out": { + "coll": "HASH<outColl>", + "db": "HASH<testDB>" + } + } + ], + "allowDiskUse": false + }, + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } + })", + shapified); + + // Add let. + acr.setLet(BSON("var1" << BSON("$literal" + << "$foo") + << "var2" + << "bar")); + shapified = makeQueryStatsKeyAggregateRequest( + acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "collation": { + "locale": "simple" + }, + "let": { + "HASH<var1>": "?string", + "HASH<var2>": "?string" + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "$and": [ + { + "HASH<foo>": { + "$in": "?array<?string>" + } + }, + { + "HASH<bar>": { + "$gte": "?date" + } + } + ] + } + }, + { + "$unwind": { + "path": "$HASH<x>" + } + }, + { + "$group": { + "_id": "$HASH<_id>", + "HASH<c>": { + "$first": "$HASH<d>.HASH<e>" + }, + "HASH<f>": { + "$sum": "?number" + } + } + }, + { + "$limit": "?number" + }, + { + "$out": { + "coll": "HASH<outColl>", + "db": "HASH<testDB>" + } + } + ], + "allowDiskUse": false + }, + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + } + })", + shapified); + + // Add the fields that should be abstracted. + auto cursorOptions = SimpleCursorOptions(); + cursorOptions.setBatchSize(10); + acr.setCursor(cursorOptions); + acr.setMaxTimeMS(500); + acr.setBypassDocumentValidation(true); + expCtx->opCtx->setComment(BSON("comment" + << "note to self")); + shapified = makeQueryStatsKeyAggregateRequest( + acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "collation": { + "locale": "simple" + }, + "let": { + "HASH<var1>": "?string", + "HASH<var2>": "?string" + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "$and": [ + { + "HASH<foo>": { + "$in": "?array<?string>" + } + }, + { + "HASH<bar>": { + "$gte": "?date" + } + } + ] + } + }, + { + "$unwind": { + "path": "$HASH<x>" + } + }, + { + "$group": { + "_id": "$HASH<_id>", + "HASH<c>": { + "$first": "$HASH<d>.HASH<e>" + }, + "HASH<f>": { + "$sum": "?number" + } + } + }, + { + "$limit": "?number" + }, + { + "$out": { + "coll": "HASH<outColl>", + "db": "HASH<testDB>" + } + } + ], + "allowDiskUse": false + }, + "comment": "?string", + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + }, + "maxTimeMS": "?number", + "bypassDocumentValidation": true, + "cursor": { + "batchSize": "?number" + } + })", + shapified); + + // Test again but with the representative query shape. + shapified = makeQueryStatsKeyAggregateRequest( + acr, *pipeline, expCtx, LiteralSerializationPolicy::kToRepresentativeParseableValue, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "collation": { + "locale": "simple" + }, + "let": { + "HASH<var1>": { + "$const": "?" + }, + "HASH<var2>": { + "$const": "?" + } + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "$and": [ + { + "HASH<foo>": { + "$in": [ + "?" + ] + } + }, + { + "HASH<bar>": { + "$gte": {"$date":"1970-01-01T00:00:00.000Z"} + } + } + ] + } + }, + { + "$unwind": { + "path": "$HASH<x>" + } + }, + { + "$group": { + "_id": "$HASH<_id>", + "HASH<c>": { + "$first": "$HASH<d>.HASH<e>" + }, + "HASH<f>": { + "$sum": { + "$const": 1 + } + } + } + }, + { + "$limit": 1 + }, + { + "$out": { + "coll": "HASH<outColl>", + "db": "HASH<testDB>" + } + } + ], + "allowDiskUse": false + }, + "comment": "?", + "collectionType": "collection", + "hint": { + "HASH<z>": 1, + "HASH<c>": 1 + }, + "maxTimeMS": 1, + "bypassDocumentValidation": true, + "cursor": { + "batchSize": 1 + } + })", + shapified); +} + +TEST_F(QueryStatsStoreTest, CorrectlyTokenizesAggregateCommandRequestEmptyFields) { + auto expCtx = make_intrusive<ExpressionContextForTest>(*kDefaultTestNss.nss()); + AggregateCommandRequest acr(*kDefaultTestNss.nss()); + acr.setPipeline({}); + auto pipeline = Pipeline::parse({}, expCtx); + + auto shapified = makeQueryStatsKeyAggregateRequest( + acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "aggregate", + "pipeline": [] + }, + "collectionType": "collection" + })", + shapified); // NOLINT (test auto-update) + + // Test again with the representative query shape. + shapified = makeQueryStatsKeyAggregateRequest( + acr, *pipeline, expCtx, LiteralSerializationPolicy::kToRepresentativeParseableValue, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "aggregate", + "pipeline": [] + }, + "collectionType": "collection" + })", + shapified); // NOLINT (test auto-update) +} + +TEST_F(QueryStatsStoreTest, + CorrectlyTokenizesAggregateCommandRequestPipelineWithSecondaryNamespaces) { + auto expCtx = make_intrusive<ExpressionContextForTest>(*kDefaultTestNss.nss()); + auto nsToUnionWith = NamespaceString(expCtx->ns.db(), "otherColl"); + expCtx->addResolvedNamespaces({nsToUnionWith}); + + AggregateCommandRequest acr(*kDefaultTestNss.nss()); + auto unionWithStage = fromjson(R"({ + $unionWith: { + coll: "otherColl", + pipeline: [{$match: {val: "foo"}}] + } + })"); + auto sortStage = fromjson("{$sort: {age: 1}}"); + auto rawPipeline = {unionWithStage, sortStage}; + acr.setPipeline(rawPipeline); + auto pipeline = Pipeline::parse(rawPipeline, expCtx); + + auto shapified = makeQueryStatsKeyAggregateRequest( + acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "aggregate", + "pipeline": [ + { + "$unionWith": { + "coll": "HASH<otherColl>", + "pipeline": [ + { + "$match": { + "HASH<val>": { + "$eq": "?string" + } + } + } + ] + } + }, + { + "$sort": { + "HASH<age>": 1 + } + } + ] + }, + "collectionType": "collection", + "otherNss": [ + { + "db": "HASH<testDB>", + "coll": "HASH<otherColl>" + } + ] + })", + shapified); + + // Do the same thing with the representative query shape. + shapified = makeQueryStatsKeyAggregateRequest( + acr, *pipeline, expCtx, LiteralSerializationPolicy::kToRepresentativeParseableValue, true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "queryShape": { + "cmdNs": { + "db": "HASH<testDB>", + "coll": "HASH<testColl>" + }, + "command": "aggregate", + "pipeline": [ + { + "$unionWith": { + "coll": "HASH<otherColl>", + "pipeline": [ + { + "$match": { + "HASH<val>": { + "$eq": "?" + } + } + } + ] + } + }, + { + "$sort": { + "HASH<age>": 1 + } + } + ] + }, + "collectionType": "collection", + "otherNss": [ + { + "db": "HASH<testDB>", + "coll": "HASH<otherColl>" + } + ] + })", + shapified); +} + +BSONObj toBSON(AggregatedMetric am) { + BSONObjBuilder builder; + am.appendTo(builder, "m"); + return builder.obj(); +} + +TEST_F(QueryStatsStoreTest, SumOfSquaresOverflowTest) { + // Ensure sumOfSquares is initialized correctly. + AggregatedMetric aggMetric; + auto res = toBSON(aggMetric).getObjectField("m").getField("sumOfSquares").Decimal(); + + ASSERT_EQ(res, Decimal128()); + + // Aggregating with the maximum int value does not overflow the sumOfSquares field. + auto maxVal = std::numeric_limits<uint64_t>::max(); + aggMetric.aggregate(maxVal); + res = toBSON(aggMetric).getObjectField("m").getField("sumOfSquares").Decimal(); + + ASSERT_EQ(res, Decimal128(maxVal).power(Decimal128(2.0))); +} +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/query_stats_test.cpp b/src/mongo/db/query/query_stats/query_stats_test.cpp new file mode 100644 index 00000000000..4b9462e3e25 --- /dev/null +++ b/src/mongo/db/query/query_stats/query_stats_test.cpp @@ -0,0 +1,223 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/bson/bsonobj.h" +#include "mongo/db/collection_type.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/parsed_find_command.h" +#include "mongo/db/query/query_stats/find_key.h" +#include "mongo/db/query/query_stats/query_stats.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/idl/server_parameter_test_util.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQueryStats + +namespace mongo::query_stats { +class QueryStatsTest : public ServiceContextTest {}; + +TEST_F(QueryStatsTest, TwoRegisterRequestsWithSameOpCtxRateLimitedFirstCall) { + // This test simulates what happens with queries over views where two calls to registerRequest() + // can be made with the same opCtx. + + // Make query for query stats. + const NamespaceString nss = NamespaceString("testDB.testColl"); + FindCommandRequest fcr((NamespaceStringOrUUID(nss))); + fcr.setFilter(BSONObj()); + + auto fcrCopy = std::make_unique<FindCommandRequest>(fcr); + auto opCtx = makeOperationContext(); + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrCopy)})); + + auto& opDebug = CurOp::get(*opCtx)->debug(); + ASSERT_EQ(opDebug.queryStatsInfo.wasRateLimited, false); + + // First call to registerRequest() should be rate limited. + QueryStatsStoreManager::getRateLimiter(opCtx->getServiceContext()) = + std::make_unique<RateLimiting>(0, Seconds{1}); + ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() { + return std::make_unique<query_stats::FindKey>( + expCtx, *parsedFind, query_shape::CollectionType::kCollection); + })); + + // Since the query was rate limited, no key should have been created. + ASSERT(opDebug.queryStatsInfo.key == nullptr); + ASSERT_EQ(opDebug.queryStatsInfo.wasRateLimited, true); + + // Second call should not be rate limited. + QueryStatsStoreManager::getRateLimiter(opCtx->getServiceContext()) + .get() + ->setSamplingRate(INT_MAX); + + ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() { + return std::make_unique<query_stats::FindKey>( + expCtx, *parsedFind, query_shape::CollectionType::kCollection); + })); + + // queryStatsKey should not be created for previously rate limited query. + ASSERT(opDebug.queryStatsInfo.key == nullptr); + ASSERT_EQ(opDebug.queryStatsInfo.wasRateLimited, true); + ASSERT_FALSE(opDebug.queryStatsInfo.keyHash.has_value()); +} + +TEST_F(QueryStatsTest, TwoRegisterRequestsWithSameOpCtxDisabledBetween) { + // This test simulates an observed bug where an opCtx is used for two requests, and between the + // first and the second the query stats store is emptied/disabled. + + // Make query for query stats. + const NamespaceString nss = NamespaceString("testDB.testColl"); + FindCommandRequest fcr((NamespaceStringOrUUID(nss))); + fcr.setFilter(BSONObj()); + + auto serviceCtx = getServiceContext(); + auto opCtx = makeOperationContext(); + + auto& opDebug = CurOp::get(*opCtx)->debug(); + ASSERT(opDebug.queryStatsInfo.key == nullptr); + ASSERT_FALSE(opDebug.queryStatsInfo.keyHash.has_value()); + QueryStatsStoreManager::get(serviceCtx) = + std::make_unique<QueryStatsStoreManager>(16 * 1024 * 1024, 1); + + QueryStatsStoreManager::getRateLimiter(serviceCtx) = + std::make_unique<RateLimiting>(-1, Seconds{1}); + + { + auto fcrCopy = std::make_unique<FindCommandRequest>(fcr); + auto expCtx = make_intrusive<ExpressionContext>( + opCtx.get(), *fcrCopy, nullptr, true /* mayDbProfile*/); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrCopy)})); + ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() { + return std::make_unique<query_stats::FindKey>( + expCtx, *parsedFind, query_shape::CollectionType::kCollection); + })); + + ASSERT(opDebug.queryStatsInfo.key != nullptr); + ASSERT(opDebug.queryStatsInfo.keyHash.has_value()); + + ASSERT_DOES_NOT_THROW(query_stats::writeQueryStats(opCtx.get(), + opDebug.queryStatsInfo.keyHash, + std::move(opDebug.queryStatsInfo.key), + 0 /*queryExecMicros*/, + 0 /*firstResponseExecMicros*/, + 0 /*docsReturned*/)); + } + + // Second call should see that query stats are now disabled. + { + // To reproduce SERVER-84730 we need to clear out the query stats store so that writing the + // stats at the end will attempt to insert a new entry. + QueryStatsStoreManager::get(serviceCtx)->resetSize(0); + + auto fcrCopy = std::make_unique<FindCommandRequest>(fcr); + fcrCopy->setFilter(BSON("x" << 1)); + auto expCtx = make_intrusive<ExpressionContext>( + opCtx.get(), *fcrCopy, nullptr, true /* mayDbProfile*/); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrCopy)})); + + ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() { + return std::make_unique<query_stats::FindKey>( + expCtx, *parsedFind, query_shape::CollectionType::kCollection); + })); + + // queryStatsKey should not be created since we have a size budget of 0. + ASSERT(opDebug.queryStatsInfo.key == nullptr); + // This is not a rate limit, but rather a lack of space rendering it entirely disabled. + ASSERT_FALSE(opDebug.queryStatsInfo.wasRateLimited); + + // Interestingly, we purposefully leave the hash value around on the OperationContext after + // the previous operation finishes. This is because we think it may have value in being + // logged in the future, even after query stats have been written. Excepting obscure + // internal use-cases, most OperationContexts will die shortly after the query stats are + // written, so this isn't expected to be a large issue. + ASSERT(opDebug.queryStatsInfo.keyHash.has_value()); + + QueryStatsStoreManager::get(serviceCtx)->resetSize(16 * 1024 * 1024); + // SERVER-84730 this assertion used to throw since there is no key, but there is a hash. + ASSERT_DOES_NOT_THROW(query_stats::writeQueryStats(opCtx.get(), + opDebug.queryStatsInfo.keyHash, + std::move(opDebug.queryStatsInfo.key), + 0 /*queryExecMicros*/, + 0 /*firstResponseExecMicros*/, + 0 /*docsReturned*/)); + } +} + +TEST_F(QueryStatsTest, RegisterRequestAbsorbsErrors) { + const NamespaceString nss = NamespaceString("testDB.testColl"); + + auto opCtx = makeOperationContext(); + auto& opDebug = CurOp::get(*opCtx)->debug(); + + QueryStatsStoreManager::getRateLimiter(getServiceContext()) = + std::make_unique<RateLimiting>(-1, Seconds{1}); + + // First case - don't treat errors as fatal. + internalQueryStatsErrorsAreCommandFatal.store(false); + + // Skip these checks for debug builds because errors are always fatal in that environment. + if (!kDebugBuild) { + ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() { + uasserted(ErrorCodes::BSONObjectTooLarge, "size error"); + return nullptr; + })); + + opDebug.queryStatsInfo = OpDebug::QueryStatsInfo{}; + ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() { + uasserted(ErrorCodes::BadValue, "fake error"); + return nullptr; + })); + } + + // Now make sure that errors are propagated when the knob is set. + internalQueryStatsErrorsAreCommandFatal.store(true); + + // We shouldn't propagate 'BSONObjectTooLarge' errors under any circumstances. + opDebug.queryStatsInfo = OpDebug::QueryStatsInfo{}; + ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() { + uasserted(ErrorCodes::BSONObjectTooLarge, "size error"); + return nullptr; + })); + + // This should hit our assertion. + opDebug.queryStatsInfo = OpDebug::QueryStatsInfo{}; + ASSERT_THROWS(query_stats::registerRequest(opCtx.get(), + nss, + [&]() { + uasserted(ErrorCodes::BadValue, "fake error"); + return nullptr; + }), + DBException); +} + +} // namespace mongo::query_stats diff --git a/src/mongo/db/query/query_stats/rate_limiting.cpp b/src/mongo/db/query/query_stats/rate_limiting.cpp new file mode 100644 index 00000000000..aa8ca645bf1 --- /dev/null +++ b/src/mongo/db/query/query_stats/rate_limiting.cpp @@ -0,0 +1,96 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "rate_limiting.h" +#include "mongo/stdx/mutex.h" +#include "mongo/util/clock_source.h" + +namespace mongo { +RateLimiting::RateLimiting(RequestCount samplingRate, + Milliseconds timePeriod, + ClockSource* clockSource) + : _clockSource(clockSource != nullptr ? clockSource : SystemClockSource::get()), + _samplingRate(samplingRate), + _timePeriod(timePeriod), + _windowStart(_clockSource->now()), + _prevCount(0), + _currentCount(0) {} + +Date_t RateLimiting::tickWindow() { + Date_t currentTime = _clockSource->now(); + + // Elapsed time since window start exceeds the time period. Start a new window. + if (currentTime - _windowStart > _timePeriod) { + _windowStart = currentTime; + _prevCount = _currentCount; + _currentCount = 0; + } + return currentTime; +} + +bool RateLimiting::handleRequestFixedWindow() { + stdx::unique_lock windowLock{_windowMutex}; + tickWindow(); + + if (_currentCount < _samplingRate.load()) { + _currentCount += 1; + return true; + } + return false; +} + +bool RateLimiting::handleRequestSlidingWindow() { + stdx::unique_lock windowLock{_windowMutex}; + + Date_t currentTime = tickWindow(); + auto windowStart = _windowStart; + auto prevCount = _prevCount; + + // Sliding window is implemented over fixed size time periods/blocks as follows. Instead of + // making the decision to limit the rate using only the current time period, we look to the rate + // of the previous period to predicate the rate of the current. This smooths the "sampling" of + // the events by predicting a constant rate and limiting accordingly. + + // Percentage of time remaining in current window. + double percentRemainingOfCurrentWindow = + ((double)(_timePeriod.count() - (currentTime - windowStart).count())) / _timePeriod.count(); + // Estimate the number of requests remaining in the current period. We assume the requests in + // the previous time block occurred at a constant rate. We multiply the total number of requests + // in the previous period by the percentage of time remaining in the current period. + double estimatedRemaining = prevCount * percentRemainingOfCurrentWindow; + // Add this estimate to the requests we know have taken place within the current time block. + double estimatedCount = _currentCount + estimatedRemaining; + + if (estimatedCount < _samplingRate.load()) { + _currentCount += 1; + return true; + } + return false; +} +} // namespace mongo diff --git a/src/mongo/db/query/query_stats/rate_limiting.h b/src/mongo/db/query/query_stats/rate_limiting.h new file mode 100644 index 00000000000..66e38d7119b --- /dev/null +++ b/src/mongo/db/query/query_stats/rate_limiting.h @@ -0,0 +1,126 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/util/clock_source.h" +#include "mongo/util/concurrency/mutex.h" +#include "mongo/util/system_clock_source.h" + +namespace mongo { + +/** + * Rate limiting is used to put a bound on the number of requests to a certain resource over a fixed + * time window. This implementation is approximate in the sense that it may permit the bound to + * exceeded. The bound is approximate as a trade off to reduce contention on internal resources. + */ +class RateLimiting { + using RequestCount = uint32_t; + +public: + /* + * Constructor for a rate limiter. Specify the number of requests you want to take place, as + * well as the time period in milliseconds. + */ + RateLimiting(RequestCount samplingRate, + Milliseconds timePeriod = Seconds{1}, + ClockSource* clockSource = nullptr); + + /* + * Getter for the sampling rate. + */ + RequestCount getSamplingRate() { + return _samplingRate.load(); + } + + /* + * Setter for the sampling rate. + */ + void setSamplingRate(RequestCount samplingRate) { + _samplingRate.store(samplingRate); + } + + /* + * A simple method for rate limiting. Returns false if we have reached the request limit for the + * current time window; otherwise, returns true and adds the request to the count for the + * current window. If we have passed the end of the previous window, the slate is wiped clean. + */ + bool handleRequestFixedWindow(); + + /* + * A method that ensures a more steady rate of requests. Rather than only looking at the current + * time block, this method simulates a sliding window to estimate how many requests occurred in + * the last full time period. Like the above, returns whether the request should be handled, and + * resets the window if enough time has passed. + */ + bool handleRequestSlidingWindow(); + +private: + /* + * Resets the current window if it has ended. Returns the current time. This must be called in + * the beginning of each handleRequest...() method. + */ + Date_t tickWindow(); + + /* + * Clock source used to track time. + */ + ClockSource* const _clockSource; + + /* + * Sampling rate is the bound on the number of requests we want to admit per window. + */ + AtomicWord<RequestCount> _samplingRate; + + /* + * Time period is the window size in ms. + */ + const Milliseconds _timePeriod; + + /* + * Window start. + */ + Date_t _windowStart; + + /* + * Count of requests handled in the previous window. + */ + RequestCount _prevCount; + + /* + * Count of requests handled in the current window. + */ + RequestCount _currentCount; + + /* + * Mutex used when reading/writing the window. + */ + SimpleMutex _windowMutex; +}; +} // namespace mongo diff --git a/src/mongo/db/query/query_stats/rate_limiting_bm.cpp b/src/mongo/db/query/query_stats/rate_limiting_bm.cpp new file mode 100644 index 00000000000..06308e6b0d8 --- /dev/null +++ b/src/mongo/db/query/query_stats/rate_limiting_bm.cpp @@ -0,0 +1,144 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include <benchmark/benchmark.h> +#include <climits> +#include <memory> + +#include "mongo/bson/json.h" +#include "mongo/db/matcher/expression_leaf.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_stats/rate_limiting.h" +#include "mongo/util/duration.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/time_support.h" + +namespace mongo { +namespace { + +// Local testing determined that these parameter values drove the most lock contention, which is +// what we want to capture in this benchmark. +constexpr long long rateLimitedWorkTimeMicros = 5; +constexpr long long consistentWorkTimeMicros = 10; + +constexpr long long numThreads = 256; + +// Rate limit some fraction of the overall work for a request with a sliding window. +int requestWithSlidingWindow(RateLimiting& limit) { + if (limit.handleRequestSlidingWindow()) { + sleepmicros(rateLimitedWorkTimeMicros); + } + sleepmicros(consistentWorkTimeMicros); + return 0; +} + +// Represent a request that bypasses the rate limiter. +int requestUnlimited() { + constexpr long long totalTime = rateLimitedWorkTimeMicros + consistentWorkTimeMicros; + sleepmicros(totalTime); + return 0; +} + +// Represent a request without the rate limited work. +int requestDeactivated() { + sleepmicros(consistentWorkTimeMicros); + return 0; +} + +// Benchmark sliding window rate limiting. +void BM_SlidingWindow(benchmark::State& state) { + // The rate limiter needs a clock source passed in. + static std::unique_ptr<ClockSource> clockSource; + static std::unique_ptr<RateLimiting> rateLimit; + + // Initialize the rate limiter only on the first thread to start up. + if (state.thread_index == 0) { + clockSource = std::make_unique<SystemClockSource>(); + rateLimit = + std::make_unique<RateLimiting>(state.range(0), Milliseconds(1), clockSource.get()); + } + + // Run the benchmark. + for (auto keepRunning : state) { + benchmark::DoNotOptimize(requestWithSlidingWindow(*rateLimit)); + } + + // Clean up the rate limiter when the benchmark is done. + if (state.thread_index == 0) { + rateLimit.reset(); + clockSource.reset(); + } +} + +// "Control" benchmark that does not rate limit requests. In other words, the extra work is always +// done for every request. This benchmark can be thought of as the "goal" performance for the peak, +// or the highest rate limit in BM_SlidingWindow, to compare against. +void BM_Unlimited(benchmark::State& state) { + for (auto keepRunning : state) { + benchmark::DoNotOptimize(requestUnlimited()); + } +} +// Another control benchmark, where the extra work is never done for any request. This can be +// thought of as the goal performance for when rate limit equals 0. +void BM_Deactivated(benchmark::State& state) { + for (auto keepRunning : state) { + benchmark::DoNotOptimize(requestDeactivated()); + } +} + +// Google microbenchmarks report time T (in nanoseconds) spent per operation. But at Mongo we are +// interested in total opereations performed per second. The former can easily be converted to the +// latter by diving 10^6 by T. Use this benchmark to determine the natural throughput of the +// operation. This can be compared to the rate limited benchmarks (BM_SlidingWindow) to determine +// the overhead of rate limiting. Looking at the percentage change in throughput between the control +// benchmarks and the rate limited benchmark, will indicate how much overhead is due to lock +// contention. +BENCHMARK(BM_Unlimited)->Threads(numThreads); + +BENCHMARK(BM_Deactivated)->Threads(numThreads); + +// Local testing has confirmed that the higher the rate limit, the worse the throughput. This makes +// sense as putting a higher upper bound on number of requests allowed in a given time period, means +// longer wait times for the lock. +BENCHMARK(BM_SlidingWindow) + ->ArgName("rate limit") + ->Arg(0) + ->Arg(64) + ->Arg(128) + ->Arg(256) + ->Arg(512) + ->Arg(1024) + ->Arg(2048) + ->Arg(4816) + ->Threads(numThreads); + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/query/query_stats/rate_limiting_test.cpp b/src/mongo/db/query/query_stats/rate_limiting_test.cpp new file mode 100644 index 00000000000..380636a2a20 --- /dev/null +++ b/src/mongo/db/query/query_stats/rate_limiting_test.cpp @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_stats/rate_limiting.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/time_support.h" + +namespace mongo { +TEST(RateLimitingTest, FixedWindowSucceeds) { + auto rl = RateLimiting(1); + ASSERT_TRUE(rl.handleRequestFixedWindow()); +} + +TEST(RateLimitingTest, SlidingWindowSucceeds) { + auto rl = RateLimiting(1); + ASSERT_TRUE(rl.handleRequestSlidingWindow()); +} + +TEST(RateLimitingTest, FixedWindowFails) { + auto rl = RateLimiting(0); + ASSERT_FALSE(rl.handleRequestFixedWindow()); +} + +TEST(RateLimitingTest, SlidingWindowFails) { + auto rl = RateLimiting(0); + ASSERT_FALSE(rl.handleRequestSlidingWindow()); +} + +TEST(RateLimitingTest, FixedWindowSucceedsThenFails) { + auto rl = RateLimiting(1, Hours{1}); + ASSERT_TRUE(rl.handleRequestFixedWindow()); + ASSERT_FALSE(rl.handleRequestFixedWindow()); + ASSERT_FALSE(rl.handleRequestFixedWindow()); +} + +TEST(RateLimitingTest, SlidingWindowSucceedsThenFails) { + auto rl = RateLimiting(1, Hours{1}); + ASSERT_TRUE(rl.handleRequestSlidingWindow()); + ASSERT_FALSE(rl.handleRequestSlidingWindow()); + ASSERT_FALSE(rl.handleRequestSlidingWindow()); +} + +TEST(RateLimitingTest, FixedWindowPermitsRequestAfterWindowExpires) { + auto rl = RateLimiting(1, Milliseconds{10}); + ASSERT_TRUE(rl.handleRequestFixedWindow()); + ASSERT_FALSE(rl.handleRequestFixedWindow()); + sleepmillis(11); + ASSERT_TRUE(rl.handleRequestFixedWindow()); +} + +} // namespace mongo diff --git a/src/mongo/db/query/query_stats/shapifying_bm.cpp b/src/mongo/db/query/query_stats/shapifying_bm.cpp new file mode 100644 index 00000000000..fd7f605c855 --- /dev/null +++ b/src/mongo/db/query/query_stats/shapifying_bm.cpp @@ -0,0 +1,142 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include <benchmark/benchmark.h> +#include <climits> +#include <memory> + +#include "mongo/bson/json.h" +#include "mongo/db/concurrency/locker_noop_client_observer.h" +#include "mongo/db/matcher/expression_leaf.h" +#include "mongo/db/matcher/expression_parser.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_stats/find_key.h" +#include "mongo/db/query/query_stats/query_stats.h" +#include "mongo/db/query/query_stats/rate_limiting.h" +#include "mongo/idl/server_parameter_test_util.h" +#include "mongo/rpc/metadata/client_metadata.h" +#include "mongo/util/duration.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/testing_proctor.h" +#include "mongo/util/time_support.h" + +namespace mongo { +namespace { + +static const NamespaceStringOrUUID kDefaultTestNss = + NamespaceStringOrUUID{NamespaceString("testDB.testColl")}; + +static constexpr auto kCollectionType = query_shape::CollectionType::kCollection; + +// This is a snapshot of the client metadata generated from our IDHACK genny workload. The +// specifics aren't so important, but it chosen in an attempt to be indicative of the size/shape +// of this kind of thing "in the wild". +const auto kMetadataWrapper = fromjson(R"({metadata: { + "application" : { + "name" : "Genny" + }, + "driver" : { + "name" : "mongoc / mongocxx", + "version" : "1.23.2 / 3.7.0" + }, + "os" : { + "type" : "Linux", + "name" : "Ubuntu", + "version" : "22.04", + "architecture" : "aarch64" + }, + "platform" : "cfg=0x03215e88e9 posix=200809 stdc=201710 CC=GCC 11.3.0 CFLAGS=\"-fPIC\" LDFLAGS=\"\"" + }})"); +auto kMockClientMetadataElem = kMetadataWrapper["metadata"]; + +auto makeFindKey(const boost::intrusive_ptr<ExpressionContext>& expCtx, + const ParsedFindCommand& parsedFind) { + return std::make_unique<const query_stats::FindKey>(expCtx, parsedFind, kCollectionType); +} + +int shapifyAndHashRequest(const boost::intrusive_ptr<ExpressionContext>& expCtx, + const ParsedFindCommand& parsedFind) { + auto key = makeFindKey(expCtx, parsedFind); + [[maybe_unused]] auto hash = absl::Hash<query_stats::Key>{}(*key); + return 0; +} + +// Benchmark the performance of computing and hashing the query stats key for an IDHACK query. +void BM_ShapfiyIDHack(benchmark::State& state) { + auto serviceCtx = ServiceContext::make(); + serviceCtx->registerClientObserver(std::make_unique<LockerNoopClientObserver>()); + + auto client = serviceCtx->makeClient("query_test"); + + auto opCtx = client->makeOperationContext(); + auto expCtx = make_intrusive<ExpressionContextForTest>(opCtx.get()); + auto fcr = std::make_unique<FindCommandRequest>(expCtx->ns); + fcr->setFilter(fromjson("{_id: 4}")); + ClientMetadata::setFromMetadata(opCtx->getClient(), kMockClientMetadataElem); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcr))); + + // Run the benchmark. + for (auto keepRunning : state) { + benchmark::DoNotOptimize(shapifyAndHashRequest(expCtx, *parsedFind)); + } +} + +// Benchmark computing the query stats key and its hash for a mildly complex query predicate. +void BM_ShapfiyMildlyComplex(benchmark::State& state) { + auto serviceCtx = ServiceContext::make(); + serviceCtx->registerClientObserver(std::make_unique<LockerNoopClientObserver>()); + + auto client = serviceCtx->makeClient("query_test"); + + auto opCtx = client->makeOperationContext(); + auto expCtx = make_intrusive<ExpressionContextForTest>(opCtx.get()); + auto fcr = std::make_unique<FindCommandRequest>(expCtx->ns); + fcr->setFilter(fromjson(R"({ + clientId: {$nin: ["432345", "4386945", "111111"]}, + nEmployees: {$gte: 4, $lt: 20}, + deactivated: false, + region: "US", + yearlySpend: {$lte: 1000} + })")); + ClientMetadata::setFromMetadata(opCtx->getClient(), kMockClientMetadataElem); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcr))); + + // Run the benchmark. + for (auto keepRunning : state) { + benchmark::DoNotOptimize(shapifyAndHashRequest(expCtx, *parsedFind)); + } +} + +BENCHMARK(BM_ShapfiyIDHack)->Threads(1); +BENCHMARK(BM_ShapfiyMildlyComplex)->Threads(1); + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/query/query_stats/transform_algorithm.idl b/src/mongo/db/query/query_stats/transform_algorithm.idl new file mode 100644 index 00000000000..cd0a5ba43db --- /dev/null +++ b/src/mongo/db/query/query_stats/transform_algorithm.idl @@ -0,0 +1,37 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# +global: + cpp_namespace: "mongo" + +enums: + TransformAlgorithm: + description: "The type of algorithm to be used for the transformIdentifiers field of $queryStats." + type: string + values: + kHmacSha256: "hmac-sha-256" + kNone: "none" diff --git a/src/mongo/db/query/record_id_bound.h b/src/mongo/db/query/record_id_bound.h index 99400ae938d..d6a37617a4f 100644 --- a/src/mongo/db/query/record_id_bound.h +++ b/src/mongo/db/query/record_id_bound.h @@ -29,6 +29,7 @@ #pragma once +#include <boost/operators.hpp> #include <boost/optional.hpp> #include <fmt/format.h> #include <ostream> @@ -44,7 +45,7 @@ namespace mongo { /** * A RecordId bound for a collection scan, with an optional BSON representation for pretty printing. */ -class RecordIdBound { +class RecordIdBound : boost::totally_ordered<RecordIdBound> { public: RecordIdBound() = default; @@ -81,6 +82,14 @@ public: return _recordId.compare(rhs._recordId); } + bool operator==(const RecordIdBound& rhs) const { + return compare(rhs) == 0; + } + + bool operator<(const RecordIdBound& rhs) const { + return compare(rhs) < 0; + } + private: RecordId _recordId; boost::optional<BSONObj> _bson; diff --git a/src/mongo/db/query/record_id_range.cpp b/src/mongo/db/query/record_id_range.cpp new file mode 100644 index 00000000000..88720183ef4 --- /dev/null +++ b/src/mongo/db/query/record_id_range.cpp @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include <boost/optional.hpp> + +#include "mongo/db/query/record_id_range.h" + +namespace mongo { +void RecordIdRange::maybeNarrowMin(const BSONObj& newMin, bool inclusive) { + maybeNarrowMin(RecordIdBound(record_id_helpers::keyForObj(newMin), newMin), inclusive); +} + +void RecordIdRange::maybeNarrowMin(const RecordIdBound& newMin, bool inclusive) { + if (_min) { + auto cmp = _min->compare(newMin); + // The range only needs updating if: + // * There's no existing _min + // * The provided value is greater than the current _min + // * The value == _min, but is _not_ inclusive, but the existing value is + + if (cmp > 0) { + // Current min is strictly greater than the provided value (and existing value has been + // initialised), nothing to do. + return; + } + + if (cmp == 0) { + // Inclusivity moving true -> false narrows the range. + _minInclusive = _minInclusive && inclusive; + return; + } + } + _min = newMin; + // The bound value changed, so the previous value of _minInclusive is irrelevant. + _minInclusive = inclusive; +} + +void RecordIdRange::maybeNarrowMax(const BSONObj& newMax, bool inclusive) { + maybeNarrowMax(RecordIdBound(record_id_helpers::keyForObj(newMax), newMax), inclusive); +} + +void RecordIdRange::maybeNarrowMax(const RecordIdBound& newMax, bool inclusive) { + if (_max) { + auto cmp = _max->compare(newMax); + // The range only needs updating if: + // * There's no existing _max + // * The provided value is less than the current _max + // * The value == _max, but is _not_ inclusive, but the existing value is + + if (cmp < 0) { + // Current max is strictly less than the provided value (and existing value has been + // initialised), nothing to do. + return; + } + + if (cmp == 0) { + // Inclusivity moving true -> false narrows the range. + _maxInclusive = _maxInclusive && inclusive; + return; + } + } + _max = newMax; + // The bound value changed, so the previous value of _maxInclusive is irrelevant. + _maxInclusive = inclusive; +} + +void RecordIdRange::intersectRange(const RecordIdRange& other) { + intersectRange(other._min, other._max, other._minInclusive, other._maxInclusive); +} + +void RecordIdRange::intersectRange(const boost::optional<RecordIdBound>& min, + const boost::optional<RecordIdBound>& max, + bool minInclusive, + bool maxInclusive) { + if (min) { + maybeNarrowMin(*min, minInclusive); + } + if (max) { + maybeNarrowMax(*max, maxInclusive); + } +} + +} // namespace mongo diff --git a/src/mongo/db/query/record_id_range.h b/src/mongo/db/query/record_id_range.h new file mode 100644 index 00000000000..d25e17356c1 --- /dev/null +++ b/src/mongo/db/query/record_id_range.h @@ -0,0 +1,117 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + + +#include <boost/optional.hpp> + +#include "mongo/bson/bsonobj.h" +#include "mongo/db/query/record_id_bound.h" + +namespace mongo { + +class RecordIdRange { +public: + /** + * If the provided value @p newMin is greater than the existing min, + * update the lower bound to equal @p newMin + * + * @return true if range was adjusted + */ + void maybeNarrowMin(const BSONObj& newMin, bool inclusive); + void maybeNarrowMin(const RecordIdBound& newMin, bool inclusive); + + /** + * If the provided value @p newMax is less than the existing max, + * update the upper bound to equal @p newMax + * + * @return true if range was adjusted + */ + void maybeNarrowMax(const BSONObj& newMax, bool inclusive); + void maybeNarrowMax(const RecordIdBound& newMax, bool inclusive); + + /** + * Update this range to the intersection of this range + * and @p other. This may update both, one of, or neither of + * min and max. + * + * Results in a range which is either unchanged, or made + * narrower (possibly becoming an empty range). + */ + void intersectRange(const RecordIdRange& other); + /** + * Overload of intersectRange taking the components of a RecordIdRange, + * for convenience when the other range is not handled as a RecordIdRange. + */ + void intersectRange(const boost::optional<RecordIdBound>& min, + const boost::optional<RecordIdBound>& max, + bool minInclusive = true, + bool maxInclusive = true); + + bool isEmpty() const; + + + const auto& getMin() const { + return _min; + } + + const auto& getMax() const { + return _max; + } + + bool isMinInclusive() const { + return _minInclusive; + } + + bool isMaxInclusive() const { + return _maxInclusive; + } + + +private: + // If present, this parameter sets the start point of a forward scan or the end point of a + // reverse scan. + boost::optional<RecordIdBound> _min; + + // If present, this parameter sets the start point of a reverse scan or the end point of a + // forward scan. + boost::optional<RecordIdBound> _max; + + // TODO: investigate folding this into RecordIdBound; many other usages pair RecordIdBound + // with ScanBoundInclusion to convey this information + // If min is present, this indicates whether the range is inclusive or exclusive of the + // set min value + bool _minInclusive = true; + // If max is present, this indicates whether the range is inclusive or exclusive of the + // set max value + bool _maxInclusive = true; +}; + +} // namespace mongo diff --git a/src/mongo/db/query/record_id_range_test.cpp b/src/mongo/db/query/record_id_range_test.cpp new file mode 100644 index 00000000000..5db866a73d3 --- /dev/null +++ b/src/mongo/db/query/record_id_range_test.cpp @@ -0,0 +1,119 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/base/error_extra_info.h" +#include "mongo/bson/bsonmisc.h" +#include "mongo/db/query/record_id_range.h" + +#include "mongo/bson/bsonmisc.h" +#include "mongo/unittest/unittest.h" + +namespace { + +using namespace mongo; + +template <class BoundsCheck, class NarrowingCheck, class InclusivityCheck> +void testRange(const BoundsCheck& getBoundOptional, + const NarrowingCheck& maybeNarrowBound, + const InclusivityCheck& checkInclusivity, + int initialValue, + int narrowerValue, + int widerValue) { + ASSERT_FALSE(getBoundOptional()); + + auto assertValueEq = [&](auto value) { + auto bson = BSON("value" << value); + auto recordId = record_id_helpers::keyForObj(bson); + ASSERT_EQ(recordId, getBoundOptional()->recordId()); + }; + + // narrow from unset + maybeNarrowBound(BSON("value" << initialValue), true /* inclusive */); + ASSERT_TRUE(getBoundOptional()); + assertValueEq(initialValue); + ASSERT_TRUE(checkInclusivity()); + + // narrow by removing inclusivity of the bound + maybeNarrowBound(BSON("value" << initialValue), false /* not inclusive */); + ASSERT_TRUE(getBoundOptional()); + assertValueEq(initialValue); + ASSERT_FALSE(checkInclusivity()); + + // cannot widen by re-adding inclusivity + maybeNarrowBound(BSON("value" << initialValue), true /* inclusive */); + ASSERT_TRUE(getBoundOptional()); + assertValueEq(initialValue); + ASSERT_FALSE(checkInclusivity()); + + // cannot widen by setting a wider bound + maybeNarrowBound(BSON("value" << widerValue), true /* inclusive */); + ASSERT_TRUE(getBoundOptional()); + assertValueEq(initialValue); + ASSERT_FALSE(checkInclusivity()); + + // cannot widen by setting a wider bound, regardless of inclusivity + maybeNarrowBound(BSON("value" << widerValue), false /* not inclusive */); + ASSERT_TRUE(getBoundOptional()); + assertValueEq(initialValue); + ASSERT_FALSE(checkInclusivity()); + + // narrow to a non-inclusive bound at a narrower value + maybeNarrowBound(BSON("value" << narrowerValue), false /* not inclusive */); + ASSERT_TRUE(getBoundOptional()); + assertValueEq(narrowerValue); + ASSERT_FALSE(checkInclusivity()); +} + +TEST(RecordIdRangeTest, NarrowMin) { + RecordIdRange range; + + testRange([&] { return range.getMin(); }, + [&](const BSONObj& newVal, bool inclusive) { + return range.maybeNarrowMin(newVal, inclusive); + }, + [&] { return range.isMinInclusive(); }, + 10, + 11, + 9); +} + +TEST(RecordIdRangeTest, NarrowMax) { + RecordIdRange range; + + testRange([&] { return range.getMax(); }, + [&](const BSONObj& newVal, bool inclusive) { + return range.maybeNarrowMax(newVal, inclusive); + }, + [&] { return range.isMaxInclusive(); }, + 10, + 9, + 11); +} + +} // namespace diff --git a/src/mongo/db/query/sbe_cached_solution_planner.cpp b/src/mongo/db/query/sbe_cached_solution_planner.cpp index c594c70f22e..cd7f23016eb 100644 --- a/src/mongo/db/query/sbe_cached_solution_planner.cpp +++ b/src/mongo/db/query/sbe_cached_solution_planner.cpp @@ -74,6 +74,7 @@ CandidatePlans CachedSolutionPlanner::plan( std::move(roots[0].first), std::move(roots[0].second), maxReadsBeforeReplan); + auto explainer = plan_explainer_factory::make( candidate.root.get(), &candidate.data, diff --git a/src/mongo/db/query/sbe_plan_cache.cpp b/src/mongo/db/query/sbe_plan_cache.cpp index c8944057639..70d864e7074 100644 --- a/src/mongo/db/query/sbe_plan_cache.cpp +++ b/src/mongo/db/query/sbe_plan_cache.cpp @@ -31,7 +31,7 @@ #include "mongo/db/query/sbe_plan_cache.h" -#include "mongo/db/query/plan_cache_size_parameter.h" +#include "mongo/db/query/util/memory_util.h" #include "mongo/db/server_options.h" #include "mongo/logv2/log.h" #include "mongo/util/processinfo.h" @@ -42,71 +42,23 @@ namespace { const auto sbePlanCacheDecoration = ServiceContext::declareDecoration<std::unique_ptr<sbe::PlanCache>>(); -size_t convertToSizeInBytes(const plan_cache_util::PlanCacheSizeParameter& param) { - constexpr size_t kBytesInMB = 1024 * 1024; - constexpr size_t kMBytesInGB = 1024; - - double sizeInMB = param.size; - - switch (param.units) { - case plan_cache_util::PlanCacheSizeUnits::kPercent: - sizeInMB *= ProcessInfo::getMemSizeMB() / 100.0; - break; - case plan_cache_util::PlanCacheSizeUnits::kMB: - break; - case plan_cache_util::PlanCacheSizeUnits::kGB: - sizeInMB *= kMBytesInGB; - break; - } - - return static_cast<size_t>(sizeInMB * kBytesInMB); -} - -/** - * Sets upper size limit on the PlanCache size to 500GB or 25% of the system's memory, whichever is - * smaller. - */ -size_t capPlanCacheSize(size_t planCacheSize) { - constexpr size_t kBytesInGB = 1024 * 1024 * 1024; - - // Maximum size of the plan cache expressed in bytes. - constexpr size_t kMaximumPlanCacheSize = 500 * kBytesInGB; - - // Maximum size of the plan cache expressed as a share of the memory available to the process. - const plan_cache_util::PlanCacheSizeParameter limitToProcessSize{ - 25, plan_cache_util::PlanCacheSizeUnits::kPercent}; - const size_t limitToProcessSizeInBytes = convertToSizeInBytes(limitToProcessSize); - - // The size will be capped by the minimum of the two values defined above. - const size_t maxPlanCacheSize = std::min(kMaximumPlanCacheSize, limitToProcessSizeInBytes); - - if (planCacheSize > maxPlanCacheSize) { - planCacheSize = maxPlanCacheSize; - LOGV2_DEBUG(6007000, - 1, - "The plan cache size has been capped", - "maxPlanCacheSize"_attr = maxPlanCacheSize); - } - - return planCacheSize; -} - -size_t getPlanCacheSizeInBytes(const plan_cache_util::PlanCacheSizeParameter& param) { - size_t planCacheSize = convertToSizeInBytes(param); - uassert(5968001, - "Cache size must be at least 1KB * number of cores", - planCacheSize >= 1024 * ProcessInfo::getNumCores()); - return capPlanCacheSize(planCacheSize); -} class PlanCacheOnParamChangeUpdaterImpl final : public plan_cache_util::OnParamChangeUpdater { public: - void updateCacheSize(ServiceContext* serviceCtx, - plan_cache_util::PlanCacheSizeParameter parameter) final { + void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) final { if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) { - auto size = getPlanCacheSizeInBytes(parameter); + auto newSizeBytes = memory_util::getRequestedMemSizeInBytes(memSize); + auto cappedCacheSize = memory_util::capMemorySize(newSizeBytes /*requestedSizeBytes*/, + 500 /*maximumSizeGB*/, + 25 /*percentTotalSystemMemory*/); + if (cappedCacheSize < newSizeBytes) { + LOGV2_DEBUG(6007001, + 1, + "The plan cache size has been capped", + "cappedSize"_attr = cappedCacheSize); + } auto& globalPlanCache = sbePlanCacheDecoration(serviceCtx); - globalPlanCache->reset(size); + globalPlanCache->reset(cappedCacheSize); } } @@ -124,12 +76,21 @@ ServiceContext::ConstructorActionRegisterer planCacheRegisterer{ std::make_unique<PlanCacheOnParamChangeUpdaterImpl>(); if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) { - auto status = plan_cache_util::PlanCacheSizeParameter::parse(planCacheSize.get()); + auto status = memory_util::MemorySize::parse(planCacheSize.get()); uassertStatusOK(status); - - auto size = getPlanCacheSizeInBytes(status.getValue()); + auto size = memory_util::getRequestedMemSizeInBytes(status.getValue()); + auto cappedCacheSize = memory_util::capMemorySize(size /*requestedSizeBytes*/, + 500 /*maximumSizeGB*/, + 25 /*percentTotalSystemMemory*/); + if (cappedCacheSize < size) { + LOGV2_DEBUG(6007000, + 1, + "The plan cache size has been capped", + "cappedSize"_attr = cappedCacheSize); + } auto& globalPlanCache = sbePlanCacheDecoration(serviceCtx); - globalPlanCache = std::make_unique<sbe::PlanCache>(size, ProcessInfo::getNumCores()); + globalPlanCache = + std::make_unique<sbe::PlanCache>(cappedCacheSize, ProcessInfo::getNumCores()); } }}; diff --git a/src/mongo/db/query/sbe_plan_cache.h b/src/mongo/db/query/sbe_plan_cache.h index 6e7853fa817..090df251bdf 100644 --- a/src/mongo/db/query/sbe_plan_cache.h +++ b/src/mongo/db/query/sbe_plan_cache.h @@ -190,7 +190,14 @@ struct CachedSbePlan { using PlanCacheEntry = PlanCacheEntryBase<CachedSbePlan, plan_cache_debug_info::DebugInfoSBE>; struct BudgetEstimator { - size_t operator()(const std::shared_ptr<const PlanCacheEntry>& entry) { + /** + * This estimator function is called when an entry is added or removed to LRU cache in order to + * make sure the total plan cache size does not exceed the maximum size. + */ + size_t operator()(const sbe::PlanCacheKey& key, + const std::shared_ptr<const PlanCacheEntry>& entry) { + // TODO: SERVER-73649 include size of underlying query shape and size of int_32 key hash in + // total size estimation. return entry->estimatedEntrySizeBytes; } }; diff --git a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.cpp b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.cpp index 4279a142e4b..9f6cbef6d4f 100644 --- a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.cpp +++ b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.cpp @@ -56,7 +56,7 @@ Status clearSbeCacheOnParameterChangeHelper() { } Status onPlanCacheSizeUpdate(const std::string& str) { - auto newSize = PlanCacheSizeParameter::parse(str); + auto newSize = memory_util::MemorySize::parse(str); if (!newSize.isOK()) { return newSize.getStatus(); } diff --git a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h index e126dda9f48..d0f8a026a05 100644 --- a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h +++ b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h @@ -32,7 +32,7 @@ #include <string> #include "mongo/base/status.h" -#include "mongo/db/query/plan_cache_size_parameter.h" +#include "mongo/db/query/util/memory_util.h" #include "mongo/db/service_context.h" namespace mongo::plan_cache_util { @@ -70,11 +70,11 @@ public: virtual ~OnParamChangeUpdater() = default; /** - * Resizes the SBE plan cache decorating 'serviceCtx' to the new size given by 'parameter'. If + * Resizes the SBE plan cache decorating 'serviceCtx' to the new size given by 'memSize'. If * the new cache size is smaller than the old, cache entries are evicted in order to ensure the * cache fits within the new size bound. */ - virtual void updateCacheSize(ServiceContext* serviceCtx, PlanCacheSizeParameter parameter) = 0; + virtual void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) = 0; /** * Deletes all plans from the SBE plan cache decorating 'serviceCtx'. diff --git a/src/mongo/db/query/sbe_stage_builder_expression.cpp b/src/mongo/db/query/sbe_stage_builder_expression.cpp index 89a8c2d68ff..8ecc4961435 100644 --- a/src/mongo/db/query/sbe_stage_builder_expression.cpp +++ b/src/mongo/db/query/sbe_stage_builder_expression.cpp @@ -2122,12 +2122,52 @@ public: std::vector<EvalExprStagePair> branches; branches.reserve(numChildren); + auto childStageCount = 0; for (size_t i = 0; i < numChildren; ++i) { auto [expr, stage] = _context->popFrame(); + if (stage.stage.get() != nullptr) { + childStageCount++; + } branches.emplace_back(std::move(expr), std::move(stage)); } std::reverse(branches.begin(), branches.end()); + // If there is no separate child stage branch, then we can implement $ifNull as a simple + // projection of SBE if expression, instead of with union stages. + if (childStageCount == 0) { + auto stage = _context->extractCurrentEvalStage(); + + std::vector<sbe::value::SlotId> slots; + slots.reserve(branches.size()); + sbe::value::SlotMap<std::unique_ptr<sbe::EExpression>> projects; + for (auto& branch : branches) { + if (branch.first.getSlot()) { + slots.push_back(*branch.first.getSlot()); + } else { + auto slot = _context->state.slotId(); + slots.push_back(slot); + projects.emplace(slot, branch.first.extractExpr()); + } + } + if (!projects.empty()) { + stage = makeProject(std::move(stage), std::move(projects), _context->planNodeId); + } + + auto expr = sbe::makeE<sbe::EVariable>(slots[slots.size() - 1]); + for (int i = slots.size() - 2; i >= 0; i--) { + auto thenExpr = sbe::makeE<sbe::EVariable>(slots[i]); + auto condExpr = makeNot(generateNullOrMissing(thenExpr->clone())); + expr = + sbe::makeE<sbe::EIf>(std::move(condExpr), std::move(thenExpr), std::move(expr)); + } + + auto outSlot = _context->state.slotId(); + stage = makeProject(std::move(stage), _context->planNodeId, outSlot, std::move(expr)); + + _context->pushExpr(outSlot, std::move(stage)); + return; + } + // Prepare to create limit-1/union with N branches (where N is the number of operands). Each // branch will be evaluated from left to right until one of the branches produces a value. auto branchFn = [](EvalExpr evalExpr, diff --git a/src/mongo/db/query/sort_pattern.cpp b/src/mongo/db/query/sort_pattern.cpp index fcd3cd177e1..5a444cd31f2 100644 --- a/src/mongo/db/query/sort_pattern.cpp +++ b/src/mongo/db/query/sort_pattern.cpp @@ -112,13 +112,13 @@ QueryMetadataBitSet SortPattern::metadataDeps(QueryMetadataBitSet unavailableMet return depsTracker.metadataDeps(); } -Document SortPattern::serialize(SortKeySerialization serializationMode) const { +Document SortPattern::serialize(SortKeySerialization serializationMode, + const SerializationOptions& options) const { MutableDocument keyObj; const size_t n = _sortPattern.size(); for (size_t i = 0; i < n; ++i) { if (_sortPattern[i].fieldPath) { - // Append a named integer based on whether the sort is ascending/descending. - keyObj.setField(_sortPattern[i].fieldPath->fullPath(), + keyObj.setField(options.serializeFieldPath(*_sortPattern[i].fieldPath), Value(_sortPattern[i].isAscending ? 1 : -1)); } else { // Sorting by an expression, use a made up field name. @@ -127,7 +127,12 @@ Document SortPattern::serialize(SortKeySerialization serializationMode) const { case SortKeySerialization::kForExplain: case SortKeySerialization::kForPipelineSerialization: { const bool isExplain = (serializationMode == SortKeySerialization::kForExplain); - keyObj[computedFieldName] = _sortPattern[i].expression->serialize(isExplain); + auto opts = SerializationOptions{}; + if (isExplain) { + opts.verbosity = + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner); + } + keyObj[computedFieldName] = _sortPattern[i].expression->serialize(opts); break; } case SortKeySerialization::kForSortKeyMerging: { diff --git a/src/mongo/db/query/sort_pattern.h b/src/mongo/db/query/sort_pattern.h index b659ed0124e..c709a43eac8 100644 --- a/src/mongo/db/query/sort_pattern.h +++ b/src/mongo/db/query/sort_pattern.h @@ -33,6 +33,7 @@ #include "mongo/db/exec/document_value/document.h" #include "mongo/db/pipeline/document_path_support.h" #include "mongo/db/pipeline/expression.h" +#include "mongo/db/query/query_shape/serialization_options.h" namespace mongo { class SortPattern { @@ -72,7 +73,8 @@ public: /** * Write out a Document whose contents are the sort key pattern. */ - Document serialize(SortKeySerialization) const; + Document serialize(SortKeySerialization serializationMode, + const SerializationOptions& options = {}) const; /** * Serializes the document to BSON, only keeping the paths specified in the sort pattern. diff --git a/src/mongo/db/query/sort_pattern_test.cpp b/src/mongo/db/query/sort_pattern_test.cpp new file mode 100644 index 00000000000..2abe1bffa81 --- /dev/null +++ b/src/mongo/db/query/sort_pattern_test.cpp @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "query_shape/serialization_options.h" + +#include "mongo/db/exec/document_value/document.h" +#include "mongo/db/exec/document_value/document_value_test_util.h" +#include "mongo/db/exec/document_value/value.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/sort_pattern.h" +#include "mongo/unittest/unittest.h" +namespace mongo { +namespace { + +auto getExpCtx() { + auto nss = NamespaceString("db", "coll"); + return boost::intrusive_ptr<ExpressionContextForTest>{new ExpressionContextForTest(nss)}; +} + +TEST(SerializeSortPatternTest, SerializeAndRedactFieldName) { + auto expCtx = getExpCtx(); + auto sortPattern = SortPattern(fromjson("{val: 1}"), expCtx); + SerializationOptions opts = SerializationOptions::kMarkIdentifiers_FOR_TEST; + + // Most basic sort pattern, confirm that field name gets redacted. + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<val>":1})", + sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts)); + + // Confirm that multiple sort fields get redacted. + sortPattern = SortPattern(fromjson("{val: 1, test: -1, third: -1}"), expCtx); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<val>":1,"HASH<test>":-1,"HASH<third>":-1})", + sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts)); + + // Test sort pattern that contains an expression. + sortPattern = SortPattern(fromjson("{val: 1, test: {$meta: \"randVal\"}}"), expCtx); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<val>":1,"$computed1":{"$meta":"randVal"}})", + sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts)); + + // Sorting by only an expression results in a made up field name in serialization and therefore + // doesn't get redacted. + sortPattern = SortPattern(fromjson("{val: {$meta: \"textScore\"}}"), expCtx); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$computed0":{"$meta":"textScore"}})", + sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts)); + + sortPattern = SortPattern(fromjson("{'a.b.c': 1}"), expCtx); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"HASH<a>.HASH<b>.HASH<c>":1})", + sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts)); +} + +TEST(SerializeSortPatternTest, SerializeNoRedaction) { + auto expCtx = getExpCtx(); + auto sortPattern = SortPattern(fromjson("{val: 1}"), expCtx); + SerializationOptions opts = {}; + opts.transformIdentifiers = false; + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"val":1})", + sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts)); + + // Call serialize() with no options. + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"val":1})", + sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization)); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/query/util/deferred.h b/src/mongo/db/query/util/deferred.h new file mode 100644 index 00000000000..a2609bb6b49 --- /dev/null +++ b/src/mongo/db/query/util/deferred.h @@ -0,0 +1,118 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <functional> + +namespace mongo { + +/** + * A template class that provides a way to defer the initialization of an object until its value is + * actually required. This is also commonly referred to as lazy initialization. + * + * Dangers: + * - This implementation is currently not thread safe, and it shouldn't be used in multi-threaded + * fashion. + * - Be careful about using this for lazy initialization of data members and capturing the 'this' + * variable. Code like this will result in buggy/unsafe move constructors, which would have a + * dangling reference to the moved-from type: + * + * class MyType { + * int x; + * // !!! Dangling 'this' when moved !!! + * Deferred<int> xSquared{[this]() { return this->x * this-> x; }; + * }; + * Instead, it is better to do something like this: + * class MyType { + * int xSquared() const { + * return *_xSquared.get(_x); + * } + * + * int _x; + * Deferred<int, int> _xSquared{[](int x) { return x * x; }; + * }; + * - As a similar danger, the value is only computed once. if you initialize it with arguments like + * the above 'xSquared()' implementation, then be cogniscent that the value will never change. If + * '_x' changes, '_xSquared' will not. + * + * A Deferred class can be constructed with either an initial value (eager initialization) or a + * function that will generate the value when needed. + */ +template <typename T, typename... Args> +class Deferred { +public: + /** + * Instantiates a Deffered<T> with the given data - no callbacks or lazy initialization. + */ + Deferred(T data) : _data(data) {} + + /** + * Stores a function to compute a T later. Please note the warnings described in this class + * comment. + */ + Deferred(std::function<T(Args&&...)> initializer) : _initializer(std::move(initializer)) {} + + /** + * Returns a pointer to the managed object. Initializes the object if it hasn't done so already. + */ + T& get(Args&&... args) const { + if (_initializer) { + _data = _initializer(std::forward<Args>(args)...); + _initializer = nullptr; + } + return _data; + } + + /** + * Dereferences the pointer to the managed object. Note this is only a valid shortcut if there + * are no arguments to '_initializer'. + */ + T* operator->() const { + return &get(); + } + + /** + * Returns a referenced to the managed object. Initializes the object if it hasn't done so + * already. Note this is only a valid shortcut if there are no arguments to '_initializer'. + */ + const T& operator*() const { + return get(); + } + + bool isInitialized() const { + return _initializer ? false : true; + } + +private: + mutable T _data; + mutable std::function<T(Args&&...)> _initializer; +}; + +} // namespace mongo diff --git a/src/mongo/db/query/util/deferred_test.cpp b/src/mongo/db/query/util/deferred_test.cpp new file mode 100644 index 00000000000..de256394787 --- /dev/null +++ b/src/mongo/db/query/util/deferred_test.cpp @@ -0,0 +1,98 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/util/deferred.h" + +#include "mongo/unittest/unittest.h" + +namespace mongo { +using std::string; +using namespace std::string_literals; + + +TEST(DeferredTest, EagerInitialization) { + Deferred<string> eager{"someString"}; + ASSERT_TRUE(eager.isInitialized()); + ASSERT_EQ(eager.get(), "someString"s); + ASSERT_EQ(*eager, "someString"s); +} + +TEST(DeferredTest, DeferredInitialization) { + size_t initializationCount = 0; + Deferred<string> deferred{[&]() { + initializationCount++; + return "someString"s; + }}; + ASSERT_FALSE(deferred.isInitialized()); + + // Ensure the deferred object wasn't initialized on creation. + ASSERT_EQ(initializationCount, 0); + + // Ensure that the deferred object is initialized on pointer dereferences. + ASSERT_FALSE(deferred->empty()); + ASSERT_TRUE(deferred.isInitialized()); + + ASSERT_EQ(initializationCount, 1); + + // Ensure that the content of the deferred object is equal to its raw counterpart, while also + // verifing that it is initialized at most once. + ASSERT_EQ(deferred.get(), "someString"s); + ASSERT_EQ(initializationCount, 1); +} + +TEST(DeferredTest, DeferredInitializationWithOneArgument) { + size_t initializationCount = 0; + Deferred<string, const string&> deferred{[&](const string& input) { + initializationCount++; + return "{" + input + "}"; + }}; + + // Ensure the deferred object wasn't initialized on creation. + ASSERT_EQ(initializationCount, 0); + + // Ensure that the content of the deferred object is equal to its raw counterpart, while also + // verifing that it is initialized at most once. + ASSERT_EQ(deferred.get("more curlies"), "{more curlies}"s); + ASSERT_EQ(initializationCount, 1); + + // Note that the value is cached, so it's not really valid to call it with a different argument. + ASSERT_EQ(deferred.get("merganser"), "{more curlies}"s); + ASSERT_EQ(initializationCount, 1); +} + +TEST(DeferredTest, DeferredInitializationWithTwoArgs) { + Deferred<string, const string&, const string&> deferred{ + [&](const auto& input, const auto& prefix) { return prefix + input; }}; + + ASSERT_EQ(deferred.get("cowbell", "more "), "more cowbell"s); + ASSERT_EQ(deferred.get("cowbell", "more "), "more cowbell"s); + ASSERT_EQ(deferred.get("cowbell", "less?"), "more cowbell"s); + ASSERT_EQ(deferred.get("tests", "better"), "more cowbell"s); +} +} // namespace mongo diff --git a/src/mongo/db/query/util/memory_util.cpp b/src/mongo/db/query/util/memory_util.cpp new file mode 100644 index 00000000000..8a206deb2b7 --- /dev/null +++ b/src/mongo/db/query/util/memory_util.cpp @@ -0,0 +1,128 @@ +/** + * Copyright (C) 2021-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + +#include "mongo/db/query/util/memory_util.h" + +#include <cstddef> +#include <pcrecpp.h> + +#include "mongo/logv2/log.h" +#include "mongo/util/processinfo.h" + + +namespace mongo::memory_util { + +StatusWith<MemoryUnits> parseUnitString(const std::string& strUnit) { + if (strUnit.empty()) { + return Status(ErrorCodes::Error{6007010}, "Unit value cannot be empty"); + } + + if (strUnit[0] == '%') { + return MemoryUnits::kPercent; + } else if (strUnit[0] == 'M' || strUnit[0] == 'm') { + return MemoryUnits::kMB; + } else if (strUnit[0] == 'G' || strUnit[0] == 'g') { + return MemoryUnits::kGB; + } + + return Status(ErrorCodes::Error{6007011}, "Incorrect unit value"); +} + +StatusWith<MemorySize> MemorySize::parse(const std::string& str) { + pcrecpp::RE_Options opt; + opt.set_caseless(true); + // Looks for a floating point number with followed by a unit suffix (MB, GB, %). + pcrecpp::RE re("\\s*(\\d+\\.?\\d*)\\s*(MB|GB|%)\\s*", opt); + + double size{}; + std::string strUnit{}; + if (!re.FullMatch(str, &size, &strUnit)) { + return {ErrorCodes::Error{6007012}, "Unable to parse memory size string"}; + } + + auto statusWithUnit = parseUnitString(strUnit); + if (!statusWithUnit.isOK()) { + return statusWithUnit.getStatus(); + } + return MemorySize{size, statusWithUnit.getValue()}; +} + +size_t convertToSizeInBytes(const MemorySize& memSize) { + constexpr size_t kBytesInMB = 1024 * 1024; + constexpr size_t kMBytesInGB = 1024; + + double sizeInMB = memSize.size; + + switch (memSize.units) { + case MemoryUnits::kPercent: + sizeInMB *= ProcessInfo::getMemSizeMB() / 100.0; + break; + case MemoryUnits::kMB: + break; + case MemoryUnits::kGB: + sizeInMB *= kMBytesInGB; + break; + } + + return static_cast<size_t>(sizeInMB * kBytesInMB); +} + +size_t getRequestedMemSizeInBytes(const MemorySize& memSize) { + size_t planCacheSize = convertToSizeInBytes(memSize); + uassert(5968001, + "Cache size must be at least 1KB * number of cores", + planCacheSize >= 1024 * ProcessInfo::getNumCores()); + return planCacheSize; +} + +/** + * Sets upper limit on a storage structure's size. Either that structure's maximumSize or to + * percentage of the total system's memory (both known at call site), whichever is smaller. + */ +size_t capMemorySize(size_t requestedSizeBytes, + size_t maximumSizeGB, + double percentTotalSystemMemory) { + constexpr size_t kBytesInGB = 1024 * 1024 * 1024; + // Express maximum size in bytes. + const size_t maximumSizeBytes = maximumSizeGB * kBytesInGB; + const memory_util::MemorySize limitToProcessSize{percentTotalSystemMemory, + memory_util::MemoryUnits::kPercent}; + const size_t limitToProcessSizeInBytes = convertToSizeInBytes(limitToProcessSize); + + // The size will be capped by the minimum of the two values defined above. + const size_t upperLimit = std::min(maximumSizeBytes, limitToProcessSizeInBytes); + + if (requestedSizeBytes > upperLimit) { + requestedSizeBytes = upperLimit; + } + return requestedSizeBytes; +} +} // namespace mongo::memory_util diff --git a/src/mongo/db/query/plan_cache_size_parameter.h b/src/mongo/db/query/util/memory_util.h index 322a1fff564..345780b4c84 100644 --- a/src/mongo/db/query/plan_cache_size_parameter.h +++ b/src/mongo/db/query/util/memory_util.h @@ -31,29 +31,36 @@ #include <string> +#include "mongo/base/error_codes.h" #include "mongo/base/status_with.h" -namespace mongo::plan_cache_util { +namespace mongo::memory_util { /** - * Defines units of planCacheSize parameter. + * Defines units of memory. */ -enum class PlanCacheSizeUnits { +enum class MemoryUnits { kPercent, kMB, kGB, }; -StatusWith<PlanCacheSizeUnits> parseUnitString(const std::string& strUnit); - /** - * Represents parsed planCacheSize parameter. + * Represents parsed memory size parameter. */ -struct PlanCacheSizeParameter { - static StatusWith<PlanCacheSizeParameter> parse(const std::string& str); +struct MemorySize { + static StatusWith<MemorySize> parse(const std::string& str); const double size; - const PlanCacheSizeUnits units; + const MemoryUnits units; }; -} // namespace mongo::plan_cache_util +StatusWith<MemoryUnits> parseUnitString(const std::string& strUnit); +size_t convertToSizeInBytes(const MemorySize& memSize); +size_t capMemorySize(size_t requestedSizeBytes, + size_t maximumSizeGB, + double percentTotalSystemMemory); +size_t getRequestedMemSizeInBytes(const MemorySize& memSize); + + +} // namespace mongo::memory_util diff --git a/src/mongo/db/query/util/memory_util_test.cpp b/src/mongo/db/query/util/memory_util_test.cpp new file mode 100644 index 00000000000..78f7b3098d6 --- /dev/null +++ b/src/mongo/db/query/util/memory_util_test.cpp @@ -0,0 +1,73 @@ +/** + * Copyright (C) 2021-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/util/memory_util.h" + +#include "mongo/unittest/unittest.h" + +namespace mongo::memory_util { + +bool operator==(const MemorySize& lhs, const MemorySize& rhs) { + constexpr double kEpsilon = 1e-10; + return std::abs(lhs.size - rhs.size) < kEpsilon && lhs.units == rhs.units; +} + +TEST(MemorySizeTest, ParseUnitStringPercent) { + ASSERT_TRUE(MemoryUnits::kPercent == parseUnitString("%")); +} + +TEST(MemorySizeTest, ParseUnitStringMB) { + ASSERT_TRUE(MemoryUnits::kMB == parseUnitString("MB")); + ASSERT_TRUE(MemoryUnits::kMB == parseUnitString("mb")); + ASSERT_TRUE(MemoryUnits::kMB == parseUnitString("mB")); + ASSERT_TRUE(MemoryUnits::kMB == parseUnitString("Mb")); +} + +TEST(MemorySizeTest, ParseUnitStringGB) { + ASSERT_TRUE(MemoryUnits::kGB == parseUnitString("GB")); + ASSERT_TRUE(MemoryUnits::kGB == parseUnitString("gb")); + ASSERT_TRUE(MemoryUnits::kGB == parseUnitString("gB")); + ASSERT_TRUE(MemoryUnits::kGB == parseUnitString("Gb")); +} + +TEST(MemorySizeTest, ParseUnitStringIncorrectValue) { + ASSERT_NOT_OK(parseUnitString("").getStatus()); + ASSERT_NOT_OK(parseUnitString(" ").getStatus()); + ASSERT_NOT_OK(parseUnitString("KB").getStatus()); +} + +TEST(MemorySizeTest, ParseMemorySize) { + ASSERT_TRUE((MemorySize{10.0, MemoryUnits::kPercent}) == MemorySize::parse("10%")); + ASSERT_TRUE((MemorySize{300.0, MemoryUnits::kMB}) == MemorySize::parse("300MB")); + ASSERT_TRUE((MemorySize{4.0, MemoryUnits::kGB}) == MemorySize::parse("4GB")); + ASSERT_TRUE((MemorySize{5.1, MemoryUnits::kPercent}) == MemorySize::parse(" 5.1%")); + ASSERT_TRUE((MemorySize{11.1, MemoryUnits::kMB}) == MemorySize::parse("11.1 mb")); + ASSERT_TRUE((MemorySize{12.1, MemoryUnits::kGB}) == MemorySize::parse(" 12.1 Gb ")); +} +} // namespace mongo::memory_util diff --git a/src/mongo/db/query/view_response_formatter.cpp b/src/mongo/db/query/view_response_formatter.cpp index 76c8457c8b5..397f5e6769a 100644 --- a/src/mongo/db/query/view_response_formatter.cpp +++ b/src/mongo/db/query/view_response_formatter.cpp @@ -28,6 +28,8 @@ */ #include "mongo/platform/basic.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/str.h" #include "mongo/db/query/view_response_formatter.h" @@ -57,7 +59,11 @@ Status ViewResponseFormatter::appendAsCountResponse(BSONObjBuilder* resultBuilde } else { invariant(cursorFirstBatch.size() == 1); auto countObj = cursorFirstBatch.back(); - resultBuilder->append(kCountField, countObj["count"].Int()); + auto countElem = countObj["count"]; + tassert(9384400, + str::stream() << "the 'count' should be of number type, but found " << countElem, + countElem.isNumber()); + resultBuilder->appendAs(countElem, kCountField); } resultBuilder->append(kOkField, 1); return Status::OK(); diff --git a/src/mongo/db/query/view_response_formatter_test.cpp b/src/mongo/db/query/view_response_formatter_test.cpp index dc86c5c9fc7..c0d710d838e 100644 --- a/src/mongo/db/query/view_response_formatter_test.cpp +++ b/src/mongo/db/query/view_response_formatter_test.cpp @@ -51,6 +51,16 @@ TEST(ViewResponseFormatter, FormatInitialCountResponseSuccessfully) { ASSERT_BSONOBJ_EQ(fromjson("{'n': 7, ok: 1}"), builder.obj()); } +TEST(ViewResponseFormatter, FormatInitialCountResponseWithNumberLong) { + CursorResponse cr( + testNss, testCursor, {BSON("count" << std::numeric_limits<long long>::max())}); + ViewResponseFormatter formatter(cr.toBSON(CursorResponse::ResponseType::InitialResponse)); + BSONObjBuilder builder; + ASSERT_OK(formatter.appendAsCountResponse(&builder)); + ASSERT_BSONOBJ_EQ(BSON("n" << std::numeric_limits<long long>::max() << "ok" << 1), + builder.obj()); +} + TEST(ViewResponseFormatter, FormatSubsequentCountResponseSuccessfully) { CursorResponse cr(testNss, testCursor, {BSON("count" << 7)}); ViewResponseFormatter formatter(cr.toBSON(CursorResponse::ResponseType::SubsequentResponse)); @@ -59,6 +69,16 @@ TEST(ViewResponseFormatter, FormatSubsequentCountResponseSuccessfully) { ASSERT_BSONOBJ_EQ(fromjson("{'n': 7, ok: 1}"), builder.obj()); } +TEST(ViewResponseFormatter, FormatSubsequentCountResponseWithLong) { + CursorResponse cr( + testNss, testCursor, {BSON("count" << std::numeric_limits<long long>::max())}); + ViewResponseFormatter formatter(cr.toBSON(CursorResponse::ResponseType::SubsequentResponse)); + BSONObjBuilder builder; + ASSERT_OK(formatter.appendAsCountResponse(&builder)); + ASSERT_BSONOBJ_EQ(BSON("n" << std::numeric_limits<long long>::max() << "ok" << 1), + builder.obj()); +} + TEST(ViewResponseFormatter, FormatEmptyInitialCountResponseSuccessfully) { CursorResponse cr(testNss, testCursor, {}); ViewResponseFormatter formatter(cr.toBSON(CursorResponse::ResponseType::InitialResponse)); diff --git a/src/mongo/db/record_id.h b/src/mongo/db/record_id.h index 21a0d7e53f3..91eed920098 100644 --- a/src/mongo/db/record_id.h +++ b/src/mongo/db/record_id.h @@ -30,8 +30,10 @@ #pragma once #include <boost/functional/hash.hpp> +#include <boost/operators.hpp> #include <boost/optional.hpp> #include <climits> +#include <cstddef> #include <cstdint> #include <fmt/format.h> #include <ostream> @@ -47,7 +49,7 @@ namespace mongo { /** * The key that uniquely identifies a Record in a Collection or RecordStore. */ -class RecordId { +class RecordId : boost::totally_ordered<RecordId> { public: // This set of constants define the boundaries of the 'normal' id ranges for the int64_t format. static constexpr int64_t kMinRepr = LLONG_MIN; @@ -237,6 +239,14 @@ public: MONGO_UNREACHABLE; } + bool operator==(const RecordId& rhs) const { + return compare(rhs) == 0; + } + + bool operator<(const RecordId& rhs) const { + return compare(rhs) < 0; + } + size_t hash() const { size_t hash = 0; withFormat([](Null n) {}, @@ -424,25 +434,6 @@ private: ConstSharedBuffer _sharedBuffer; }; -inline bool operator==(const RecordId& lhs, const RecordId& rhs) { - return lhs.compare(rhs) == 0; -} -inline bool operator!=(const RecordId& lhs, const RecordId& rhs) { - return lhs.compare(rhs); -} -inline bool operator<(const RecordId& lhs, const RecordId& rhs) { - return lhs.compare(rhs) < 0; -} -inline bool operator<=(const RecordId& lhs, const RecordId& rhs) { - return lhs.compare(rhs) <= 0; -} -inline bool operator>(const RecordId& lhs, const RecordId& rhs) { - return lhs.compare(rhs) > 0; -} -inline bool operator>=(const RecordId& lhs, const RecordId& rhs) { - return lhs.compare(rhs) >= 0; -} - inline StringBuilder& operator<<(StringBuilder& stream, const RecordId& id) { return stream << "RecordId(" << id.toString() << ')'; } diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index 22eb5f4cbb4..bf3ff27ed2e 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -40,6 +40,16 @@ env.Library( ) env.Library( + target='oplog_constraint_violation_logger', + source=[ + 'oplog_constraint_violation_logger.cpp', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/base', + ], +) + +env.Library( target='oplog', source=[ 'apply_ops.cpp', @@ -55,6 +65,7 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/catalog/catalog_helpers', '$BUILD_DIR/mongo/db/catalog/database_holder', + '$BUILD_DIR/mongo/db/catalog/health_log_interface', '$BUILD_DIR/mongo/db/catalog/import_collection_oplog_entry', '$BUILD_DIR/mongo/db/catalog/index_build_oplog_entry', '$BUILD_DIR/mongo/db/catalog/local_oplog_info', @@ -77,6 +88,7 @@ env.Library( '$BUILD_DIR/mongo/rpc/command_status', 'dbcheck', 'image_collection_entry', + 'oplog_constraint_violation_logger', 'repl_coordinator_interface', 'repl_server_parameters', 'repl_settings', @@ -196,6 +208,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/catalog/health_log_interface', '$BUILD_DIR/mongo/db/db_raii', '$BUILD_DIR/mongo/idl/idl_parser', @@ -205,6 +218,7 @@ env.Library( '$BUILD_DIR/mongo/db/query_exec', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/util/md5', + 'repl_server_parameters', ], ) @@ -589,7 +603,7 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/db/auth/authorization_manager_global', '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/curop', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query_exec', '$BUILD_DIR/mongo/db/stats/timer_stats', '$BUILD_DIR/mongo/db/storage/storage_options', @@ -1475,11 +1489,11 @@ env.Library( '$BUILD_DIR/mongo/db/auth/auth', '$BUILD_DIR/mongo/db/cloner', '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/kill_sessions_local', '$BUILD_DIR/mongo/db/logical_time', '$BUILD_DIR/mongo/db/not_primary_error_tracker', '$BUILD_DIR/mongo/db/op_observer', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/query_exec', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/stats/counters', @@ -1562,6 +1576,8 @@ if wiredtiger: LIBDEPS=[ '$BUILD_DIR/mongo/db/catalog/database_holder', '$BUILD_DIR/mongo/db/catalog/document_validation', + '$BUILD_DIR/mongo/db/catalog/health_log', + '$BUILD_DIR/mongo/db/catalog/health_log_interface', '$BUILD_DIR/mongo/db/index_builds_coordinator_interface', '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/service_context_d_test_fixture', @@ -1678,6 +1694,7 @@ if wiredtiger: '$BUILD_DIR/mongo/db/auth/authmocks', '$BUILD_DIR/mongo/db/auth/authorization_manager_global', '$BUILD_DIR/mongo/db/catalog/catalog_helpers', + '$BUILD_DIR/mongo/db/catalog/health_log', '$BUILD_DIR/mongo/db/catalog_raii', '$BUILD_DIR/mongo/db/commands/feature_compatibility_parsers', '$BUILD_DIR/mongo/db/commands/mongod_fcv', diff --git a/src/mongo/db/repl/collection_bulk_loader_impl.cpp b/src/mongo/db/repl/collection_bulk_loader_impl.cpp index 73f86c67c59..eb39fdadd55 100644 --- a/src/mongo/db/repl/collection_bulk_loader_impl.cpp +++ b/src/mongo/db/repl/collection_bulk_loader_impl.cpp @@ -82,49 +82,54 @@ Status CollectionBulkLoaderImpl::init(const std::vector<BSONObj>& secondaryIndex // locks as yielding a MODE_X/MODE_S lock isn't allowed. _secondaryIndexesBlock->setIndexBuildMethod(IndexBuildMethod::kForeground); _idIndexBlock->setIndexBuildMethod(IndexBuildMethod::kForeground); - return writeConflictRetry( - _opCtx.get(), - "CollectionBulkLoader::init", - _collection->getNss().ns(), - [&secondaryIndexSpecs, this] { - WriteUnitOfWork wuow(_opCtx.get()); - // All writes in CollectionBulkLoaderImpl should be unreplicated. - // The opCtx is accessed indirectly through _secondaryIndexesBlock. - UnreplicatedWritesBlock uwb(_opCtx.get()); - // This enforces the buildIndexes setting in the replica set configuration. - CollectionWriter collWriter(_opCtx.get(), *_collection); - auto indexCatalog = collWriter.getWritableCollection()->getIndexCatalog(); - auto specs = indexCatalog->removeExistingIndexesNoChecks( - _opCtx.get(), collWriter.get(), secondaryIndexSpecs); - if (specs.size()) { - _secondaryIndexesBlock->ignoreUniqueConstraint(); - auto status = - _secondaryIndexesBlock - ->init(_opCtx.get(), collWriter, specs, MultiIndexBlock::kNoopOnInitFn) - .getStatus(); - if (!status.isOK()) { - return status; - } - } else { - _secondaryIndexesBlock.reset(); - } - if (!_idIndexSpec.isEmpty()) { - auto status = _idIndexBlock - ->init(_opCtx.get(), - collWriter, - _idIndexSpec, - MultiIndexBlock::kNoopOnInitFn) - .getStatus(); - if (!status.isOK()) { - return status; - } - } else { - _idIndexBlock.reset(); - } - - wuow.commit(); - return Status::OK(); - }); + return writeConflictRetry(_opCtx.get(), + "CollectionBulkLoader::init", + _collection->getNss().ns(), + [&secondaryIndexSpecs, this] { + WriteUnitOfWork wuow(_opCtx.get()); + // All writes in CollectionBulkLoaderImpl should be + // unreplicated. The opCtx is accessed indirectly through + // _secondaryIndexesBlock. + UnreplicatedWritesBlock uwb(_opCtx.get()); + // This enforces the buildIndexes setting in the replica set + // configuration. + CollectionWriter collWriter(_opCtx.get(), *_collection); + auto indexCatalog = + collWriter.getWritableCollection()->getIndexCatalog(); + auto specs = indexCatalog->removeExistingIndexesNoChecks( + _opCtx.get(), collWriter.get(), secondaryIndexSpecs); + if (specs.size()) { + _secondaryIndexesBlock->ignoreUniqueConstraint(); + auto status = _secondaryIndexesBlock + ->init(_opCtx.get(), + collWriter, + specs, + MultiIndexBlock::kNoopOnInitFn, + /*forRecovery=*/false) + .getStatus(); + if (!status.isOK()) { + return status; + } + } else { + _secondaryIndexesBlock.reset(); + } + if (!_idIndexSpec.isEmpty()) { + auto status = _idIndexBlock + ->init(_opCtx.get(), + collWriter, + _idIndexSpec, + MultiIndexBlock::kNoopOnInitFn) + .getStatus(); + if (!status.isOK()) { + return status; + } + } else { + _idIndexBlock.reset(); + } + + wuow.commit(); + return Status::OK(); + }); }); } diff --git a/src/mongo/db/repl/dbcheck.cpp b/src/mongo/db/repl/dbcheck.cpp index c02ebb9512b..cc140a71817 100644 --- a/src/mongo/db/repl/dbcheck.cpp +++ b/src/mongo/db/repl/dbcheck.cpp @@ -27,6 +27,8 @@ * it in the license file. */ +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand + #include "mongo/platform/basic.h" #include "mongo/bson/simple_bsonelement_comparator.h" @@ -43,10 +45,13 @@ #include "mongo/db/repl/dbcheck_gen.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/optime.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" +#include "mongo/logv2/log.h" namespace mongo { MONGO_FAIL_POINT_DEFINE(SleepDbCheckInBatch); +MONGO_FAIL_POINT_DEFINE(hangAfterGeneratingHashForExtraIndexKeysCheck); namespace { @@ -409,6 +414,14 @@ Status dbCheckBatchOnSecondary(OperationContext* opCtx, // every N batches. HealthLogInterface::get(opCtx)->log(*logEntry); } + + if (MONGO_unlikely(hangAfterGeneratingHashForExtraIndexKeysCheck.shouldFail())) { + LOGV2_DEBUG(3083200, + 3, + "Hanging due to hangAfterGeneratingHashForExtraIndexKeysCheck failpoint"); + // hangAfterGeneratingHashForExtraIndexKeysCheck.pauseWhileSet(opCtx); + opCtx->sleepFor(Milliseconds(1000)); + } } catch (const DBException& exception) { // In case of an error, report it to the health log, auto logEntry = dbCheckErrorHealthLogEntry( @@ -438,7 +451,6 @@ Status dbCheckOplogCommand(OperationContext* opCtx, IDLParserErrorContext ctx("o"); auto skipDbCheck = mode != OplogApplication::Mode::kSecondary; - auto severity = skipDbCheck ? SeverityEnum::Warning : SeverityEnum::Info; std::string oplogApplicationMode; if (mode == OplogApplication::Mode::kInitialSync) { oplogApplicationMode = "initial sync"; @@ -459,7 +471,7 @@ Status dbCheckOplogCommand(OperationContext* opCtx, // TODO SERVER-78399: Clean up handling minKey/maxKey once feature flag is removed. // If the dbcheck oplog entry doesn't contain batchStart, convert minKey to a BSONObj to // be used as batchStart. - BSONObj batchStart, batchEnd; + BSONObj batchStart, batchEnd, batchId; if (!invocation.getBatchStart()) { batchStart = BSON("_id" << invocation.getMinKey().elem()); } else { @@ -472,21 +484,42 @@ Status dbCheckOplogCommand(OperationContext* opCtx, } */ - if (!skipDbCheck) { + if (!skipDbCheck && !repl::skipApplyingDbCheckBatchOnSecondary.load()) { return dbCheckBatchOnSecondary(opCtx, opTime, invocation); } + // TODO SERVER-89921: Uncomment once the relevant tickets are backported. /* + if (invocation.getBatchId()) { + batchId = invocation.getBatchId().get().toBSON(); + } + BSONObjBuilder data; data.append("batchStart", batchStart); data.append("batchEnd", batchEnd); + + if (!batchId.isEmpty()) { + data.append("batchId", batchId); + } */ + + auto warningMsg = "cannot execute dbcheck due to ongoing " + oplogApplicationMode; + if (repl::skipApplyingDbCheckBatchOnSecondary.load()) { + warningMsg = + "skipping applying dbcheck batch because the " + "'skipApplyingDbCheckBatchOnSecondary' parameter is on"; + } + + LOGV2_DEBUG(8888500, 3, "skipping applying dbcheck batch", "reason"_attr = warningMsg); + // TODO SERVER-89921: Uncomment these logging attributes once the relevant tickets are + // backported. + //"batchStart"_attr = batchStart, + //"batchEnd"_attr = batchEnd, + //"batchId"_attr = batchId); + auto healthLogEntry = mongo::dbCheckHealthLogEntry( - invocation.getNss(), - SeverityEnum::Warning, - "cannot execute dbcheck due to ongoing " + oplogApplicationMode, - type, - boost::none /*data*/); + invocation.getNss(), SeverityEnum::Warning, warningMsg, type, boost::none /*data*/); + HealthLogInterface::get(Client::getCurrent()->getServiceContext()) ->log(*healthLogEntry); return Status::OK(); @@ -500,7 +533,7 @@ Status dbCheckOplogCommand(OperationContext* opCtx, case OplogEntriesEnum::Stop: const auto healthLogEntry = mongo::dbCheckHealthLogEntry( boost::none /*nss*/, - severity, + skipDbCheck ? SeverityEnum::Warning : SeverityEnum::Info, skipDbCheck ? "cannot execute dbcheck due to ongoing " + oplogApplicationMode : "", type, boost::none /*data*/ diff --git a/src/mongo/db/repl/member_config.h b/src/mongo/db/repl/member_config.h index cc668fef79c..94d76991b50 100644 --- a/src/mongo/db/repl/member_config.h +++ b/src/mongo/db/repl/member_config.h @@ -118,7 +118,7 @@ public: * Gets the horizon name for which the parameters (captured during the first `hello`) * correspond. */ - StringData determineHorizon(const SplitHorizon::Parameters& params) const { + std::string determineHorizon(const SplitHorizon::Parameters& params) const { return _splitHorizon.determineHorizon(params); } diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 9edc5f48b4f..3e79a96362e 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -45,6 +45,7 @@ #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/catalog/backwards_compatible_collection_options_util.h" #include "mongo/db/catalog/capped_utils.h" #include "mongo/db/catalog/coll_mod.h" #include "mongo/db/catalog/collection.h" @@ -55,6 +56,7 @@ #include "mongo/db/catalog/drop_collection.h" #include "mongo/db/catalog/drop_database.h" #include "mongo/db/catalog/drop_indexes.h" +#include "mongo/db/catalog/health_log_interface.h" #include "mongo/db/catalog/import_collection_oplog_entry_gen.h" #include "mongo/db/catalog/local_oplog_info.h" #include "mongo/db/catalog/multi_index_block.h" @@ -987,7 +989,8 @@ const StringMap<ApplyOpMetadata> kOpsMap = { {ErrorCodes::NamespaceNotFound}}}, {"collMod", {[](OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) -> Status { - const auto& cmd = entry.getObject(); + const auto cmd = + backwards_compatible_collection_options::parseCollModCmdFromOplogEntry(entry); auto opMsg = OpMsgRequest::fromDBAndBody(entry.getNss().db(), cmd); auto collModCmd = CollMod::parse(IDLParserErrorContext("collModOplogEntry"), opMsg); const auto nssOrUUID([&collModCmd, &entry, mode]() -> NamespaceStringOrUUID { @@ -1217,6 +1220,36 @@ void OplogApplication::checkOnOplogFailureForRecovery(OperationContext* opCtx, } } +// Logger for oplog constraint violations. +OplogConstraintViolationLogger* oplogConstraintViolationLogger; + +MONGO_INITIALIZER(CreateOplogConstraintViolationLogger)(InitializerContext* context) { + oplogConstraintViolationLogger = new OplogConstraintViolationLogger(); +} + +void logOplogConstraintViolation(OperationContext* opCtx, + const NamespaceString& nss, + OplogConstraintViolationEnum type, + const std::string& operation, + const BSONObj& opObj, + boost::optional<Status> status) { + // Log the violation. + oplogConstraintViolationLogger->logViolationIfReady(type, opObj, status); + + // Write a new entry to the health log. + HealthLogEntry entry; + entry.setNss(nss); + entry.setTimestamp(Date_t::now()); + // Oplog constraint violations should always be marked as warning. + entry.setSeverity(SeverityEnum::Warning); + entry.setScope(ScopeEnum::Document); + entry.setMsg(toString(type)); + entry.setOperation(operation); + entry.setData(opObj); + + HealthLogInterface::get(opCtx->getServiceContext())->log(entry); +} + // @return failure status if an update should have happened and the document DNE. // See replset initial sync code. Status applyOperation_inlock(OperationContext* opCtx, @@ -1495,7 +1528,17 @@ Status applyOperation_inlock(OperationContext* opCtx, return status; } if (mode == OplogApplication::Mode::kSecondary) { + const auto& opObj = redact(op.toBSONForLogging()); + opCounters->gotInsertOnExistingDoc(); + logOplogConstraintViolation( + opCtx, + op.getNss(), + OplogConstraintViolationEnum::kInsertOnExistingDoc, + "insert", + opObj, + boost::none /* status */); + if (oplogApplicationEnforcesSteadyStateConstraints) { return status; } @@ -1719,10 +1762,15 @@ Status applyOperation_inlock(OperationContext* opCtx, !ur.upsertedId.isEmpty() && !(collection && collection->isCapped())) { // This indicates we upconverted an update to an upsert, and it did indeed // upsert. In steady state mode this is unexpected. - LOGV2_WARNING(2170001, - "update needed to be converted to upsert", - "op"_attr = redact(op.toBSONForLogging())); + const auto& opObj = redact(op.toBSONForLogging()); + opCounters->gotUpdateOnMissingDoc(); + logOplogConstraintViolation(opCtx, + op.getNss(), + OplogConstraintViolationEnum::kUpdateOnMissingDoc, + "update", + opObj, + boost::none /* status */); // We shouldn't be doing upserts in secondary mode when enforcing steady state // constraints. @@ -1858,13 +1906,15 @@ Status applyOperation_inlock(OperationContext* opCtx, // It is legal for a delete operation on the pre-images collection to delete zero // documents - pre-image collections are not guaranteed to contain the same set of // documents at all times. + // + // It is also legal for a delete operation on the config.image_collection (used for + // find-and-modify retries) to delete zero documents. Since we do not write updates + // to this collection which are in the same batch as later deletes, a rollback to + // the middle of a batch with both an update and a delete may result in a missing + // document, which may be later deleted. if (result.nDeleted == 0 && mode == OplogApplication::Mode::kSecondary && - !requestNss.isChangeStreamPreImagesCollection()) { - LOGV2_WARNING(2170002, - "Applied a delete which did not delete anything in steady state " - "replication", - "op"_attr = redact(op.toBSONForLogging())); - + !requestNss.isChangeStreamPreImagesCollection() && + !requestNss.isConfigImagesCollection()) { // In FCV 4.4, each node is responsible for deleting the excess documents in // capped collections. This implies that capped deletes may not be synchronized // between nodes at times. When upgraded to FCV 5.0, the primary will generate @@ -1877,11 +1927,25 @@ Status applyOperation_inlock(OperationContext* opCtx, // capped collections when oplog application is enforcing steady state // constraints. bool isCapped = false; + const auto& opObj = redact(op.toBSONForLogging()); if (collection) { isCapped = collection->isCapped(); opCounters->gotDeleteWasEmpty(); + logOplogConstraintViolation(opCtx, + op.getNss(), + OplogConstraintViolationEnum::kDeleteWasEmpty, + "delete", + opObj, + boost::none /* status */); } else { opCounters->gotDeleteFromMissingNamespace(); + logOplogConstraintViolation( + opCtx, + op.getNss(), + OplogConstraintViolationEnum::kDeleteOnMissingNs, + "delete", + opObj, + boost::none /* status */); } if (!isCapped) { @@ -2134,12 +2198,15 @@ Status applyCommand_inlock(OperationContext* opCtx, if (mode == OplogApplication::Mode::kSecondary && status.code() != ErrorCodes::IndexNotFound) { - LOGV2_WARNING(2170000, - "Acceptable error during oplog application", - "db"_attr = nss.db(), - "error"_attr = status, - "oplogEntry"_attr = redact(entry.toBSONForLogging())); + const auto& opObj = redact(entry.toBSONForLogging()); opCounters->gotAcceptableErrorInCommand(); + logOplogConstraintViolation( + opCtx, + entry.getNss(), + OplogConstraintViolationEnum::kAcceptableErrorInCommand, + "command", + opObj, + status); } else { LOGV2_DEBUG(51776, 1, diff --git a/src/mongo/db/repl/oplog.h b/src/mongo/db/repl/oplog.h index 1752c2e64e6..3df367954b0 100644 --- a/src/mongo/db/repl/oplog.h +++ b/src/mongo/db/repl/oplog.h @@ -38,6 +38,7 @@ #include "mongo/bson/timestamp.h" #include "mongo/db/catalog/collection_options.h" #include "mongo/db/logical_session_id.h" +#include "mongo/db/repl/oplog_constraint_violation_logger.h" #include "mongo/db/repl/oplog_entry.h" #include "mongo/db/repl/oplog_entry_or_grouped_inserts.h" #include "mongo/db/repl/optime.h" @@ -220,6 +221,16 @@ inline std::ostream& operator<<(std::ostream& s, OplogApplication::Mode mode) { } /** + * Logs an oplog constraint violation and writes an entry into the health log. + */ +void logOplogConstraintViolation(OperationContext* opCtx, + const NamespaceString& nss, + OplogConstraintViolationEnum type, + const std::string& operation, + const BSONObj& opObj, + boost::optional<Status> status); + +/** * Used for applying from an oplog entry or grouped inserts. * @param opOrGroupedInserts a single oplog entry or grouped inserts to be applied. * @param alwaysUpsert convert some updates to upserts for idempotency reasons diff --git a/src/mongo/db/repl/oplog_applier_impl.cpp b/src/mongo/db/repl/oplog_applier_impl.cpp index 4f8c3f9427e..3ce566c713b 100644 --- a/src/mongo/db/repl/oplog_applier_impl.cpp +++ b/src/mongo/db/repl/oplog_applier_impl.cpp @@ -103,6 +103,11 @@ Status finishAndLogApply(OperationContext* opCtx, attrs.add("duration", Milliseconds(opDuration)); + // Obtain storage specific statistics and log them if they exist. + CurOp::get(opCtx)->debug().storageStats = + opCtx->recoveryUnit()->computeOperationStatisticsSinceLastCall(); + CurOp::get(opCtx)->debug().reportStorageStats(&attrs); + LOGV2(51801, "Applied op", attrs); } } diff --git a/src/mongo/db/repl/oplog_applier_impl_test.cpp b/src/mongo/db/repl/oplog_applier_impl_test.cpp index 68b1b6fdb47..26e142f6372 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test.cpp +++ b/src/mongo/db/repl/oplog_applier_impl_test.cpp @@ -67,7 +67,6 @@ #include "mongo/db/session_txn_record_gen.h" #include "mongo/db/stats/counters.h" #include "mongo/db/transaction_participant_gen.h" -#include "mongo/idl/server_parameter_test_util.h" #include "mongo/platform/mutex.h" #include "mongo/unittest/death_test.h" #include "mongo/unittest/unittest.h" @@ -129,7 +128,7 @@ TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, NamespaceString otherNss("test.othername"); auto op = makeOplogEntry(OpTypeEnum::kDelete, otherNss, {}); int prevDeleteFromMissing = replOpCounters.getDeleteFromMissingNamespace()->load(); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, false); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, otherNss, false); auto postDeleteFromMissing = replOpCounters.getDeleteFromMissingNamespace()->load(); ASSERT_EQ(1, postDeleteFromMissing - prevDeleteFromMissing); @@ -167,7 +166,7 @@ TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, NamespaceString otherNss(nss.getSisterNS("othername")); auto op = makeOplogEntry(OpTypeEnum::kDelete, otherNss, kUuid); int prevDeleteFromMissing = replOpCounters.getDeleteFromMissingNamespace()->load(); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, false); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, false); auto postDeleteFromMissing = replOpCounters.getDeleteFromMissingNamespace()->load(); ASSERT_EQ(1, postDeleteFromMissing - prevDeleteFromMissing); @@ -211,7 +210,7 @@ TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, // implicitly create the collection. auto op = makeOplogEntry(OpTypeEnum::kDelete, nss, {}); int prevDeleteFromMissing = replOpCounters.getDeleteFromMissingNamespace()->load(); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, false); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, false); ASSERT_FALSE(collectionExists(_opCtx.get(), nss)); auto postDeleteFromMissing = replOpCounters.getDeleteFromMissingNamespace()->load(); ASSERT_EQ(1, postDeleteFromMissing - prevDeleteFromMissing); @@ -239,7 +238,7 @@ TEST_F(OplogApplierImplTest, applyOplogEntryOrGroupedInsertsInsertDocumentCollec const NamespaceString nss("test.t"); repl::createCollection(_opCtx.get(), nss, {}); auto op = makeOplogEntry(OpTypeEnum::kInsert, nss, {}); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, true); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, true); } TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, @@ -248,7 +247,7 @@ TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, repl::createCollection(_opCtx.get(), nss, {}); auto op = makeOplogEntry(OpTypeEnum::kDelete, nss, {}); int prevDeleteWasEmpty = replOpCounters.getDeleteWasEmpty()->load(); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, false); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, false); auto postDeleteWasEmpty = replOpCounters.getDeleteWasEmpty()->load(); ASSERT_EQ(1, postDeleteWasEmpty - prevDeleteWasEmpty); @@ -274,7 +273,7 @@ TEST_F(OplogApplierImplTest, applyOplogEntryOrGroupedInsertsDeleteDocumentCollec createCollection(_opCtx.get(), nss, createRecordPreImageCollectionOptions()); ASSERT_OK(getStorageInterface()->insertDocument(_opCtx.get(), nss, {BSON("_id" << 0)}, 0)); auto op = makeOplogEntry(OpTypeEnum::kDelete, nss, {}); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, true); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, true); } TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, @@ -284,7 +283,7 @@ TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, ASSERT_OK(getStorageInterface()->insertDocument(_opCtx.get(), nss, {BSON("_id" << 0)}, 0)); auto op = makeOplogEntry(OpTypeEnum::kInsert, nss, uuid); int prevInsertOnExistingDoc = replOpCounters.getInsertOnExistingDoc()->load(); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, false); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, false); auto postInsertOnExistingDoc = replOpCounters.getInsertOnExistingDoc()->load(); ASSERT_EQ(1, postInsertOnExistingDoc - prevInsertOnExistingDoc); @@ -301,7 +300,7 @@ TEST_F(OplogApplierImplTestEnableSteadyStateConstraints, auto uuid = createCollectionWithUuid(_opCtx.get(), nss); ASSERT_OK(getStorageInterface()->insertDocument(_opCtx.get(), nss, {BSON("_id" << 0)}, 0)); auto op = makeOplogEntry(OpTypeEnum::kInsert, nss, uuid); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::DuplicateKey, op, false); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::DuplicateKey, op, nss, false); } TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, @@ -311,7 +310,7 @@ TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, auto op = makeOplogEntry( repl::OpTypeEnum::kUpdate, nss, uuid, BSON("$set" << BSON("a" << 1)), BSON("_id" << 0)); int prevUpdateOnMissingDoc = replOpCounters.getUpdateOnMissingDoc()->load(); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, true); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, true); auto postUpdateOnMissingDoc = replOpCounters.getUpdateOnMissingDoc()->load(); ASSERT_EQ(1, postUpdateOnMissingDoc - prevUpdateOnMissingDoc); @@ -328,7 +327,8 @@ TEST_F(OplogApplierImplTestEnableSteadyStateConstraints, auto uuid = createCollectionWithUuid(_opCtx.get(), nss); auto op = makeOplogEntry( repl::OpTypeEnum::kUpdate, nss, uuid, BSON("$set" << BSON("a" << 1)), BSON("_id" << 0)); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::UpdateOperationFailed, op, false); + _testApplyOplogEntryOrGroupedInsertsCrudOperation( + ErrorCodes::UpdateOperationFailed, op, nss, false); } TEST_F(OplogApplierImplTest, applyOplogEntryOrGroupedInsertsInsertDocumentCollectionLockedByUUID) { @@ -337,7 +337,7 @@ TEST_F(OplogApplierImplTest, applyOplogEntryOrGroupedInsertsInsertDocumentCollec // Test that the collection to lock is determined by the UUID and not the 'ns' field. NamespaceString otherNss(nss.getSisterNS("othername")); auto op = makeOplogEntry(OpTypeEnum::kInsert, otherNss, uuid); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, true); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, true); } TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, @@ -351,7 +351,7 @@ TEST_F(OplogApplierImplTestDisableSteadyStateConstraints, NamespaceString otherNss(nss.getSisterNS("othername")); auto op = makeOplogEntry(OpTypeEnum::kDelete, otherNss, options.uuid); int prevDeleteWasEmpty = replOpCounters.getDeleteWasEmpty()->load(); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, false); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, false); auto postDeleteWasEmpty = replOpCounters.getDeleteWasEmpty()->load(); ASSERT_EQ(1, postDeleteWasEmpty - prevDeleteWasEmpty); @@ -388,7 +388,7 @@ TEST_F(OplogApplierImplTest, applyOplogEntryOrGroupedInsertsDeleteDocumentCollec // Test that the collection to lock is determined by the UUID and not the 'ns' field. NamespaceString otherNss(nss.getSisterNS("othername")); auto op = makeOplogEntry(OpTypeEnum::kDelete, otherNss, options.uuid); - _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, true); + _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::OK, op, nss, true); } TEST_F(OplogApplierImplTest, applyOplogEntryToRecordChangeStreamPreImages) { diff --git a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp index 35f06edd12e..4afe05e7bc5 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp +++ b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp @@ -32,6 +32,8 @@ #include "mongo/db/repl/oplog_applier_impl_test_fixture.h" #include "mongo/db/catalog/document_validation.h" +#include "mongo/db/catalog/health_log.h" +#include "mongo/db/catalog/health_log_interface.h" #include "mongo/db/concurrency/exception_util.h" #include "mongo/db/curop.h" #include "mongo/db/db_raii.h" @@ -182,14 +184,22 @@ void OplogApplierImplTest::setUp() { // This is necessary to generate ghost timestamps for index builds that are not 0, since 0 is an // invalid timestamp. VectorClockMutable::get(_opCtx.get())->tickClusterTimeTo(LogicalTime(Timestamp(1, 0))); + + HealthLogInterface::set(serviceContext, std::make_unique<HealthLog>()); + HealthLogInterface::get(serviceContext)->startup(); } void OplogApplierImplTest::tearDown() { + HealthLogInterface::get(serviceContext)->shutdown(); _opCtx.reset(); _consistencyMarkers = {}; DropPendingCollectionReaper::set(serviceContext, {}); StorageInterface::set(serviceContext, {}); ServiceContextMongoDTest::tearDown(); + + for (auto serverParamController : _serverParamControllers) { + serverParamController.reset(); + } } ReplicationConsistencyMarkers* OplogApplierImplTest::getConsistencyMarkers() const { @@ -213,7 +223,10 @@ Status OplogApplierImplTest::_applyOplogEntryOrGroupedInsertsWrapper( } void OplogApplierImplTest::_testApplyOplogEntryOrGroupedInsertsCrudOperation( - ErrorCodes::Error expectedError, const OplogEntry& op, bool expectedApplyOpCalled) { + ErrorCodes::Error expectedError, + const OplogEntry& op, + const NamespaceString& targetNss, + bool expectedApplyOpCalled) { bool applyOpCalled = false; auto checkOpCtx = [](OperationContext* opCtx) { @@ -228,9 +241,14 @@ void OplogApplierImplTest::_testApplyOplogEntryOrGroupedInsertsCrudOperation( _opObserver->onInsertsFn = [&](OperationContext* opCtx, const NamespaceString& nss, const std::vector<BSONObj>& docs) { + // Other threads may be calling into the opObserver. Only assert if we are writing to + // the target ns, otherwise skip these asserts. + if (targetNss != nss) { + return Status::OK(); + } + applyOpCalled = true; checkOpCtx(opCtx); - ASSERT_EQUALS(NamespaceString("test.t"), nss); ASSERT_EQUALS(1U, docs.size()); // For upserts we don't know the intended value of the document. if (op.getOpType() == repl::OpTypeEnum::kInsert) { @@ -244,18 +262,28 @@ void OplogApplierImplTest::_testApplyOplogEntryOrGroupedInsertsCrudOperation( const boost::optional<UUID>& uuid, StmtId stmtId, const OplogDeleteEntryArgs& args) { + // Other threads may be calling into the opObserver. Only assert if we are writing to + // the target ns, otherwise skip these asserts. + if (targetNss != nss) { + return Status::OK(); + } + applyOpCalled = true; checkOpCtx(opCtx); - ASSERT_EQUALS(NamespaceString("test.t"), nss); ASSERT(args.deletedDoc); ASSERT_BSONOBJ_EQ(op.getObject(), *(args.deletedDoc)); return Status::OK(); }; _opObserver->onUpdateFn = [&](OperationContext* opCtx, const OplogUpdateEntryArgs& args) { + // Other threads may be calling into the opObserver. Only assert if we are writing to + // the target ns, otherwise skip these asserts. + if (targetNss != args.nss) { + return Status::OK(); + } + applyOpCalled = true; checkOpCtx(opCtx); - ASSERT_EQUALS(NamespaceString("test.t"), args.nss); return Status::OK(); }; diff --git a/src/mongo/db/repl/oplog_applier_impl_test_fixture.h b/src/mongo/db/repl/oplog_applier_impl_test_fixture.h index e1b188232ae..28709a29b2c 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test_fixture.h +++ b/src/mongo/db/repl/oplog_applier_impl_test_fixture.h @@ -38,6 +38,7 @@ #include "mongo/db/repl/replication_consistency_markers.h" #include "mongo/db/service_context_d_test_fixture.h" #include "mongo/db/session_txn_record_gen.h" +#include "mongo/idl/server_parameter_test_util.h" namespace mongo { @@ -202,6 +203,7 @@ protected: void _testApplyOplogEntryOrGroupedInsertsCrudOperation(ErrorCodes::Error expectedError, const OplogEntry& op, + const NamespaceString& targetNss, bool expectedApplyOpCalled); Status _applyOplogEntryOrGroupedInsertsWrapper(OperationContext* opCtx, @@ -213,6 +215,11 @@ protected: ServiceContext* serviceContext; OplogApplierImplOpObserver* _opObserver = nullptr; + template <typename T> + inline void setServerParameter(const std::string& name, T value) { + _serverParamControllers.push_back(ServerParameterControllerForTest(name, value)); + } + OpTime nextOpTime() { static long long lastSecond = 1; return OpTime(Timestamp(Seconds(lastSecond++), 0), 1LL); @@ -232,6 +239,8 @@ protected: Status runOpsInitialSync(std::vector<OplogEntry> ops); UUID kUuid{UUID::gen()}; + + std::vector<ServerParameterControllerForTest> _serverParamControllers; }; // Utility class to allow easily scanning a collection. Scans in forward order, returns diff --git a/src/mongo/db/repl/oplog_applier_utils.cpp b/src/mongo/db/repl/oplog_applier_utils.cpp index 5029e0b57d6..5b1061e3998 100644 --- a/src/mongo/db/repl/oplog_applier_utils.cpp +++ b/src/mongo/db/repl/oplog_applier_utils.cpp @@ -256,7 +256,15 @@ Status OplogApplierUtils::applyOplogEntryOrGroupedInsertsCommon( !oplogApplicationEnforcesSteadyStateConstraints && oplogApplicationMode == OplogApplication::Mode::kSecondary) { if (opCounters) { + const auto& opObj = redact(op.toBSONForLogging()); opCounters->gotDeleteFromMissingNamespace(); + logOplogConstraintViolation( + opCtx, + op.getNss(), + OplogConstraintViolationEnum::kDeleteOnMissingNs, + "delete", + opObj, + boost::none /* status */); } return Status::OK(); } diff --git a/src/mongo/db/repl/oplog_constraint_violation_logger.cpp b/src/mongo/db/repl/oplog_constraint_violation_logger.cpp new file mode 100644 index 00000000000..f887393dbdf --- /dev/null +++ b/src/mongo/db/repl/oplog_constraint_violation_logger.cpp @@ -0,0 +1,90 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kReplication + +#include "mongo/db/repl/oplog_constraint_violation_logger.h" +#include "mongo/logv2/log.h" + +namespace mongo { +namespace repl { + +// Default interval set to 10 minutes. +const Seconds OplogConstraintViolationLogger::kPeriodicLogTimeout(60 * 10); + +StringData toString(OplogConstraintViolationEnum type) { + switch (type) { + case OplogConstraintViolationEnum::kInsertOnExistingDoc: + return kInsertOnExistingDocMsg; + case OplogConstraintViolationEnum::kUpdateOnMissingDoc: + return kUpdateOnMissingDocMsg; + case OplogConstraintViolationEnum::kDeleteWasEmpty: + return kDeleteWasEmptyMsg; + case OplogConstraintViolationEnum::kDeleteOnMissingNs: + return kDeleteOnMissingNs; + case OplogConstraintViolationEnum::kAcceptableErrorInCommand: + return kAcceptableErrorInCommand; + default: + return ""; + } +} + +void OplogConstraintViolationLogger::logViolationIfReady(OplogConstraintViolationEnum type, + const BSONObj& obj, + boost::optional<Status> status) { + const auto index = static_cast<int>(type); + + stdx::lock_guard lk(_mutex); + const auto lastLog = _lastLogTimes[index]; + const auto now = Date_t::now(); + + if (now < lastLog + OplogConstraintViolationLogger::kPeriodicLogTimeout) { + // We have logged this violation already within the last 10 minutes. + return; + } + + if (!status) { + LOGV2_WARNING(7149000, + "Potential replication constraint violation during steady state replication", + "msg"_attr = toString(type), + "obj"_attr = obj); + } else { + LOGV2_WARNING(7149001, + "Potential replication constraint violation during steady state replication", + "msg"_attr = toString(type), + "obj"_attr = obj, + "status"_attr = *status); + } + + // Update the last log time to now. + _lastLogTimes[index] = now; +} + +} // namespace repl +} // namespace mongo diff --git a/src/mongo/db/repl/oplog_constraint_violation_logger.h b/src/mongo/db/repl/oplog_constraint_violation_logger.h new file mode 100644 index 00000000000..321544e28d9 --- /dev/null +++ b/src/mongo/db/repl/oplog_constraint_violation_logger.h @@ -0,0 +1,85 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <boost/optional.hpp> + +#include "mongo/base/status.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/platform/mutex.h" +#include "mongo/util/time_support.h" + +namespace mongo { +namespace repl { + +static constexpr StringData kInsertOnExistingDocMsg = "attempted to insert on existing doc"_sd; +static constexpr StringData kUpdateOnMissingDocMsg = + "ran update as upsert and failed to match any documents"_sd; +static constexpr StringData kDeleteWasEmptyMsg = "applied a delete that did not delete anything"_sd; +static constexpr StringData kDeleteOnMissingNs = "applied a delete on missing namespace"_sd; +static constexpr StringData kAcceptableErrorInCommand = + "received an acceptable error during oplog application"_sd; + +enum class OplogConstraintViolationEnum { + kInsertOnExistingDoc = 0, + kUpdateOnMissingDoc, + kDeleteWasEmpty, + kDeleteOnMissingNs, + kAcceptableErrorInCommand, + NUM_VIOLATION_TYPES, +}; + +// Returns a string describing the constraint violation of the given type. +StringData toString(OplogConstraintViolationEnum type); + +/** + * Logs oplog constraint violation occurrences. + * + * To avoid flooding the logs if continuous oplog constraint violations occur, we will only log + * once every 10 minutes per each oplog constraint violation type. + */ +class OplogConstraintViolationLogger { +public: + // Minimum period of time before logging another warning log message, set to 10min. + static const Seconds kPeriodicLogTimeout; + + void logViolationIfReady(OplogConstraintViolationEnum type, + const BSONObj& obj, + boost::optional<Status> status); + +private: + mutable Mutex _mutex = MONGO_MAKE_LATCH("OplogConstraintViolationLogger::mutex"); + + std::vector<Date_t> _lastLogTimes = std::vector<Date_t>( + static_cast<int>(OplogConstraintViolationEnum::NUM_VIOLATION_TYPES)); // (M) +}; + +} // namespace repl +} // namespace mongo diff --git a/src/mongo/db/repl/oplog_entry.cpp b/src/mongo/db/repl/oplog_entry.cpp index 7f7363be06e..77fba82039f 100644 --- a/src/mongo/db/repl/oplog_entry.cpp +++ b/src/mongo/db/repl/oplog_entry.cpp @@ -764,5 +764,32 @@ int OplogEntry::getRawObjSizeBytes() const { return _entry.getRawObjSizeBytes(); } +OplogEntryParserNonStrict::OplogEntryParserNonStrict(const BSONObj& oplogEntry) + : _oplogEntryObject{oplogEntry.getOwned()} {} + +repl::OpTime OplogEntryParserNonStrict::getOpTime() const { + return uassertStatusOKWithContext(repl::OpTime::parseFromOplogEntry(_oplogEntryObject), + str::stream() << "Failed to parse opTime"); +} + +repl::OpTypeEnum OplogEntryParserNonStrict::getOpType() const { + auto opTypeElement = _oplogEntryObject[repl::OplogEntry::kOpTypeFieldName]; + uassert(8881100, + str::stream() << "Invalid '" << repl::OplogEntry::kOpTypeFieldName + << "' field type (expected String)", + opTypeElement.type() == BSONType::String); + return repl::OpType_parse(IDLParserErrorContext("ChangeStreamEntry.op"), + opTypeElement.checkAndGetStringData()); +} + +BSONObj OplogEntryParserNonStrict::getObject() const { + auto objectElement = _oplogEntryObject[repl::OplogEntry::kObjectFieldName]; + uassert(8881101, + str::stream() << "Invalid '" << repl::OplogEntry::kObjectFieldName + << "' field type (expected Object)", + objectElement.isABSONObj()); + return objectElement.Obj(); +} + } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/oplog_entry.h b/src/mongo/db/repl/oplog_entry.h index 7552faa3f9e..f41898f8c6b 100644 --- a/src/mongo/db/repl/oplog_entry.h +++ b/src/mongo/db/repl/oplog_entry.h @@ -801,6 +801,40 @@ private: bool _isForCappedCollection = false; }; +/** + * Oplog entry document parser. This parser can parse only the key fields. It parses fields on + * demand. This parser should be used only in cases when to be parsed oplog entry data structure + * version may not match the one that is used by the current server version (this can happen with + * past or future versions of oplog entries), otherwise 'OplogEntry::parse()' is supposed to be + * used. + */ +class OplogEntryParserNonStrict { +public: + /** + * Constructs the parser with to be parsed oplog entry document 'oplogEntry'. + */ + OplogEntryParserNonStrict(const BSONObj& oplogEntry); + + /** + * Parses and returns "opTime" field. + */ + repl::OpTime getOpTime() const; + + /** + * Parses and returns the type of operation field. + */ + repl::OpTypeEnum getOpType() const; + + /** + * Parses and returns the "operation applied" field. + */ + BSONObj getObject() const; + +private: + // Oplog entry as BSON object to be parsed. + const BSONObj _oplogEntryObject; +}; + std::ostream& operator<<(std::ostream& s, const DurableOplogEntry& o); std::ostream& operator<<(std::ostream& s, const OplogEntry& o); diff --git a/src/mongo/db/repl/oplog_entry_test.cpp b/src/mongo/db/repl/oplog_entry_test.cpp index c034bd3c517..6cfd6958519 100644 --- a/src/mongo/db/repl/oplog_entry_test.cpp +++ b/src/mongo/db/repl/oplog_entry_test.cpp @@ -135,7 +135,75 @@ TEST(OplogEntryTest, OpTimeBaseNonStrictParsing) { 40414); } +TEST(OplogEntryParserTest, ParseOpTimeSuccess) { + repl::OpTime opTime{Timestamp{2}, 1}; + auto const oplogEntry = opTime.toBSON(); + OplogEntryParserNonStrict parser{oplogEntry}; + ASSERT_EQ(opTime, parser.getOpTime()) << oplogEntry.toString(); +} + +TEST(OplogEntryParserTest, ParseOpTimeFailure) { + auto const oplogEntry = BSON("a" << 1); + OplogEntryParserNonStrict parser{oplogEntry}; + ASSERT_THROWS_CODE_AND_WHAT(parser.getOpTime(), + AssertionException, + 40414, + "Failed to parse opTime :: caused by :: " + "BSON field 'OpTimeBase.ts' is missing but a required field"); +} + +TEST(OplogEntryParserTest, ParseOpTypeSuccess) { + auto const oplogEntry = + BSON(OplogEntry::kOpTypeFieldName << OpType_serializer(repl::OpTypeEnum::kDelete)); + OplogEntryParserNonStrict parser{oplogEntry}; + ASSERT_TRUE(repl::OpTypeEnum::kDelete == parser.getOpType()) << oplogEntry.toString(); +} +TEST(OplogEntryParserTest, ParseOpTypeFailure) { + { + auto const oplogEntry = BSON(OplogEntry::kOpTypeFieldName << "zz"); + OplogEntryParserNonStrict parser{oplogEntry}; + ASSERT_THROWS_CODE_AND_WHAT( + parser.getOpType(), + AssertionException, + ErrorCodes::BadValue, + "Enumeration value 'zz' for field 'ChangeStreamEntry.op' is not a valid value."); + } + { + auto const oplogEntry = BSON(OplogEntry::kOpTypeFieldName << 1); + OplogEntryParserNonStrict parser{oplogEntry}; + ASSERT_THROWS_CODE_AND_WHAT(parser.getOpType(), + AssertionException, + 8881100, + "Invalid 'op' field type (expected String)"); + } +} + +TEST(OplogEntryParserTest, ParseObjectSuccess) { + auto const objectFieldValue = BSON("a" << 1); + auto const oplogEntry = BSON(OplogEntry::kObjectFieldName << objectFieldValue); + OplogEntryParserNonStrict parser{oplogEntry}; + ASSERT_BSONOBJ_BINARY_EQ(objectFieldValue, parser.getObject()); +} + +TEST(OplogEntryParserTest, ParseObjectFailure) { + { + auto const oplogEntry = BSON(OplogEntry::kObjectFieldName << "string"); + OplogEntryParserNonStrict parser{oplogEntry}; + ASSERT_THROWS_CODE_AND_WHAT(parser.getObject(), + AssertionException, + 8881101, + "Invalid 'o' field type (expected Object)"); + } + { + auto const oplogEntry = BSON("a" << 1); + OplogEntryParserNonStrict parser{oplogEntry}; + ASSERT_THROWS_CODE_AND_WHAT(parser.getObject(), + AssertionException, + 8881101, + "Invalid 'o' field type (expected Object)"); + } +} } // namespace } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/repl_server_parameters.idl b/src/mongo/db/repl/repl_server_parameters.idl index 3f4f5aac440..e178b4f4154 100644 --- a/src/mongo/db/repl/repl_server_parameters.idl +++ b/src/mongo/db/repl/repl_server_parameters.idl @@ -665,6 +665,15 @@ server_parameters: default: "" validator: { callback: 'validateHostAndPort' } + skipApplyingDbCheckBatchOnSecondary: + description: >- + Parameter for whether dbcheck batches should be applied on secondaries. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<bool> + cpp_varname: skipApplyingDbCheckBatchOnSecondary + default: false + + feature_flags: featureFlagRetryableFindAndModify: description: >- diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 8074192cb6a..fb0c8fa81fc 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -111,7 +111,6 @@ #include "mongo/util/assert_util.h" #include "mongo/util/fail_point.h" #include "mongo/util/scopeguard.h" -#include "mongo/util/stacktrace.h" #include "mongo/util/testing_proctor.h" #include "mongo/util/time_support.h" #include "mongo/util/timer.h" @@ -184,6 +183,13 @@ ServerStatusMetricField<Counter64> displayNumAutoReconfigs( "repl.reconfig.numAutoReconfigsForRemovalOfNewlyAddedFields", &numAutoReconfigsForRemovalOfNewlyAddedFields); +Atomic64Metric replicationWaiterListMetric; +ServerStatusMetricField<Atomic64Metric> displayReplicationWaiterListMetric( + "repl.waiters.replication", &replicationWaiterListMetric); +Atomic64Metric opTimeWaiterListMetric; +ServerStatusMetricField<Atomic64Metric> displayOpTimeWaiterListMetric("repl.waiters.opTime", + &opTimeWaiterListMetric); + using namespace fmt::literals; using CallbackArgs = executor::TaskExecutor::CallbackArgs; @@ -221,15 +227,24 @@ constexpr StringData kQuiesceModeShutdownMessage = } // namespace +ReplicationCoordinatorImpl::WaiterList::WaiterList(Atomic64Metric& waiterCountMetric) + : _waiterCountMetric(waiterCountMetric) {} + +void ReplicationCoordinatorImpl::WaiterList::_updateMetric_inlock() { + _waiterCountMetric.set(_waiters.size()); +} + void ReplicationCoordinatorImpl::WaiterList::add_inlock(const OpTime& opTime, SharedWaiterHandle waiter) { _waiters.emplace(opTime, std::move(waiter)); + _updateMetric_inlock(); } SharedSemiFuture<void> ReplicationCoordinatorImpl::WaiterList::add_inlock( const OpTime& opTime, boost::optional<WriteConcernOptions> wc) { auto pf = makePromiseFuture<void>(); _waiters.emplace(opTime, std::make_shared<Waiter>(std::move(pf.promise), std::move(wc))); + _updateMetric_inlock(); return std::move(pf.future); } @@ -237,6 +252,7 @@ bool ReplicationCoordinatorImpl::WaiterList::remove_inlock(SharedWaiterHandle wa for (auto iter = _waiters.begin(); iter != _waiters.end(); iter++) { if (iter->second == waiter) { _waiters.erase(iter); + _updateMetric_inlock(); return true; } } @@ -260,6 +276,7 @@ void ReplicationCoordinatorImpl::WaiterList::setValueIf_inlock(Func&& func, it = _waiters.erase(it); } } + _updateMetric_inlock(); } void ReplicationCoordinatorImpl::WaiterList::setValueAll_inlock() { @@ -267,6 +284,7 @@ void ReplicationCoordinatorImpl::WaiterList::setValueAll_inlock() { waiter->promise.emplaceValue(); } _waiters.clear(); + _updateMetric_inlock(); } void ReplicationCoordinatorImpl::WaiterList::setErrorAll_inlock(Status status) { @@ -275,6 +293,7 @@ void ReplicationCoordinatorImpl::WaiterList::setErrorAll_inlock(Status status) { waiter->promise.setError(status); } _waiters.clear(); + _updateMetric_inlock(); } namespace { @@ -330,6 +349,8 @@ ReplicationCoordinatorImpl::ReplicationCoordinatorImpl( _topCoord(std::move(topCoord)), _replExecutor(std::move(executor)), _externalState(std::move(externalState)), + _replicationWaiterList(replicationWaiterListMetric), + _opTimeWaiterList(opTimeWaiterListMetric), _inShutdown(false), _memberState(MemberState::RS_STARTUP), _rsConfigState(kConfigPreStart), @@ -423,6 +444,10 @@ executor::TaskExecutor::CallbackHandle ReplicationCoordinatorImpl::getCatchupTak return _catchupTakeoverCbh; } +int64_t ReplicationCoordinatorImpl::getLastHorizonChange_forTest() const { + return _lastHorizonTopologyChange; +} + OpTime ReplicationCoordinatorImpl::getCurrentCommittedSnapshotOpTime() const { stdx::lock_guard<Latch> lk(_mutex); return _getCurrentCommittedSnapshotOpTime_inlock(); @@ -2286,7 +2311,9 @@ long long ReplicationCoordinatorImpl::_calculateRemainingQuiesceTimeMillis() con } std::shared_ptr<HelloResponse> ReplicationCoordinatorImpl::_makeHelloResponse( - boost::optional<StringData> horizonString, WithLock lock, const bool hasValidConfig) const { + const boost::optional<std::string>& horizonString, + WithLock lock, + const bool hasValidConfig) const { uassert(ShutdownInProgressQuiesceInfo(_calculateRemainingQuiesceTimeMillis()), kQuiesceModeShutdownMessage, @@ -2334,7 +2361,7 @@ SharedSemiFuture<ReplicationCoordinatorImpl::SharedHelloResponse> ReplicationCoordinatorImpl::_getHelloResponseFuture( WithLock lk, const SplitHorizon::Parameters& horizonParams, - boost::optional<StringData> horizonString, + const boost::optional<std::string>& horizonString, boost::optional<TopologyVersion> clientTopologyVersion) { uassert(ShutdownInProgressQuiesceInfo(_calculateRemainingQuiesceTimeMillis()), @@ -2366,6 +2393,10 @@ ReplicationCoordinatorImpl::_getHelloResponseFuture( prevCounter <= topologyVersionCounter); if (prevCounter < topologyVersionCounter) { + uassert(ErrorCodes::SplitHorizonChange, + "Stale horizon detected, we have since received a reconfig that changed the " + "horizon mappings.", + prevCounter >= _lastHorizonTopologyChange); // The received hello command contains a stale topology version so we respond // immediately with a more current topology version. return SharedSemiFuture<SharedHelloResponse>( @@ -2398,11 +2429,11 @@ ReplicationCoordinatorImpl::getHelloResponseFuture( return _getHelloResponseFuture(lk, horizonParams, horizonString, clientTopologyVersion); } -boost::optional<StringData> ReplicationCoordinatorImpl::_getHorizonString( +boost::optional<std::string> ReplicationCoordinatorImpl::_getHorizonString( WithLock, const SplitHorizon::Parameters& horizonParams) const { const auto myState = _topCoord->getMemberState(); const bool hasValidConfig = _rsConfig.isInitialized() && !myState.removed(); - boost::optional<StringData> horizonString; + boost::optional<std::string> horizonString; if (hasValidConfig) { const auto& self = _rsConfig.getMemberAt(_selfIndex); horizonString = self.determineHorizon(horizonParams); @@ -2643,19 +2674,10 @@ ReplicationCoordinatorImpl::AutoGetRstlForStepUpStepDown::AutoGetRstlForStepUpSt auto lockerInfo = opCtx->lockState()->getLockerInfo(CurOp::get(opCtx)->getLockStatsBase()); BSONObjBuilder lockRep; lockerInfo->stats.report(&lockRep); - - LOGV2_FATAL_CONTINUE( - 5675600, - "Time out exceeded waiting for RSTL, stepUp/stepDown is not possible thus " - "calling abort() to allow cluster to progress", - "lockRep"_attr = lockRep.obj()); - -#if defined(MONGO_STACKTRACE_CAN_DUMP_ALL_THREADS) - // Dump the stack of each thread. - printAllThreadStacksBlocking(); -#endif - - fassertFailed(7152000); + LOGV2_FATAL(5675600, + "Time out exceeded waiting for RSTL, stepUp/stepDown is not possible thus " + "calling abort() to allow cluster to progress", + "lockRep"_attr = lockRep.obj()); }); }; @@ -4442,6 +4464,8 @@ void ReplicationCoordinatorImpl::_errorOnPromisesIfHorizonChanged(WithLock lk, promise->setError({ErrorCodes::SplitHorizonChange, "Received a reconfig that changed the horizon mappings."}); } + _topCoord->incrementTopologyVersion(); + _lastHorizonTopologyChange = _topCoord->getTopologyVersion().getCounter(); _sniToValidConfigPromiseMap.clear(); HelloMetrics::get(opCtx)->resetNumAwaitingTopologyChanges(); } @@ -4456,6 +4480,10 @@ void ReplicationCoordinatorImpl::_errorOnPromisesIfHorizonChanged(WithLock lk, promise->setError({ErrorCodes::SplitHorizonChange, "Received a reconfig that changed the horizon mappings."}); } + // Increment topology version to mark a horizon change, since a reconfig doesn't + // increment the topology version until the end. + _topCoord->incrementTopologyVersion(); + _lastHorizonTopologyChange = _topCoord->getTopologyVersion().getCounter(); _createHorizonTopologyChangePromiseMapping(lk); HelloMetrics::get(opCtx)->resetNumAwaitingTopologyChanges(); } @@ -4476,7 +4504,7 @@ void ReplicationCoordinatorImpl::_fulfillTopologyChangePromise(WithLock lock) { Status(ShutdownInProgressQuiesceInfo(_calculateRemainingQuiesceTimeMillis()), kQuiesceModeShutdownMessage)); } else { - StringData horizonString = iter->first; + boost::optional<std::string> horizonString = iter->first; auto response = _makeHelloResponse(horizonString, lock, hasValidConfig); // Fulfill the promise and replace with a new one for future waiters. iter->second->emplaceValue(response); @@ -4496,7 +4524,8 @@ void ReplicationCoordinatorImpl::_fulfillTopologyChangePromise(WithLock lock) { "The original request horizon parameter does not exist in the " "current replica set config"}); } else { - const auto horizon = sni.empty() ? SplitHorizon::kDefaultHorizon : iter->second; + const boost::optional<std::string> horizon = + sni.empty() ? SplitHorizon::kDefaultHorizon.toString() : iter->second; const auto response = _makeHelloResponse(horizon, lock, hasValidConfig); promise->emplaceValue(response); } diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h index 8c207c70958..4da218cec58 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.h +++ b/src/mongo/db/repl/replication_coordinator_impl.h @@ -468,6 +468,11 @@ public: executor::TaskExecutor::CallbackHandle getCatchupTakeoverCbh_forTest() const; /** + * Returns the cached horizon topology version from most recent SplitHorizonChange. + */ + int64_t getLastHorizonChange_forTest() const; + + /** * Simple wrappers around _setLastOptime to make it easier to test. */ Status setLastAppliedOptime_forTest(long long cfgVer, @@ -764,6 +769,9 @@ private: class WaiterList { public: + WaiterList() = delete; + WaiterList(Atomic64Metric& waiterCountMetric); + // Adds waiter into the list. void add_inlock(const OpTime& opTime, SharedWaiterHandle waiter); // Adds a waiter into the list and returns the future of the waiter's promise. @@ -781,8 +789,13 @@ private: void setErrorAll_inlock(Status status); private: + void _updateMetric_inlock(); + // Waiters sorted by OpTime. std::multimap<OpTime, SharedWaiterHandle> _waiters; + // We keep a separate count outside _waiters.size() in order to avoid having to + // take a lock to read the metric. + Atomic64Metric& _waiterCountMetric; }; enum class HeartbeatState { kScheduled = 0, kSent = 1 }; @@ -1380,9 +1393,8 @@ private: * Fills a HelloResponse with the appropriate replication related fields. horizonString * should be passed in if hasValidConfig is true. */ - std::shared_ptr<HelloResponse> _makeHelloResponse(boost::optional<StringData> horizonString, - WithLock, - bool hasValidConfig) const; + std::shared_ptr<HelloResponse> _makeHelloResponse( + const boost::optional<std::string>& horizonString, WithLock, bool hasValidConfig) const; /** * Creates a semi-future for HelloResponse. horizonString should be passed in if and only if @@ -1391,14 +1403,14 @@ private: virtual SharedSemiFuture<SharedHelloResponse> _getHelloResponseFuture( WithLock, const SplitHorizon::Parameters& horizonParams, - boost::optional<StringData> horizonString, + const boost::optional<std::string>& horizonString, boost::optional<TopologyVersion> clientTopologyVersion); /** * Returns the horizon string by parsing horizonParams if the node is a valid member of the * replica set. Otherwise, return boost::none. */ - boost::optional<StringData> _getHorizonString( + boost::optional<std::string> _getHorizonString( WithLock, const SplitHorizon::Parameters& horizonParams) const; /** @@ -1836,6 +1848,9 @@ private: // The cached value of the 'counter' field in the server's TopologyVersion. AtomicWord<int64_t> _cachedTopologyVersionCounter; // (S) + // The cached value of the topology from the most recent SplitHorizonChange. + int64_t _lastHorizonTopologyChange{-1}; // (M) + // This should be set during sharding initialization. boost::optional<bool> _wasCWWCSetOnConfigServerOnStartup; diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index 98a6f722a73..ce5c77a1e7d 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -77,6 +77,7 @@ MONGO_FAIL_POINT_DEFINE(blockHeartbeatStepdown); MONGO_FAIL_POINT_DEFINE(blockHeartbeatReconfigFinish); MONGO_FAIL_POINT_DEFINE(hangAfterTrackingNewHandleInHandleHeartbeatResponseForTest); MONGO_FAIL_POINT_DEFINE(waitForPostActionCompleteInHbReconfig); +MONGO_FAIL_POINT_DEFINE(pauseInHandleHeartbeatResponse); } // namespace @@ -184,6 +185,12 @@ void ReplicationCoordinatorImpl::handleHeartbeatResponse_forTest(BSONObj respons void ReplicationCoordinatorImpl::_handleHeartbeatResponse( const executor::TaskExecutor::RemoteCommandCallbackArgs& cbData, const std::string& setName) { + pauseInHandleHeartbeatResponse.executeIf( + [](const BSONObj& data) { pauseInHandleHeartbeatResponse.pauseWhileSet(); }, + [&cbData](const BSONObj& data) -> bool { + StringData dtarget = data["target"].valueStringDataSafe(); + return dtarget == cbData.request.target.toString(); + }); stdx::unique_lock<Latch> lk(_mutex); // remove handle from queued heartbeats @@ -194,7 +201,15 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( Status responseStatus = cbData.response.status; const HostAndPort& target = cbData.request.target; - if (responseStatus == ErrorCodes::CallbackCanceled) { + // It is possible that the callback was canceled after handleHeartbeatResponse was called but + // before it got the lock above. + // + // In this case, the responseStatus will be OK and we can process the heartbeat. However, if + // we do so, cancelling heartbeats no longer establishes a barrier after which all heartbeats + // processed are "new" (sent subsequent to the cancel), which is something we care about for + // catchup takeover. So if we detect this situation (by checking if the handle was canceled) + // we will NOT process the 'stale' heartbeat. + if (responseStatus == ErrorCodes::CallbackCanceled || cbData.myHandle.isCanceled()) { LOGV2_FOR_HEARTBEATS(4615619, 2, "Received response to heartbeat (requestId: {requestId}) from " diff --git a/src/mongo/db/repl/replication_coordinator_impl_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_test.cpp index 9f0f0531710..f0ece048ca3 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_test.cpp @@ -85,6 +85,9 @@ namespace mongo { namespace repl { +extern Atomic64Metric replicationWaiterListMetric; +extern Atomic64Metric opTimeWaiterListMetric; + namespace { using executor::NetworkInterfaceMock; @@ -4074,6 +4077,131 @@ TEST_F(ReplCoordTest, AwaitHelloResponseReturnsErrorOnHorizonChange) { getHelloThread.join(); } +TEST_F(ReplCoordTest, ServerUassertAfterStaleHorizonTopology) { + init(); + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0) + << BSON("host" + << "node2:12345" + << "_id" << 1))), + HostAndPort("node1", 12345)); + + // Become primary. + ASSERT_OK(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + replCoordSetMyLastAppliedAndDurableOpTime(OpTimeWithTermOne(100, 1), Date_t() + Seconds(100)); + simulateSuccessfulV1Election(); + ASSERT(getReplCoord()->getMemberState().primary()); + + auto maxAwaitTime = Milliseconds(5000); + auto deadline = getNet()->now() + maxAwaitTime; + auto opCtx = makeOperationContext(); + + auto topologyVersionBeforeReconfig = getTopoCoord().getTopologyVersion(); + // awaitHelloResponse blocks and waits on a future when the request TopologyVersion equals + // the current TopologyVersion of the server. + stdx::thread getHelloThread([&] { + ASSERT_THROWS_CODE( + awaitHelloWithNewOpCtx(getReplCoord(), topologyVersionBeforeReconfig, {}, deadline), + AssertionException, + ErrorCodes::SplitHorizonChange); + }); + + auto lastHorizonBeforeReconfig = getReplCoord()->getLastHorizonChange_forTest(); + ASSERT_EQUALS(lastHorizonBeforeReconfig, -1); + + BSONObjBuilder garbage; + ReplSetReconfigArgs args; + // Use force to bypass the oplog commitment check, which we're not worried about testing here. + args.force = true; + // Do a reconfig that changes the SplitHorizon and also adds a third node. This should respond + // to all waiting hello requests with an error. + args.newConfigObj = BSON("_id" + << "mySet" + << "version" << 3 << "protocolVersion" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 0 << "host" + << "node1:12345" + << "priority" << 3 << "horizons" + << BSON("testhorizon" + << "test.monkey.example.com:24")) + << BSON("_id" << 1 << "host" + << "node2:12345" + << "horizons" + << BSON("testhorizon" + << "test.giraffe.example.com:25")) + << BSON("_id" + << 2 << "host" + << "node3:12345" + << "horizons" + << BSON("testhorizon" + << "test.elephant.example.com:26")))); + stdx::thread reconfigThread([&] { + Status status(ErrorCodes::InternalError, "Not Set"); + status = getReplCoord()->processReplSetReconfig(opCtx.get(), args, &garbage); + ASSERT_OK(status); + }); + replyToReceivedHeartbeatV1(); + reconfigThread.join(); + getHelloThread.join(); + + // After reconfig, the last horizon change topology counter should come out greater than the + // input topology counter , but less than the output topology counter. + ASSERT_GREATER_THAN(getReplCoord()->getLastHorizonChange_forTest(), + topologyVersionBeforeReconfig.getCounter()); + ASSERT_GREATER_THAN(getTopoCoord().getTopologyVersion().getCounter(), + getReplCoord()->getLastHorizonChange_forTest()); + ASSERT_GREATER_THAN(getReplCoord()->getLastHorizonChange_forTest(), lastHorizonBeforeReconfig); + + // Send hello with a TopologyVersion older than the TopologyVersion of the last horizon change. + auto requestTopologyVersion = + TopologyVersion(getTopoCoord().getTopologyVersion().getProcessId(), + getReplCoord()->getLastHorizonChange_forTest() - 1); + + // AwaitHelloResponse should throw uassert with SplitHorizonChange if topology version + // corresponds to a stale horizon. + ASSERT_THROWS_CODE(awaitHelloWithNewOpCtx(getReplCoord(), requestTopologyVersion, {}, deadline), + DBException, + ErrorCodes::SplitHorizonChange); + + // Send hello with a TopologyVersion version equal to the TopologyVersion of the last horizon + // change. + auto expectedTopologyVersion = getTopoCoord().getTopologyVersion(); + requestTopologyVersion = TopologyVersion(expectedTopologyVersion.getProcessId(), + getReplCoord()->getLastHorizonChange_forTest()); + + ASSERT_GREATER_THAN(expectedTopologyVersion.getCounter(), requestTopologyVersion.getCounter()); + // AwaitHelloResponse should return with a helloResponse that matches expectedTopologyVersion. + // Since expectedTopologyVersion > requestTopologyVersion, the call is non-blocking and will + // return immediately. + auto response = awaitHelloWithNewOpCtx(getReplCoord(), requestTopologyVersion, {}, deadline); + auto responseTopologyVersion = response->getTopologyVersion(); + ASSERT_EQUALS(responseTopologyVersion->getCounter(), expectedTopologyVersion.getCounter()); + ASSERT_EQUALS(responseTopologyVersion->getProcessId(), expectedTopologyVersion.getProcessId()); + + // Setup instance where lastHorizonChange topology counter < request topology counter < server + // topology counter. For server topology counter to be greater than both request's and horizon, + // we must increment server topology. + getTopoCoord().incrementTopologyVersion(); + expectedTopologyVersion = getTopoCoord().getTopologyVersion(); + // Send hello with a TopologyVersion version greater than the TopologyVersion of the last + // horizon change. + requestTopologyVersion = TopologyVersion(getTopoCoord().getTopologyVersion().getProcessId(), + getReplCoord()->getLastHorizonChange_forTest() + 1); + + ASSERT_GREATER_THAN(expectedTopologyVersion.getCounter(), requestTopologyVersion.getCounter()); + ASSERT_GREATER_THAN(requestTopologyVersion.getCounter(), + getReplCoord()->getLastHorizonChange_forTest()); + + // AwaitHelloResponse should return with a helloResponse that matches expectedTopologyVersion. + response = awaitHelloWithNewOpCtx(getReplCoord(), requestTopologyVersion, {}, deadline); + responseTopologyVersion = response->getTopologyVersion(); + ASSERT_EQUALS(responseTopologyVersion->getCounter(), expectedTopologyVersion.getCounter()); + ASSERT_EQUALS(responseTopologyVersion->getProcessId(), expectedTopologyVersion.getProcessId()); +} + TEST_F(ReplCoordTest, NonAwaitableHelloReturnsNoConfigsOnNodeWithUninitializedConfig) { start(); auto opCtx = makeOperationContext(); @@ -4719,6 +4847,9 @@ TEST_F(ReplCoordTest, AwaitHelloRespondsCorrectlyWhenNodeRemovedAndReadded) { }); waitForHelloFailPoint->waitForTimesEntered(timesEnteredFailPoint + 2); + auto lastHorizonBeforeReconfig = getReplCoord()->getLastHorizonChange_forTest(); + ASSERT_EQUALS(lastHorizonBeforeReconfig, -1); + const auto newHorizonNodeOne = "newhorizon.com:100"; const auto newHorizonNodeTwo = "newhorizon.com:200"; @@ -4743,10 +4874,22 @@ TEST_F(ReplCoordTest, AwaitHelloRespondsCorrectlyWhenNodeRemovedAndReadded) { }); replyToReceivedHeartbeatV1(); reconfigThread.join(); + ASSERT_OK( getReplCoord()->waitForMemberState(opCtx.get(), MemberState::RS_SECONDARY, Seconds(1))); getHelloThread.join(); + ASSERT_GREATER_THAN(getReplCoord()->getLastHorizonChange_forTest(), lastHorizonBeforeReconfig); + // Send hello with a TopologyVersion older than the TopologyVersion of the last horizon change. + auto requestTopologyVersion = + TopologyVersion(getTopoCoord().getTopologyVersion().getProcessId(), + getReplCoord()->getLastHorizonChange_forTest() - 1); + // AwaitHelloResponse should throw uassert with SplitHorizonChange if topology version + // corresponds to a stale horizon. + ASSERT_THROWS_CODE(awaitHelloWithNewOpCtx(getReplCoord(), requestTopologyVersion, {}, deadline), + DBException, + ErrorCodes::SplitHorizonChange); + stdx::thread getHelloThreadNewHorizon([&] { const auto expectedTopologyVersion = getTopoCoord().getTopologyVersion(); // Sending a hello on the rejoined node should return the appropriate horizon view. @@ -5758,6 +5901,140 @@ TEST_F(ReplCoordTest, awaiter.reset(); } + +// We need to wait for replication to start waiting before the waiter metric increases. We +// return the metric value from the function for the convenience of the assert macro below, +// which allows us to get a nice assert message without repeating the value. +template <typename T, typename U> +U expectMetricIncreaseTo(T& metric, U value) { + // If this doesn't go in 10 seconds, something's seriously wrong; even if just a slow machine, + // the test will likely fail anyway. + constexpr auto timeout = Seconds(10); + const auto deadline = Date_t::now() + timeout; + U lastValue = metric.get(); + U curValue = metric.get(); + while (curValue < value && Date_t::now() <= deadline) { + sleepFor(Milliseconds{10}); + curValue = metric.get(); + lastValue = curValue; + } + return curValue; +} + +#define ASSERT_METRIC_INCREASE_TO(metric, value) \ + ASSERT_EQ(expectMetricIncreaseTo(metric, value), value) + +TEST_F(ReplCoordTest, ReplicationWaiterMetrics) { + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0) + << BSON("host" + << "node2:12345" + << "_id" << 1) + << BSON("host" + << "node3:12345" + << "_id" << 2))), + HostAndPort("node1", 12345)); + ASSERT_OK(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + replCoordSetMyLastAppliedOpTime(OpTimeWithTermOne(100, 1), Date_t() + Seconds(100)); + replCoordSetMyLastDurableOpTime(OpTimeWithTermOne(100, 1), Date_t() + Seconds(100)); + simulateSuccessfulV1Election(); + + ReplicationAwaiter awaiter1(getReplCoord(), getServiceContext()); + ReplicationAwaiter awaiter2(getReplCoord(), getServiceContext()); + + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); + OpTimeWithTermOne time3(100, 3); + + WriteConcernOptions writeConcern; + writeConcern.wTimeout = WriteConcernOptions::kNoTimeout; + writeConcern.w = 2; + + WriteConcernOptions writeConcernLocal; + writeConcernLocal.wTimeout = WriteConcernOptions::kNoTimeout; + writeConcernLocal.w = 1; + writeConcernLocal.syncMode = WriteConcernOptions::SyncMode::UNSET; + + // 2 waiters waiting for 2 nodes to reach time1. + awaiter1.setOpTime(time1); + awaiter1.setWriteConcern(writeConcern); + awaiter1.start(); + ASSERT_METRIC_INCREASE_TO(replicationWaiterListMetric, 1); + ASSERT_EQ(opTimeWaiterListMetric.get(), 0); + + awaiter2.setOpTime(time1); + awaiter2.setWriteConcern(writeConcern); + awaiter2.start(); + ASSERT_METRIC_INCREASE_TO(replicationWaiterListMetric, 2); + ASSERT_EQ(opTimeWaiterListMetric.get(), 0); + + replCoordSetMyLastAppliedOpTime(time1, Date_t() + Seconds(100)); + replCoordSetMyLastDurableOpTime(time1, Date_t() + Seconds(100)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(2, 1, time1)); + ReplicationCoordinator::StatusAndDuration statusAndDur = awaiter1.getResult(); + ASSERT_OK(statusAndDur.status); + statusAndDur = awaiter2.getResult(); + ASSERT_OK(statusAndDur.status); + ASSERT_EQ(replicationWaiterListMetric.get(), 0); + ASSERT_EQ(opTimeWaiterListMetric.get(), 0); + awaiter1.reset(); + awaiter2.reset(); + + // 2 nodes waiting for time2, but only locally. + awaiter1.setOpTime(time2); + awaiter1.setWriteConcern(writeConcernLocal); + awaiter1.start(); + ASSERT_METRIC_INCREASE_TO(opTimeWaiterListMetric, 1); + ASSERT_EQ(replicationWaiterListMetric.get(), 0); + + awaiter2.setOpTime(time2); + awaiter2.setWriteConcern(writeConcernLocal); + awaiter2.start(); + ASSERT_METRIC_INCREASE_TO(opTimeWaiterListMetric, 2); + ASSERT_EQ(replicationWaiterListMetric.get(), 0); + + replCoordSetMyLastAppliedOpTime(time2, Date_t() + Seconds(100)); + replCoordSetMyLastDurableOpTime(time2, Date_t() + Seconds(100)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(2, 1, time2)); + statusAndDur = awaiter1.getResult(); + ASSERT_OK(statusAndDur.status); + statusAndDur = awaiter2.getResult(); + ASSERT_OK(statusAndDur.status); + awaiter1.reset(); + awaiter2.reset(); + ASSERT_EQ(replicationWaiterListMetric.get(), 0); + ASSERT_EQ(opTimeWaiterListMetric.get(), 0); + + // 2 nodes waiting for time3, one local one not, but we're going to step down first. + awaiter1.setOpTime(time3); + awaiter1.setWriteConcern(writeConcernLocal); + awaiter1.start(); + ASSERT_METRIC_INCREASE_TO(opTimeWaiterListMetric, 1); + ASSERT_EQ(replicationWaiterListMetric.get(), 0); + + awaiter2.setOpTime(time3); + awaiter2.setWriteConcern(writeConcern); + awaiter2.start(); + ASSERT_METRIC_INCREASE_TO(replicationWaiterListMetric, 1); + ASSERT_EQ(opTimeWaiterListMetric.get(), 1); + + const auto opCtx = makeOperationContext(); + getReplCoord()->stepDown(opCtx.get(), true, Milliseconds(0), Milliseconds(1000)); + + statusAndDur = awaiter1.getResult(); + ASSERT_EQUALS(ErrorCodes::PrimarySteppedDown, statusAndDur.status); + statusAndDur = awaiter2.getResult(); + ASSERT_EQUALS(ErrorCodes::PrimarySteppedDown, statusAndDur.status); + awaiter1.reset(); + awaiter2.reset(); + ASSERT_EQ(replicationWaiterListMetric.get(), 0); + ASSERT_EQ(opTimeWaiterListMetric.get(), 0); +} + TEST_F(ReplCoordTest, NodeReturnsFromMajorityWriteConcernOnlyOnceTheWriteAppearsInACommittedSnapShot) { // Test that we can satisfy majority write concern can only be diff --git a/src/mongo/db/repl/split_horizon.cpp b/src/mongo/db/repl/split_horizon.cpp index f588445398b..6201d5f9ba5 100644 --- a/src/mongo/db/repl/split_horizon.cpp +++ b/src/mongo/db/repl/split_horizon.cpp @@ -181,7 +181,8 @@ auto SplitHorizon::getParameters(const Client* const client) -> Parameters { return getSplitHorizonParameters(*client); } -StringData SplitHorizon::determineHorizon(const SplitHorizon::Parameters& horizonParameters) const { +std::string SplitHorizon::determineHorizon( + const SplitHorizon::Parameters& horizonParameters) const { if (horizonParameters.sniName) { const auto sniName = *horizonParameters.sniName; const auto found = _reverseHostMapping.find(sniName); @@ -189,7 +190,7 @@ StringData SplitHorizon::determineHorizon(const SplitHorizon::Parameters& horizo return found->second; } } - return kDefaultHorizon; + return kDefaultHorizon.toString(); } void SplitHorizon::toBSON(BSONObjBuilder& configBuilder) const { diff --git a/src/mongo/db/repl/split_horizon.h b/src/mongo/db/repl/split_horizon.h index b597cc7dec3..e9e11f1bcbd 100644 --- a/src/mongo/db/repl/split_horizon.h +++ b/src/mongo/db/repl/split_horizon.h @@ -91,7 +91,7 @@ public: * Gets the horizon name for which the parameters (captured during the first `isMaster`) * correspond. */ - StringData determineHorizon(const Parameters& horizonParameters) const; + std::string determineHorizon(const Parameters& horizonParameters) const; const HostAndPort& getHostAndPort(StringData horizon) const { invariant(!_forwardMapping.empty()); diff --git a/src/mongo/db/repl/split_horizon_test.cpp b/src/mongo/db/repl/split_horizon_test.cpp index 486c43f2ade..9f728012614 100644 --- a/src/mongo/db/repl/split_horizon_test.cpp +++ b/src/mongo/db/repl/split_horizon_test.cpp @@ -110,7 +110,7 @@ TEST(SplitHorizonTesting, determineHorizon) { const auto& input = test.input; const std::string witness = - SplitHorizon(input.forwardMapping).determineHorizon(input.horizonParameters).toString(); + SplitHorizon(input.forwardMapping).determineHorizon(input.horizonParameters); ASSERT_EQUALS(witness, expected); } diff --git a/src/mongo/db/repl/storage_timestamp_test.cpp b/src/mongo/db/repl/storage_timestamp_test.cpp index acb142f339e..6431be37aed 100644 --- a/src/mongo/db/repl/storage_timestamp_test.cpp +++ b/src/mongo/db/repl/storage_timestamp_test.cpp @@ -90,7 +90,6 @@ #include "mongo/db/transaction_participant_gen.h" #include "mongo/db/vector_clock_mutable.h" #include "mongo/dbtests/dbtests.h" -#include "mongo/idl/server_parameter_test_util.h" #include "mongo/logv2/log.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/stdx/future.h" diff --git a/src/mongo/db/repl/topology_coordinator.cpp b/src/mongo/db/repl/topology_coordinator.cpp index 630fefa541c..77405c9a84f 100644 --- a/src/mongo/db/repl/topology_coordinator.cpp +++ b/src/mongo/db/repl/topology_coordinator.cpp @@ -339,6 +339,12 @@ HostAndPort TopologyCoordinator::_chooseNearbySyncSource(Date_t now, // // This loop attempts to set 'closestIndex', to select a viable candidate. for (int attempts = 0; attempts < 2; ++attempts) { + if (attempts == 1) { + LOGV2_INFO( + 8423402, + "Failed to select a sync source on the first attempt. Starting second attempt"); + } + for (size_t candidateIndex = 0; candidateIndex < _memberData.size(); candidateIndex++) { if (!_isEligibleSyncSource(candidateIndex, now, @@ -359,18 +365,21 @@ HostAndPort TopologyCoordinator::_chooseNearbySyncSource(Date_t now, const auto syncSourceCandidate = _rsConfig.getMemberAt(candidateIndex).getHostAndPort(); const auto closestNode = _rsConfig.getMemberAt(closestIndex).getHostAndPort(); + LOGV2_INFO(8423401, + "Sync source candidate is eligible", + "syncSourceCandidate"_attr = syncSourceCandidate); + // Do not update 'closestIndex' if the candidate is not the closest node we've seen. auto syncSourceCandidatePing = _getPing(syncSourceCandidate); auto closestPing = _getPing(closestNode); if (syncSourceCandidatePing > closestPing) { - LOGV2_DEBUG(3873114, - 2, - "Cannot select sync source with higher latency than the best " - "candidate", - "syncSourceCandidate"_attr = syncSourceCandidate, - "syncSourceCandidatePing"_attr = syncSourceCandidatePing, - "closestNode"_attr = closestNode, - "closestPing"_attr = closestPing); + LOGV2_INFO(3873114, + "Cannot select sync source with higher latency than the best " + "candidate", + "syncSourceCandidate"_attr = syncSourceCandidate, + "syncSourceCandidatePing"_attr = syncSourceCandidatePing, + "closestNode"_attr = closestNode, + "closestPing"_attr = closestPing); continue; } closestIndex = candidateIndex; @@ -435,18 +444,16 @@ bool TopologyCoordinator::_isEligibleSyncSource(int candidateIndex, // Candidate must be up to be considered. if (!memberData.up()) { - LOGV2_DEBUG(3873106, - 2, - "Cannot select sync source because it is not up", - "syncSourceCandidate"_attr = syncSourceCandidate); + LOGV2_INFO(3873106, + "Cannot select sync source because it is not up", + "syncSourceCandidate"_attr = syncSourceCandidate); return false; } // Candidate must be PRIMARY or SECONDARY state to be considered. if (!memberData.getState().readable()) { - LOGV2_DEBUG(3873107, - 2, - "Cannot select sync source because it is not readable", - "syncSourceCandidate"_attr = syncSourceCandidate); + LOGV2_INFO(3873107, + "Cannot select sync source because it is not readable", + "syncSourceCandidate"_attr = syncSourceCandidate); return false; } @@ -454,11 +461,10 @@ bool TopologyCoordinator::_isEligibleSyncSource(int candidateIndex, if (readPreference == ReadPreference::SecondaryOnly || (readPreference == ReadPreference::SecondaryPreferred && firstAttempt)) { if (memberData.getState().primary()) { - LOGV2_DEBUG(3873101, - 2, - "Cannot select sync source because it is a primary and we are " - "looking for a secondary", - "syncSourceCandidate"_attr = syncSourceCandidate); + LOGV2_INFO(3873101, + "Cannot select sync source because it is a primary and we are " + "looking for a secondary", + "syncSourceCandidate"_attr = syncSourceCandidate); return false; } } @@ -467,73 +473,66 @@ bool TopologyCoordinator::_isEligibleSyncSource(int candidateIndex, if (firstAttempt) { // Candidate must be a voter if we are a voter. if (_selfConfig().isVoter() && !memberConfig.isVoter()) { - LOGV2_DEBUG(3873108, - 2, - "Cannot select sync source because we are a voter and it is not", - "syncSourceCandidate"_attr = syncSourceCandidate); + LOGV2_INFO(3873108, + "Cannot select sync source because we are a voter and it is not", + "syncSourceCandidate"_attr = syncSourceCandidate); return false; } // Candidates must not be hidden. if (memberConfig.isHidden()) { - LOGV2_DEBUG(3873109, - 2, - "Cannot select sync source because it is hidden", - "syncSourceCandidate"_attr = syncSourceCandidate); + LOGV2_INFO(3873109, + "Cannot select sync source because it is hidden", + "syncSourceCandidate"_attr = syncSourceCandidate); return false; } // Candidates cannot be excessively behind, if we are checking for staleness. if (shouldCheckStaleness) { const auto oldestSyncOpTime = _getOldestSyncOpTime(); if (memberData.getHeartbeatAppliedOpTime() < oldestSyncOpTime) { - LOGV2_DEBUG(3873110, - 2, - "Cannot select sync source because it is too far behind", - "syncSourceCandidate"_attr = syncSourceCandidate, - "syncSourceCandidateOpTime"_attr = - memberData.getHeartbeatAppliedOpTime(), - "oldestAcceptableOpTime"_attr = oldestSyncOpTime); + LOGV2_INFO(3873110, + "Cannot select sync source because it is too far behind", + "syncSourceCandidate"_attr = syncSourceCandidate, + "syncSourceCandidateOpTime"_attr = + memberData.getHeartbeatAppliedOpTime(), + "oldestAcceptableOpTime"_attr = oldestSyncOpTime); return false; } } // Candidate must not have a configured delay larger than ours. if (_selfConfig().getSecondaryDelay() < memberConfig.getSecondaryDelay()) { - LOGV2_DEBUG(3873111, - 2, - "Cannot select sync source with larger secondaryDelaySecs than ours", - "syncSourceCandidate"_attr = syncSourceCandidate, - "syncSourceCandidateSecondaryDelaySecs"_attr = - memberConfig.getSecondaryDelay(), - "secondaryDelaySecs"_attr = _selfConfig().getSecondaryDelay()); + LOGV2_INFO(3873111, + "Cannot select sync source with larger secondaryDelaySecs than ours", + "syncSourceCandidate"_attr = syncSourceCandidate, + "syncSourceCandidateSecondaryDelaySecs"_attr = + memberConfig.getSecondaryDelay(), + "secondaryDelaySecs"_attr = _selfConfig().getSecondaryDelay()); return false; } } // Candidate must build indexes if we build indexes, to be considered. if (_selfConfig().shouldBuildIndexes()) { if (!memberConfig.shouldBuildIndexes()) { - LOGV2_DEBUG(3873112, - 2, - "Cannot select sync source which does not build indexes when we do", - "syncSourceCandidate"_attr = syncSourceCandidate); + LOGV2_INFO(3873112, + "Cannot select sync source which does not build indexes when we do", + "syncSourceCandidate"_attr = syncSourceCandidate); return false; } } // Only select a candidate that is ahead of me, if we are checking for staleness. if (shouldCheckStaleness && memberData.getHeartbeatAppliedOpTime() <= lastOpTimeFetched) { - LOGV2_DEBUG(3873113, - 1, - "Cannot select sync source which is not ahead of me", - "syncSourceCandidate"_attr = syncSourceCandidate, - "syncSourceCandidateLastAppliedOpTime"_attr = - memberData.getHeartbeatAppliedOpTime().toBSON(), - "lastOpTimeFetched"_attr = lastOpTimeFetched.toBSON()); + LOGV2_INFO(3873113, + "Cannot select sync source which is not ahead of me", + "syncSourceCandidate"_attr = syncSourceCandidate, + "syncSourceCandidateLastAppliedOpTime"_attr = + memberData.getHeartbeatAppliedOpTime().toBSON(), + "lastOpTimeFetched"_attr = lastOpTimeFetched.toBSON()); return false; } // Candidate cannot be denylisted. if (_memberIsDenylisted(memberConfig, now)) { - LOGV2_DEBUG(3873115, - 1, - "Cannot select sync source which is denylisted", - "syncSourceCandidate"_attr = syncSourceCandidate); + LOGV2_INFO(3873115, + "Cannot select sync source which is denylisted", + "syncSourceCandidate"_attr = syncSourceCandidate); return false; } // This candidate has passed all tests. diff --git a/src/mongo/db/repl/topology_version_observer_test.cpp b/src/mongo/db/repl/topology_version_observer_test.cpp index 6d7d54d57b8..d7ee56b4778 100644 --- a/src/mongo/db/repl/topology_version_observer_test.cpp +++ b/src/mongo/db/repl/topology_version_observer_test.cpp @@ -42,7 +42,9 @@ #include "mongo/db/repl/replication_coordinator_impl.h" #include "mongo/db/repl/replication_coordinator_test_fixture.h" #include "mongo/db/repl/topology_version_observer.h" +#include "mongo/logv2/log.h" #include "mongo/unittest/barrier.h" +#include "mongo/unittest/log_test.h" #include "mongo/unittest/unittest.h" #include "mongo/util/assert_util.h" #include "mongo/util/clock_source.h" @@ -118,6 +120,9 @@ protected: const Milliseconds sleepTime = Milliseconds(100); std::unique_ptr<TopologyVersionObserver> observer; + + unittest::MinimumLoggedSeverityGuard severityGuard{logv2::LogComponent::kDefault, + logv2::LogSeverity::Debug(4)}; }; @@ -140,11 +145,15 @@ TEST_F(TopologyVersionObserverTest, UpdateCache) { auto electionTimeoutWhen = getReplCoord()->getElectionTimeout_forTest(); simulateSuccessfulV1ElectionWithoutExitingDrainMode(electionTimeoutWhen, opCtx.get()); + auto sleepCounter = 0; // Wait for the observer to update its cache while (observer->getCached()->getTopologyVersion()->getCounter() == cachedResponse->getTopologyVersion()->getCounter()) { sleepFor(sleepTime); + // Make sure the test doesn't wait here for longer than 15 seconds. + ASSERT_LTE(sleepCounter++, 150); } + LOGV2(9326401, "Observer topology incremented after successful election"); auto newResponse = observer->getCached(); ASSERT(newResponse && newResponse->getTopologyVersion()); diff --git a/src/mongo/db/repl/transaction_oplog_application.cpp b/src/mongo/db/repl/transaction_oplog_application.cpp index a71f8470212..20d8459a18b 100644 --- a/src/mongo/db/repl/transaction_oplog_application.cpp +++ b/src/mongo/db/repl/transaction_oplog_application.cpp @@ -33,6 +33,7 @@ #include "mongo/db/repl/transaction_oplog_application.h" +#include "mongo/db/catalog/document_validation.h" #include "mongo/db/catalog_raii.h" #include "mongo/db/commands/txn_cmds_gen.h" #include "mongo/db/concurrency/exception_util.h" @@ -538,6 +539,8 @@ void _reconstructPreparedTransaction(OperationContext* opCtx, const OplogEntry& prepareEntry, repl::OplogApplication::Mode mode) { repl::UnreplicatedWritesBlock uwb(opCtx); + // The transaction may have been prepared originally with document validation bypassed. + DisableDocumentValidation validationDisabler(opCtx); // Snapshot transaction can never conflict with the PBWM lock. opCtx->lockState()->setShouldConflictWithSecondaryBatchApplication(false); diff --git a/src/mongo/db/repl_index_build_state.h b/src/mongo/db/repl_index_build_state.h index 428d1f99004..166c59c3db5 100644 --- a/src/mongo/db/repl_index_build_state.h +++ b/src/mongo/db/repl_index_build_state.h @@ -141,12 +141,17 @@ public: */ kPrepareCommit = 1 << 2, /** - * Below state indicates that index build was successfully able to commit or abort. For - * kCommitted, the state is set immediately before it commits the index build. For - * kAborted, this state is set after the build is cleaned up and the abort oplog entry is - * replicated. + * Below state indicates that index build was successfully able to commit and is set + * immediately before it commits the index build. */ kCommitted = 1 << 3, + /** + * Below state indicates that index build was successfully able to abort. In case of self + * abort this state is set after the build is cleaned up and the abort oplog entry is + * replicated. In case of an external abort, this state is set before interrupting the + * builder thread, as a way of indicating that a self abort is not required. Cleanup and + * oplog entry replicating in this case is done after setting the state. + */ kAborted = 1 << 4, }; diff --git a/src/mongo/db/s/add_shard_cmd.cpp b/src/mongo/db/s/add_shard_cmd.cpp index 3bfc1cc4a18..af64045ca6a 100644 --- a/src/mongo/db/s/add_shard_cmd.cpp +++ b/src/mongo/db/s/add_shard_cmd.cpp @@ -72,6 +72,14 @@ public: auto addShardCmd = request(); auto shardIdUpsertCmd = add_shard_util::createShardIdentityUpsertForAddShard(addShardCmd); + + // A request dispatched through a local client is served within the same thread that + // submits it (so that the opCtx needs to be used as the vehicle to pass the WC to the + // ServiceEntryPoint). + const auto originalWC = opCtx->getWriteConcern(); + ScopeGuard resetWCGuard([&] { opCtx->setWriteConcern(originalWC); }); + opCtx->setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern); + DBDirectClient localClient(opCtx); BSONObj res; diff --git a/src/mongo/db/s/add_shard_util.cpp b/src/mongo/db/s/add_shard_util.cpp index 957c2626e26..750dbf98d23 100644 --- a/src/mongo/db/s/add_shard_util.cpp +++ b/src/mongo/db/s/add_shard_util.cpp @@ -73,7 +73,6 @@ BSONObj createShardIdentityUpsertForAddShard(const AddShard& addShardCmd) { return updateOp; }()); - request.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); return request.toBSON(); } diff --git a/src/mongo/db/s/balancer/balancer.cpp b/src/mongo/db/s/balancer/balancer.cpp index d194d900d29..caf1a8cbe3d 100644 --- a/src/mongo/db/s/balancer/balancer.cpp +++ b/src/mongo/db/s/balancer/balancer.cpp @@ -78,6 +78,7 @@ using std::vector; namespace { MONGO_FAIL_POINT_DEFINE(overrideBalanceRoundInterval); +MONGO_FAIL_POINT_DEFINE(forceBalancerWarningChecks); const Milliseconds kBalanceRoundDefaultInterval(10 * 1000); @@ -89,9 +90,6 @@ static constexpr StringData kBalancerPolicyStatusZoneViolation = "zoneViolation" static constexpr StringData kBalancerPolicyStatusChunksImbalance = "chunksImbalance"_sd; static constexpr StringData kBalancerPolicyStatusDefragmentingChunks = "defragmentingChunks"_sd; -// Time interval between checks on draining shards. -constexpr Minutes kDrainingShardsCheckInterval{10}; - /** * Utility class to generate timing and statistics for a single balancer round. */ @@ -237,6 +235,7 @@ std::vector<std::string> getDrainingShardNames(OperationContext* opCtx) { // Build the list of the draining shard names. std::vector<std::string> drainingShardNames; + drainingShardNames.reserve(drainingShardsDocs.size()); std::transform(drainingShardsDocs.begin(), drainingShardsDocs.end(), std::back_inserter(drainingShardNames), @@ -247,6 +246,105 @@ std::vector<std::string> getDrainingShardNames(OperationContext* opCtx) { return drainingShardNames; } +class BalancerWarning { + // Time interval between checks on draining shards. + constexpr static Minutes kDrainingShardsCheckInterval{10}; + +public: + BalancerWarning() = default; + + void warnIfRequired(OperationContext* opCtx, BalancerSettingsType::BalancerMode balancerMode) { + if (Date_t::now() - _lastDrainingShardsCheckTime < kDrainingShardsCheckInterval && + MONGO_likely(!forceBalancerWarningChecks.shouldFail())) { + return; + } + _lastDrainingShardsCheckTime = Date_t::now(); + + LOGV2(7977401, "Performing balancer warning checks"); + + const auto drainingShardNames{getDrainingShardNames(opCtx)}; + if (drainingShardNames.empty()) { + return; + } + + if (balancerMode == BalancerSettingsType::BalancerMode::kOff) { + LOGV2_WARNING( + 6434000, + "Draining of removed shards cannot be completed because the balancer is disabled", + "shards"_attr = drainingShardNames); + return; + } + + _warnIfDrainingShardHasChunksForCollectionWithBalancingDisabled(opCtx, drainingShardNames); + } + +private: + void _warnIfDrainingShardHasChunksForCollectionWithBalancingDisabled( + OperationContext* opCtx, const std::vector<std::string>& drainingShardNames) { + // Balancer is on, emit warning if balancer is disabled for collections which have chunks in + // shards in draining mode. + const auto catalogClient = Grid::get(opCtx)->catalogClient(); + auto collections = + catalogClient->getCollections(opCtx, + {}, + repl::ReadConcernLevel::kMajorityReadConcern, + BSON(CollectionType::kNssFieldName << 1)); + if (collections.empty()) { + return; + } + + // Construct BSONArray of draining shard names. + const auto drainingShardNameArray = [&]() { + BSONArrayBuilder shardNameArrayBuilder; + std::for_each(drainingShardNames.begin(), + drainingShardNames.end(), + [&shardNameArrayBuilder](const auto& shardName) { + shardNameArrayBuilder.append(shardName); + }); + return shardNameArrayBuilder.arr(); + }(); + + // For each collection, check if the collection has balancing disabled. If it is disabled, + // checks if the collection has any chunks in any of the draining shards. In which case a + // warning is emitted. + for (const auto& collType : collections) { + if (!collType.getAllowBalance() || !collType.getAllowMigrations() || + !collType.getPermitMigrations()) { + const auto findQuery = + BSON(ChunkType::collectionUUID() << collType.getUuid() << ChunkType::shard() + << BSON("$in" << drainingShardNameArray)); + + auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); + auto findResponse = uassertStatusOK(configShard->exhaustiveFindOnConfig( + opCtx, + ReadPreferenceSetting(ReadPreference::PrimaryOnly), + repl::ReadConcernLevel::kMajorityReadConcern, + ChunkType::ConfigNS, + findQuery, + BSONObj(), + boost::none)); + + const auto& chunks = findResponse.docs; + if (!chunks.empty()) { + stdx::unordered_set<std::string> shardsWithChunks; + std::for_each( + chunks.begin(), chunks.end(), [&shardsWithChunks](const BSONObj& chunkObj) { + shardsWithChunks.emplace(chunkObj.getStringField(ChunkType::shard())); + }); + LOGV2_WARNING( + 7977400, + "Draining of removed shards cannot be completed because the balancer is " + "disabled for a collection which has chunks in those shards", + "uuid"_attr = collType.getUuid(), + "nss"_attr = collType.getNss(), + "shardsWithChunks"_attr = shardsWithChunks); + } + } + } + } + + Date_t _lastDrainingShardsCheckTime{Date_t::fromMillisSinceEpoch(0)}; +}; } // namespace Balancer* Balancer::get(ServiceContext* serviceContext) { @@ -271,24 +369,29 @@ Balancer::Balancer() _imbalancedCollectionsCache(std::make_unique<stdx::unordered_set<NamespaceString>>()) {} Balancer::~Balancer() { - // Terminate the balancer thread so it doesn't leak memory. - interruptBalancer(); - waitForBalancerToStop(); + onShutdown(); } void Balancer::onStepUpBegin(OperationContext* opCtx, long long term) { - // Before starting step-up, ensure the balancer is ready to start. Specifically, that the - // balancer is actually stopped, because it may still be in the process of stopping if this - // node was previously primary. - waitForBalancerToStop(); + // Before starting step-up, ensure the balancer is ready to start. Specifically, that there is + // not an outstanding termination sequence requested during a previous step down of this node. + joinTermination(); } void Balancer::onStepUpComplete(OperationContext* opCtx, long long term) { - initiateBalancer(opCtx); + initiate(opCtx); } void Balancer::onStepDown() { - interruptBalancer(); + // Asynchronously request to terminate all the worker threads and allow the stepdown sequence to + // continue. + requestTermination(); +} + +void Balancer::onShutdown() { + // Terminate the balancer thread so it doesn't leak memory. + requestTermination(); + joinTermination(); } void Balancer::onBecomeArbiter() { @@ -297,11 +400,11 @@ void Balancer::onBecomeArbiter() { MONGO_UNREACHABLE; } -void Balancer::initiateBalancer(OperationContext* opCtx) { +void Balancer::initiate(OperationContext* opCtx) { stdx::lock_guard<Latch> scopedLock(_mutex); _imbalancedCollectionsCache->clear(); - invariant(_state == kStopped); - _state = kRunning; + invariant(_threadSetState == ThreadSetState::Terminated); + _threadSetState = ThreadSetState::Running; invariant(!_thread.joinable()); invariant(!_actionStreamConsumerThread.joinable()); @@ -309,13 +412,13 @@ void Balancer::initiateBalancer(OperationContext* opCtx) { _thread = stdx::thread([this] { _mainThread(); }); } -void Balancer::interruptBalancer() { +void Balancer::requestTermination() { stdx::lock_guard<Latch> scopedLock(_mutex); - if (_state != kRunning) { + if (_threadSetState != ThreadSetState::Running) { return; } - _state = kStopping; + _threadSetState = ThreadSetState::Terminating; // Interrupt the balancer thread if it has been started. We are guaranteed that the operation // context of that thread is still alive, because we hold the balancer mutex. @@ -328,9 +431,9 @@ void Balancer::interruptBalancer() { _defragmentationCondVar.notify_all(); } -void Balancer::waitForBalancerToStop() { +void Balancer::joinTermination() { stdx::unique_lock<Latch> scopedLock(_mutex); - _joinCond.wait(scopedLock, [this] { return _state == kStopped; }); + _joinCond.wait(scopedLock, [this] { return _threadSetState == ThreadSetState::Terminated; }); if (_thread.joinable()) { _thread.join(); } @@ -529,10 +632,10 @@ void Balancer::_consumeActionStreamLoop() { } auto canConsumeStream = selectedStream != nullptr && _outstandingStreamingOps.load() <= kMaxOutstandingStreamingOperations; - return _state != kRunning || + return _threadSetState != ThreadSetState::Running || (canConsumeStream && (!streamDrained || _newInfoOnStreamingActions.load())); }); - if (_state != kRunning) { + if (_threadSetState != ThreadSetState::Running) { break; } } @@ -543,9 +646,23 @@ void Balancer::_consumeActionStreamLoop() { "selectedStream"_attr = selectedStream->getName()); } - _newInfoOnStreamingActions.store(false); - auto nextAction = selectedStream->getNextStreamingAction(opCtx.get()); - if ((streamDrained = !nextAction.is_initialized())) { + boost::optional<DefragmentationAction> nextAction; + try { + _newInfoOnStreamingActions.store(false); + nextAction = selectedStream->getNextStreamingAction(opCtx.get()); + } catch (const DBException& e) { + LOGV2_WARNING(7435001, + "Failed to get next action from action stream", + "error"_attr = redact(e), + "stream"_attr = selectedStream->getName()); + + _newInfoOnStreamingActions.store(true); + continue; + } + + if (!nextAction.is_initialized()) { + // No action was returned by this stream. This means that the stream is drained. + streamDrained = true; continue; } @@ -636,8 +753,8 @@ void Balancer::_mainThread() { ON_BLOCK_EXIT([this] { { stdx::lock_guard<Latch> scopedLock(_mutex); - _state = kStopped; - LOGV2_DEBUG(21855, 1, "Balancer thread terminated"); + _threadSetState = ThreadSetState::Terminated; + LOGV2_DEBUG(21855, 1, "Balancer thread set terminated"); } _joinCond.notify_all(); }); @@ -656,7 +773,7 @@ void Balancer::_mainThread() { const Seconds kInitBackoffInterval(10); auto balancerConfig = shardingContext->getBalancerConfiguration(); - while (!_stopRequested()) { + while (!_terminationRequested()) { Status refreshStatus = balancerConfig->refreshAndCheck(opCtx.get()); if (!refreshStatus.isOK()) { LOGV2_WARNING( @@ -686,9 +803,9 @@ void Balancer::_mainThread() { LOGV2(6036606, "Balancer worker thread initialised. Entering main loop."); // Main balancer loop - auto lastDrainingShardsCheckTime{Date_t::fromMillisSinceEpoch(0)}; auto lastMigrationTime = Date_t::fromMillisSinceEpoch(0); - while (!_stopRequested()) { + BalancerWarning balancerWarning; + while (!_terminationRequested()) { BalanceRoundDetails roundDetails; _beginRound(opCtx.get()); @@ -708,22 +825,11 @@ void Balancer::_mainThread() { continue; } - if (!balancerConfig->shouldBalance() || _stopRequested() || - _clusterChunksResizePolicy->isActive()) { - - if (balancerConfig->getBalancerMode() == BalancerSettingsType::BalancerMode::kOff && - Date_t::now() - lastDrainingShardsCheckTime >= kDrainingShardsCheckInterval) { - const auto drainingShardNames{getDrainingShardNames(opCtx.get())}; - if (!drainingShardNames.empty()) { - LOGV2_WARNING(6434000, - "Draining of removed shards cannot be completed because the " - "balancer is disabled", - "shards"_attr = drainingShardNames); - } - - lastDrainingShardsCheckTime = Date_t::now(); - } + // Warn before we skip the iteration due to balancing being disabled. + balancerWarning.warnIfRequired(opCtx.get(), balancerConfig->getBalancerMode()); + if (!balancerConfig->shouldBalance() || _terminationRequested() || + _clusterChunksResizePolicy->isActive()) { LOGV2_DEBUG(21859, 1, "Skipping balancing round because balancing is disabled"); _endRound(opCtx.get(), kBalanceRoundDefaultInterval); continue; @@ -863,7 +969,7 @@ void Balancer::_mainThread() { { stdx::lock_guard<Latch> scopedLock(_mutex); - invariant(_state == kStopping); + invariant(_threadSetState == ThreadSetState::Terminating); } _commandScheduler->stop(); @@ -894,9 +1000,9 @@ void Balancer::_applyDefragmentationActionResponseToPolicy( policy->applyActionResult(opCtx.get(), action, response); }; -bool Balancer::_stopRequested() { +bool Balancer::_terminationRequested() { stdx::lock_guard<Latch> scopedLock(_mutex); - return (_state != kRunning); + return (_threadSetState != ThreadSetState::Running); } void Balancer::_beginRound(OperationContext* opCtx) { @@ -919,7 +1025,9 @@ void Balancer::_endRound(OperationContext* opCtx, Milliseconds waitTimeout) { void Balancer::_sleepFor(OperationContext* opCtx, Milliseconds waitTimeout) { stdx::unique_lock<Latch> lock(_mutex); - _condVar.wait_for(lock, waitTimeout.toSystemDuration(), [&] { return _state != kRunning; }); + _condVar.wait_for(lock, waitTimeout.toSystemDuration(), [&] { + return _threadSetState != ThreadSetState::Running; + }); } bool Balancer::_checkOIDs(OperationContext* opCtx) { @@ -931,7 +1039,7 @@ bool Balancer::_checkOIDs(OperationContext* opCtx) { map<int, ShardId> oids; for (const ShardId& shardId : all) { - if (_stopRequested()) { + if (_terminationRequested()) { return false; } @@ -1044,7 +1152,7 @@ int Balancer::_moveChunks(OperationContext* opCtx, auto catalogClient = Grid::get(opCtx)->catalogClient(); // If the balancer was disabled since we started this round, don't start new chunk moves - if (_stopRequested() || !balancerConfig->shouldBalance() || + if (_terminationRequested() || !balancerConfig->shouldBalance() || _clusterChunksResizePolicy->isActive()) { LOGV2_DEBUG(21870, 1, "Skipping balancing round because balancer was stopped"); return 0; @@ -1196,10 +1304,11 @@ void Balancer::_disableBalancer(OperationContext* opCtx, NamespaceString nss) { return updateOp; }()); - updateRequest.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); - - auto response = configShard->runBatchWriteCommand( - opCtx, Shard::kDefaultConfigCommandTimeout, updateRequest, Shard::RetryPolicy::kIdempotent); + auto response = configShard->runBatchWriteCommand(opCtx, + Shard::kDefaultConfigCommandTimeout, + updateRequest, + ShardingCatalogClient::kMajorityWriteConcern, + Shard::RetryPolicy::kIdempotent); uassertStatusOK(response.toStatus()); } diff --git a/src/mongo/db/s/balancer/balancer.h b/src/mongo/db/s/balancer/balancer.h index b84df5558bb..b9cdd2a6b6c 100644 --- a/src/mongo/db/s/balancer/balancer.h +++ b/src/mongo/db/s/balancer/balancer.h @@ -75,38 +75,40 @@ public: /** * Invoked when the config server primary enters the 'PRIMARY' state and is invoked while the - * caller is holding the global X lock. Kicks off the main balancer thread and returns - * immediately. Auto-balancing (if enabled) should commence shortly, and manual migrations will - * be processed and run. + * caller is holding the global X lock. Kicks off the main balancer thread (which will in turn + * instantiate a secondary worker and the CommandsScheduler) and returns immediately. + * Auto-balancing (if enabled) should commence shortly, and manual migrations will be processed + * and run. * - * Must only be called if the balancer is in the stopped state (i.e., just constructed or - * waitForBalancerToStop has been called before). Any code in this call must not try to acquire - * any locks or to wait on operations, which acquire locks. + * Must only be called if the balancer thread set is in the Terminated state (i.e., just + * constructed or joinTermination() has been called before). + * Any code in this call must not try to acquire any locks or to wait on operations, which + * acquire locks. */ - void initiateBalancer(OperationContext* opCtx); + void initiate(OperationContext* opCtx); /** * Invoked when this node which is currently serving as a 'PRIMARY' steps down and is invoked - * while the global X lock is held. Requests the main balancer thread to stop and returns - * immediately without waiting for it to terminate. Once the balancer has stopped, manual - * migrations will be rejected. + * while the global X lock is held. Requests to the hierarchy of balancer threads to leave and + * returns immediately without waiting for them to terminate. (Once the termination is complete, + * manual migrations will be rejected). * * This method might be called multiple times in succession, which is what happens as a result * of incomplete transition to primary so it is resilient to that. * - * The waitForBalancerToStop method must be called afterwards in order to wait for the main + * The joinTermination() method must be called afterwards in order to wait for the main * balancer thread to terminate and to allow initiateBalancer to be called again. */ - void interruptBalancer(); + void requestTermination(); /** * Invoked when a node on its way to becoming a primary finishes draining and is about to - * acquire the global X lock in order to allow writes. Waits for the balancer thread to - * terminate and primes the balancer so that initiateBalancer can be called. + * acquire the global X lock in order to allow writes. Waits for the hierarchy of balancer + * threads to terminate and primes the balancer so that initiateBalancer can be called. * * This must not be called while holding any locks! */ - void waitForBalancerToStop(); + void joinTermination(); /** * Potentially blocking method, which will return immediately if the balancer is not running a @@ -187,12 +189,20 @@ private: static constexpr int kMaxOutstandingStreamingOperations = 50; /** - * Possible runtime states of the balancer. The comments indicate the allowed next state. + * Possible runtime states of the set of threads instantiated by the balancer. + * The diagram below depicts the allowed transitions. + * Terminated --> Running --> Terminating + * ^ / / + * | / / + * \--------------------- */ - enum State { - kStopped, // kRunning - kRunning, // kStopping | kStopped - kStopping, // kStopped + enum class ThreadSetState { + // There is no worker thread instantiated by the balancer + Terminated, + // The balancer is initiliasing its worker threads (or they are all already active) + Running, + // A request to terminate all the balancer worker threads is ongoing + Terminating, }; /** @@ -200,7 +210,7 @@ private: */ void onStartup(OperationContext* opCtx) final {} void onInitialDataAvailable(OperationContext* opCtx, bool isMajorityDataAvailable) final {} - void onShutdown() final {} + void onShutdown() final; void onStepUpBegin(OperationContext* opCtx, long long term) final; void onStepUpComplete(OperationContext* opCtx, long long term) final; void onStepDown() final; @@ -217,9 +227,9 @@ private: void _consumeActionStreamLoop(); /** - * Checks whether the balancer main thread has been requested to stop. + * Checks whether the balancer is going through a termination sequence of its threads. */ - bool _stopRequested(); + bool _terminationRequested(); /** * Signals the beginning and end of a balancing round. @@ -277,8 +287,9 @@ private: // Protects the state below Mutex _mutex = MONGO_MAKE_LATCH("Balancer::_mutex"); - // Indicates the current state of the balancer - State _state{kStopped}; + // Indicates the current state of the worker threads instantiated by the balancer + // (_thread, _actionStreamConsumerThread and _commandScheduler) + ThreadSetState _threadSetState{ThreadSetState::Terminated}; // The main balancer threads stdx::thread _thread; diff --git a/src/mongo/db/s/balancer_stats_registry.cpp b/src/mongo/db/s/balancer_stats_registry.cpp index 508648eb29f..0d262b2c299 100644 --- a/src/mongo/db/s/balancer_stats_registry.cpp +++ b/src/mongo/db/s/balancer_stats_registry.cpp @@ -284,9 +284,12 @@ void BalancerStatsRegistry::updateOrphansCount(const UUID& collectionUUID, long stats.numOrphanDocs += delta; if (stats.numOrphanDocs < 0) { - // This should happen only in case of direct manipulation of range deletion tasks - // documents or direct writes into orphaned ranges - LOGV2_ERROR(6419611, + // This could happen in case of direct manipulation of range deletion tasks documents or + // direct writes into orphaned ranges, but also in some other benign situations. + // numOrphanDocs is a best-effort counter, miscounting or even being negative in some + // scenarios is expected. + LOGV2_DEBUG(6419611, + 1, "Cached orphan documents count became negative, resetting it to 0", "collectionUUID"_attr = collectionUUID, "numOrphanDocs"_attr = stats.numOrphanDocs, @@ -334,7 +337,8 @@ void BalancerStatsRegistry::_loadOrphansCount(OperationContext* opCtx) { auto numRangeDeletionTasks = collObj[kNumRangeDeletionTasksLabel].exactNumberLong(); invariant(numRangeDeletionTasks > 0); if (orphanCount < 0) { - LOGV2_ERROR(6419621, + LOGV2_DEBUG(6419621, + 1, "Found negative orphan count in range deletion task documents", "collectionUUID"_attr = collUUID, "numOrphanDocs"_attr = orphanCount, diff --git a/src/mongo/db/s/check_sharding_index_command.cpp b/src/mongo/db/s/check_sharding_index_command.cpp index 7849ab169d3..dd5331e33f1 100644 --- a/src/mongo/db/s/check_sharding_index_command.cpp +++ b/src/mongo/db/s/check_sharding_index_command.cpp @@ -101,10 +101,8 @@ public: keyPattern, /*requireSingleKey=*/true, &tmpErrMsg); - if (!shardKeyIdx) { - errmsg = tmpErrMsg; - return false; - } + + uassert(ErrorCodes::InvalidOptions, str::stream() << tmpErrMsg, shardKeyIdx); return true; } diff --git a/src/mongo/db/s/config/initial_split_policy.cpp b/src/mongo/db/s/config/initial_split_policy.cpp index 15c5a345c59..0d623d81fba 100644 --- a/src/mongo/db/s/config/initial_split_policy.cpp +++ b/src/mongo/db/s/config/initial_split_policy.cpp @@ -179,7 +179,7 @@ StringMap<std::vector<ShardId>> buildTagsToShardIdsMap(OperationContext* opCtx, } // namespace std::vector<BSONObj> InitialSplitPolicy::calculateHashedSplitPoints( - const ShardKeyPattern& shardKeyPattern, BSONObj prefix, int numInitialChunks) { + const ShardKeyPattern& shardKeyPattern, BSONObj prefix, size_t numInitialChunks) { invariant(shardKeyPattern.isHashedPattern()); invariant(numInitialChunks > 0); @@ -224,7 +224,7 @@ std::vector<BSONObj> InitialSplitPolicy::calculateHashedSplitPoints( current += intervalSize / 2; } - for (int i = 0; i < (numInitialChunks - 1) / 2; i++) { + for (size_t i = 0; i < (numInitialChunks - 1) / 2; i++) { splitPoints.push_back(buildSplitPoint(current)); splitPoints.push_back(buildSplitPoint(-current)); current += intervalSize; diff --git a/src/mongo/db/s/config/initial_split_policy.h b/src/mongo/db/s/config/initial_split_policy.h index e492e9c4cb2..7103b89d91b 100644 --- a/src/mongo/db/s/config/initial_split_policy.h +++ b/src/mongo/db/s/config/initial_split_policy.h @@ -104,7 +104,7 @@ public: */ static std::vector<BSONObj> calculateHashedSplitPoints(const ShardKeyPattern& shardKeyPattern, BSONObj prefix, - int numInitialChunks); + size_t numInitialChunks); /** * Produces the initial chunks that need to be written for an *empty* collection which is being diff --git a/src/mongo/db/s/config/sharding_catalog_manager.cpp b/src/mongo/db/s/config/sharding_catalog_manager.cpp index bb4980b5eb0..d380fe58468 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager.cpp @@ -44,6 +44,7 @@ #include "mongo/db/internal_transactions_feature_flag_gen.h" #include "mongo/db/operation_context.h" #include "mongo/db/ops/write_ops.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/db/query/query_request_helper.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/s/balancer/type_migration.h" diff --git a/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp index db1d21033bd..aa483a030a5 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp @@ -460,11 +460,7 @@ TEST_F(AddShardTest, CreateShardIdentityUpsertForAddShard) { << shardName << "clusterId" << _clusterId << "configsvrConnectionString" << replicationCoordinator()->getConfigConnectionString().toString()) - << "multi" << false << "upsert" << true)) - << "writeConcern" - << BSON("w" - << "majority" - << "wtimeout" << 60000)); + << "multi" << false << "upsert" << true))); auto addShardCmd = add_shard_util::createAddShardCmd(operationContext(), shardName); auto actualBSON = add_shard_util::createShardIdentityUpsertForAddShard(addShardCmd); ASSERT_BSONOBJ_EQ(expectedBSON, actualBSON); diff --git a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp index 0c3a23b1582..d1bc8fcd343 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp @@ -43,6 +43,7 @@ #include "mongo/db/logical_session_cache.h" #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/db/query/distinct_command_gen.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/s/sharding_logging.h" @@ -645,6 +646,11 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkSplit( newChunkBounds.push_back(range.getMax()); if (isSplitAlreadyDone(opCtx, range, shardName, origChunk.getValue(), newChunkBounds)) { + // In case the request was already fullfilled, we still need to wait until the original + // request is majority written. The timestamp is not known, so we use the system's last + // optime. Otherwise the next RoutingInfo cache refresh from the shard may not see the + // newest information. + repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); return buildChunkVersionBSON(collVersion); } @@ -1404,10 +1410,12 @@ void ShardingCatalogManager::upgradeChunksHistory(OperationContext* opCtx, }()}); return updateOp; }()); - request.setWriteConcern(ShardingCatalogClient::kLocalWriteConcern.toBSON()); - auto response = configShard->runBatchWriteCommand( - opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kIdempotent); + auto response = configShard->runBatchWriteCommand(opCtx, + Shard::kDefaultConfigCommandTimeout, + request, + ShardingCatalogClient::kLocalWriteConcern, + Shard::RetryPolicy::kIdempotent); uassertStatusOK(response.toStatus()); uassert(ErrorCodes::Error(5760502), @@ -2073,11 +2081,13 @@ bool ShardingCatalogManager::clearChunkEstimatedSize(OperationContext* opCtx, co }()}); return updateOp; }()); - request.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto response = configShard->runBatchWriteCommand( - opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kIdempotent); + auto response = configShard->runBatchWriteCommand(opCtx, + Shard::kDefaultConfigCommandTimeout, + request, + ShardingCatalogClient::kMajorityWriteConcern, + Shard::RetryPolicy::kIdempotent); uassertStatusOK(response.toStatus()); return response.getN() > 0; diff --git a/src/mongo/db/s/create_collection_coordinator.cpp b/src/mongo/db/s/create_collection_coordinator.cpp index e1f42dfdcfe..9daff7b07a5 100644 --- a/src/mongo/db/s/create_collection_coordinator.cpp +++ b/src/mongo/db/s/create_collection_coordinator.cpp @@ -334,16 +334,14 @@ void insertCollectionEntry(OperationContext* opCtx, BatchedCommandRequest insertRequest( write_ops::InsertCommandRequest(CollectionType::ConfigNS, {coll.toBSON()})); - insertRequest.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); - - const BSONObj cmdObj = insertRequest.toBSON().addFields(osi.toBSON()); + const auto cmdObj = CommandHelpers::appendMajorityWriteConcern(insertRequest.toBSON()); BatchedCommandResponse unusedResponse; uassertStatusOK(Shard::CommandResponse::processBatchWriteResponse( configShard->runCommand(opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}, CollectionType::ConfigNS.db().toString(), - cmdObj, + cmdObj.addFields(osi.toBSON()), Shard::kDefaultConfigCommandTimeout, Shard::RetryPolicy::kIdempotent), &unusedResponse)); @@ -705,7 +703,7 @@ void CreateCollectionCoordinator::_checkCommandArguments(OperationContext* opCtx const int maxNumInitialChunksForShards = Grid::get(opCtx)->shardRegistry()->getNumShardsNoReload() * shardutil::kMaxSplitPoints; const int maxNumInitialChunksTotal = 1000 * 1000; // Arbitrary limit to memory consumption - int numChunks = _request.getNumInitialChunks().value(); + const auto numChunks = _request.getNumInitialChunks().value(); uassert(ErrorCodes::InvalidOptions, str::stream() << "numInitialChunks cannot be more than either: " << maxNumInitialChunksForShards << ", " << shardutil::kMaxSplitPoints diff --git a/src/mongo/db/s/dist_lock_catalog_replset.cpp b/src/mongo/db/s/dist_lock_catalog_replset.cpp index 5a0dd02d83d..15e5c76e492 100644 --- a/src/mongo/db/s/dist_lock_catalog_replset.cpp +++ b/src/mongo/db/s/dist_lock_catalog_replset.cpp @@ -377,38 +377,15 @@ Status DistLockCatalogImpl::unlockAll(OperationContext* opCtx, }()}); return updateOp; }()); - request.setWriteConcern(kLocalWriteConcern.toBSON()); - - BSONObj cmdObj = request.toBSON(); auto const shardRegistry = Grid::get(opCtx)->shardRegistry(); - auto response = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts( - opCtx, - ReadPreferenceSetting{ReadPreference::PrimaryOnly}, - _locksNS.db().toString(), - cmdObj, - Shard::kDefaultConfigCommandTimeout, - Shard::RetryPolicy::kIdempotent); + auto batchResponse = + shardRegistry->getConfigShard()->runBatchWriteCommand(opCtx, + Shard::kDefaultConfigCommandTimeout, + request, + kLocalWriteConcern, + Shard::RetryPolicy::kIdempotent); - if (!response.isOK()) { - return response.getStatus(); - } - if (!response.getValue().commandStatus.isOK()) { - return response.getValue().commandStatus; - } - if (!response.getValue().writeConcernStatus.isOK()) { - return response.getValue().writeConcernStatus; - } - - BatchedCommandResponse batchResponse; - std::string errmsg; - if (!batchResponse.parseBSON(response.getValue().response, &errmsg)) { - return Status(ErrorCodes::FailedToParse, - str::stream() - << "Failed to parse config server response to batch request for " - "unlocking existing distributed locks" - << causedBy(errmsg)); - } return batchResponse.toStatus(); } diff --git a/src/mongo/db/s/dist_lock_catalog_replset_test.cpp b/src/mongo/db/s/dist_lock_catalog_replset_test.cpp index a05f6db250d..63b9d91700c 100644 --- a/src/mongo/db/s/dist_lock_catalog_replset_test.cpp +++ b/src/mongo/db/s/dist_lock_catalog_replset_test.cpp @@ -1234,8 +1234,6 @@ TEST_F(DistLockCatalogReplSetTest, BasicUnlockAll) { const auto opMsgRequest(OpMsgRequest::fromDBAndBody(request.dbname, request.cmdObj)); const auto commandRequest(BatchedCommandRequest::parseUpdate(opMsgRequest)); - ASSERT_BSONOBJ_EQ(BSON("w" << 1 << "wtimeout" << 0), commandRequest.getWriteConcern()); - const auto& updateOp = commandRequest.getUpdateRequest(); ASSERT_EQUALS(LocksType::ConfigNS, updateOp.getNamespace()); diff --git a/src/mongo/db/s/flush_resharding_state_change_command.cpp b/src/mongo/db/s/flush_resharding_state_change_command.cpp index cc4a4093dd1..53b3f9953ff 100644 --- a/src/mongo/db/s/flush_resharding_state_change_command.cpp +++ b/src/mongo/db/s/flush_resharding_state_change_command.cpp @@ -109,7 +109,11 @@ public: "Can't call _flushReshardingStateChange if in read-only mode", !storageGlobalParams.readOnly); - ExecutorFuture<void>(Grid::get(opCtx)->getExecutorPool()->getArbitraryExecutor()) + // We use the fixed executor here since it may cause the thread to block. This would + // cause potential liveness issues since the arbitrary executor is a NetworkInterfaceTL + // executor in sharded clusters and that executor is one that executes networking + // operations. + ExecutorFuture<void>(Grid::get(opCtx)->getExecutorPool()->getFixedExecutor()) .then([svcCtx = opCtx->getServiceContext(), nss = ns()] { ThreadClient tc("FlushReshardingStateChange", svcCtx); { diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp index 807d7d0da0a..776e620f072 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp +++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp @@ -214,7 +214,8 @@ public: MONGO_UNREACHABLE; } - bool doesTimeseriesBucketsDocContainMixedSchemaData(const BSONObj& bucketsDoc) const override { + StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData( + const BSONObj& bucketsDoc) const override { return _coll->doesTimeseriesBucketsDocContainMixedSchemaData(bucketsDoc); } diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp index 840c7e7ee5e..bbe4f34bf19 100644 --- a/src/mongo/db/s/migration_destination_manager.cpp +++ b/src/mongo/db/s/migration_destination_manager.cpp @@ -1488,7 +1488,7 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* outerOpCtx, runWithoutSession(outerOpCtx, [&] { auto awaitReplicationResult = repl::ReplicationCoordinator::get(opCtx)->awaitReplication( - opCtx, lastOpApplied, _writeConcern); + opCtx, lastOpApplied, WriteConcerns::kMajorityWriteConcernShardingTimeout); uassertStatusOKWithContext(awaitReplicationResult.status, awaitReplicationResult.status.codeString()); }); diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp index c4f301d52c7..9ca15fe9fad 100644 --- a/src/mongo/db/s/migration_source_manager.cpp +++ b/src/mongo/db/s/migration_source_manager.cpp @@ -426,6 +426,12 @@ void MigrationSourceManager::startClone() { _state = kCloning; } + // Refreshing the collection routing information after starting the clone driver will give us a + // stable view on whether the recipient is owning other chunks of the collection (a condition + // that will be later evaluated). + uassertStatusOK( + Grid::get(_opCtx)->catalogCache()->getCollectionRoutingInfoWithRefresh(_opCtx, nss())); + if (replEnabled) { auto const readConcernArgs = repl::ReadConcernArgs( replCoord->getMyLastAppliedOpTime(), repl::ReadConcernLevel::kLocalReadConcern); @@ -471,11 +477,12 @@ void MigrationSourceManager::enterCriticalSection() { _stats.totalDonorChunkCloneTimeMillis.addAndFetch(_cloneAndCommitTimer.millis()); _cloneAndCommitTimer.reset(); - const auto& metadata = _getCurrentMetadataAndCheckEpoch(); + const auto cm = + uassertStatusOK(Grid::get(_opCtx)->catalogCache()->getCollectionRoutingInfo(_opCtx, nss())); // Check that there are no chunks on the recepient shard. Write an oplog event for change // streams if this is the first migration to the recipient. - if (!metadata.getChunkManager()->getVersion(_args.getToShard()).isSet()) { + if (!cm.getVersion(_args.getToShard()).isSet()) { migrationutil::notifyChangeStreamsOnRecipientFirstChunk( _opCtx, nss(), _args.getFromShard(), _args.getToShard(), _collectionUUID); } diff --git a/src/mongo/db/s/range_deletion_util.cpp b/src/mongo/db/s/range_deletion_util.cpp index 9ca0a8f2518..54944dcb0bd 100644 --- a/src/mongo/db/s/range_deletion_util.cpp +++ b/src/mongo/db/s/range_deletion_util.cpp @@ -52,6 +52,8 @@ #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/wait_for_majority_service.h" #include "mongo/db/s/migration_util.h" +#include "mongo/db/s/operation_sharding_state.h" +#include "mongo/db/s/shard_filtering_metadata_refresh.h" #include "mongo/db/s/shard_key_index_util.h" #include "mongo/db/s/sharding_runtime_d_params_gen.h" #include "mongo/db/s/sharding_statistics.h" @@ -668,45 +670,59 @@ void setOrphanCountersOnRangeDeletionTasks(OperationContext* opCtx) { opCtx, BSONObj(), [opCtx, &store, &setNumOrphansOnTask](const RangeDeletionTask& deletionTask) { - AutoGetCollection collection(opCtx, deletionTask.getNss(), MODE_IX); - if (!collection || collection->uuid() != deletionTask.getCollectionUuid()) { - // The deletion task is referring to a collection that has been dropped - setNumOrphansOnTask(deletionTask, 0); - return true; - } + // The operation context is not bound to any specific namespace; acquire the shard role + // to ensure that the collection key pattern may be retrieved through the + // AutoGetCollection object. + ScopedSetShardRole scopedRole( + opCtx, deletionTask.getNss(), ChunkVersion::IGNORED(), boost::none); + while (true) { + try { + AutoGetCollection collection(opCtx, deletionTask.getNss(), MODE_IX); + if (!collection || collection->uuid() != deletionTask.getCollectionUuid()) { + // The deletion task is referring to a collection that has been dropped + setNumOrphansOnTask(deletionTask, 0); + return true; + } - KeyPattern keyPattern; - uassertStatusOK(deletionTask.getRange().extractKeyPattern(&keyPattern)); - auto shardKeyIdx = findShardKeyPrefixedIndex(opCtx, - *collection, - collection->getIndexCatalog(), - keyPattern.toBSON(), - /*requireSingleKey=*/false); - - uassert(ErrorCodes::IndexNotFound, - str::stream() << "couldn't find index over shard key " << keyPattern.toBSON() - << " for collection " << deletionTask.getNss() - << " (uuid: " << deletionTask.getCollectionUuid() << ")", - shardKeyIdx); - - const auto& range = deletionTask.getRange(); - auto forwardIdxScanner = - InternalPlanner::shardKeyIndexScan(opCtx, - &(*collection), - *shardKeyIdx, - range.getMin(), - range.getMax(), - BoundInclusion::kIncludeStartKeyOnly, - PlanYieldPolicy::YieldPolicy::YIELD_AUTO, - InternalPlanner::FORWARD); - int64_t numOrphansInRange = 0; - BSONObj indexEntry; - while (forwardIdxScanner->getNext(&indexEntry, nullptr) != PlanExecutor::IS_EOF) { - ++numOrphansInRange; - } - setNumOrphansOnTask(deletionTask, numOrphansInRange); - return true; + const auto keyPattern = collection.getCollection().getShardKeyPattern(); + auto shardKeyIdx = findShardKeyPrefixedIndex(opCtx, + *collection, + collection->getIndexCatalog(), + keyPattern, + /*requireSingleKey=*/false); + + uassert(ErrorCodes::IndexNotFound, + str::stream() << "couldn't find index over shard key " << keyPattern + << " for collection " << deletionTask.getNss() + << " (uuid: " << deletionTask.getCollectionUuid() << ")", + shardKeyIdx); + + const auto& range = deletionTask.getRange(); + auto forwardIdxScanner = + InternalPlanner::shardKeyIndexScan(opCtx, + &(*collection), + *shardKeyIdx, + range.getMin(), + range.getMax(), + BoundInclusion::kIncludeStartKeyOnly, + PlanYieldPolicy::YieldPolicy::YIELD_AUTO, + InternalPlanner::FORWARD); + int64_t numOrphansInRange = 0; + BSONObj indexEntry; + while (forwardIdxScanner->getNext(&indexEntry, nullptr) != + PlanExecutor::IS_EOF) { + ++numOrphansInRange; + } + + setNumOrphansOnTask(deletionTask, numOrphansInRange); + return true; + + } catch (const ExceptionFor<ErrorCodes::StaleConfig>& e) { + onShardVersionMismatchNoExcept(opCtx, e->getNss(), e->getVersionReceived()) + .ignore(); + } + } }); } diff --git a/src/mongo/db/s/range_deletion_util_test.cpp b/src/mongo/db/s/range_deletion_util_test.cpp index 1a8154dffd8..c45eeb53e82 100644 --- a/src/mongo/db/s/range_deletion_util_test.cpp +++ b/src/mongo/db/s/range_deletion_util_test.cpp @@ -32,6 +32,7 @@ #include "mongo/db/catalog/create_collection.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" +#include "mongo/db/hasher.h" #include "mongo/db/persistent_task_store.h" #include "mongo/db/repl/wait_for_majority_service.h" #include "mongo/db/s/collection_sharding_runtime.h" @@ -51,7 +52,7 @@ namespace { const NamespaceString kNss = NamespaceString("foo", "bar"); const std::string kShardKey = "_id"; -const BSONObj kShardKeyPattern = BSON(kShardKey << 1); +const BSONObj kRangeBasedShardKeyPattern = BSON(kShardKey << 1); class RangeDeleterTest : public ShardServerTestFixture { public: @@ -96,13 +97,14 @@ public: ShardServerTestFixture::tearDown(); } - void setFilteringMetadataWithUUID(const UUID& uuid) { + void setFilteringMetadataWithUUID(const UUID& uuid, + const BSONObj& shardKeyPattern = kRangeBasedShardKeyPattern) { const OID epoch = OID::gen(); auto rt = RoutingTableHistory::makeNew( kNss, uuid, - kShardKeyPattern, + shardKeyPattern, nullptr, false, epoch, @@ -228,7 +230,7 @@ TEST_F(RangeDeleterTest, std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, task.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -257,7 +259,7 @@ TEST_F(RangeDeleterTest, std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, task.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -280,7 +282,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeInsertsDocumentToNotifySecondarie std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, task.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -313,7 +315,7 @@ TEST_F( std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, task.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -346,7 +348,7 @@ TEST_F(RangeDeleterTest, std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, task.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -378,7 +380,7 @@ TEST_F(RangeDeleterTest, std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, task.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -407,7 +409,7 @@ TEST_F(RangeDeleterTest, kNss, // Use a different UUID from the collection UUID. UUID::gen(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, task.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -429,7 +431,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeThrowsErrorWhenCollectionDoesNotE std::move(queriesComplete), NamespaceString("someFake", "namespace"), UUID::gen(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, ChunkRange(BSON(kShardKey << 0), BSON(kShardKey << 10)), task.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -472,7 +474,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeLeavesDocumentsWhenTaskDocumentDo std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, UUID::gen(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -522,7 +524,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeWaitsForReplicationAfterDeletingS std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, t.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -572,7 +574,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeWaitsForReplicationOnlyOnceAfterS std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, t.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -618,7 +620,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeDoesNotWaitForReplicationIfErrorD std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, t.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -648,7 +650,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRetriesOnWriteConflictException) std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, t.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -679,7 +681,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRetriesOnUnexpectedError) { std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, t.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -720,7 +722,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRespectsDelayInBetweenBatches) { std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, task.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -752,7 +754,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRespectsOrphanCleanupDelay) { std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, task.getId(), orphanCleanupDelay); @@ -790,7 +792,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRemovesRangeDeletionTaskOnSuccess std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, t.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -820,7 +822,7 @@ TEST_F(RangeDeleterTest, std::move(queriesComplete), kNss, fakeUuid, - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, t.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -857,7 +859,7 @@ TEST_F(RangeDeleterTest, std::move(queriesComplete), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, t.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -888,7 +890,7 @@ DEATH_TEST_F(RangeDeleterTest, RemoveDocumentsInRangeCrashesIfInputFutureHasErro std::move((queriesCompletePf.future)).semi(), kNss, uuid(), - kShardKeyPattern, + kRangeBasedShardKeyPattern, range, t.getId(), Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -1025,8 +1027,9 @@ TEST_F(RenameRangeDeletionsTest, IdempotentRenameRangeDeletionsTest) { ASSERT_EQ(0, forRenameStore.count(_opCtx, BSONObj())); } -TEST_F(RangeDeleterTest, - setOrphanCountersOnRangeDeletionTasksUpdatesTaskWithExpectedNumberOfOrphans) { +TEST_F( + RangeDeleterTest, + setOrphanCountersOnRangeDeletionTasksUpdatesTaskForCollectionWithRangeShardKeyWithExpectedNumberOfOrphans) { const auto numOrphansInRange = 5; setFilteringMetadataWithUUID(uuid()); @@ -1046,6 +1049,44 @@ TEST_F(RangeDeleterTest, 1); } +TEST_F( + RangeDeleterTest, + setOrphanCountersOnRangeDeletionTasksUpdatesTaskForCollectionWithHashedShardKeyWithExpectedNumberOfOrphans) { + const BSONObj kHashedShardKeyPattern = BSON(kShardKey << "hashed"); + + DBDirectClient dbClient(_opCtx); + dbClient.createIndex(kNss.ns(), + BSON("_id" + << "hashed")); + + setFilteringMetadataWithUUID(uuid(), kHashedShardKeyPattern); + + const auto orphanedRangeLowerBoud = std::numeric_limits<int64_t>::max() / 2; + const ChunkRange orphansRange(BSON(kShardKey << orphanedRangeLowerBoud), + BSON(kShardKey << MAXKEY)); + + auto t = insertRangeDeletionTask(_opCtx, uuid(), orphansRange); + const auto numDocInserted = 10; + auto numOrphansInRange = 0; + for (auto i = 0; i < numDocInserted; ++i) { + dbClient.insert(kNss.toString(), BSON(kShardKey << i)); + const auto hashedDocId = BSONElementHasher::hash64(BSON("_id" << i).firstElement(), + BSONElementHasher::DEFAULT_HASH_SEED); + if (hashedDocId >= orphanedRangeLowerBoud) { + ++numOrphansInRange; + } + } + + ASSERT(numOrphansInRange > 0); + + setOrphanCountersOnRangeDeletionTasks(_opCtx); + + PersistentTaskStore<RangeDeletionTask> store(NamespaceString::kRangeDeletionNamespace); + ASSERT_EQ( + store.count(_opCtx, BSON(RangeDeletionTask::kNumOrphanDocsFieldName << numOrphansInRange)), + 1); +} + TEST_F(RangeDeleterTest, setOrphanCountersOnRangeDeletionTasksAddsZeroValueWhenNamespaceNotFound) { NamespaceString unexistentCollection("foo", "iDontExist"); auto collUuid = UUID::gen(); diff --git a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp index 75cb7be7049..2695ed842c0 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp +++ b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp @@ -101,8 +101,7 @@ StageConstraints DocumentSourceReshardingAddResumeId::constraints( ChangeStreamRequirement::kDenylist); } -Value DocumentSourceReshardingAddResumeId::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceReshardingAddResumeId::serialize(const SerializationOptions& opts) const { return Value(Document{{kStageName, Value(Document{})}}); } diff --git a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h index 31cbd97c694..4fb27980c68 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h +++ b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h @@ -53,7 +53,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp index 8075111e3af..8260cf3e4cc 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp +++ b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp @@ -112,7 +112,7 @@ StageConstraints DocumentSourceReshardingIterateTransaction::constraints( } Value DocumentSourceReshardingIterateTransaction::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { + const SerializationOptions& opts) const { return Value( Document{{kStageName, Value(Document{{kIncludeCommitTransactionTimestampFieldName, diff --git a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h index 9589cb64a08..8c792116e6c 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h +++ b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h @@ -66,7 +66,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp index 3144723bf2d..157876391d3 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp +++ b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp @@ -90,12 +90,11 @@ StageConstraints DocumentSourceReshardingOwnershipMatch::constraints( ChangeStreamRequirement::kDenylist); } -Value DocumentSourceReshardingOwnershipMatch::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceReshardingOwnershipMatch::serialize(const SerializationOptions& opts) const { return Value{Document{{kStageName, DocumentSourceReshardingOwnershipMatchSpec( _recipientShardId, _reshardingKey.getKeyPattern()) - .toBSON()}}}; + .toBSON(opts)}}}; } DepsTracker::State DocumentSourceReshardingOwnershipMatch::getDependencies( diff --git a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h index 7a6db2bc125..b7da07a5a57 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h +++ b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h @@ -58,7 +58,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp index 5700f0326ae..c87cb3667dd 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp @@ -91,17 +91,19 @@ CoordinatorCommitMonitor::CoordinatorCommitMonitor( std::vector<ShardId> recipientShards, CoordinatorCommitMonitor::TaskExecutorPtr executor, CancellationToken cancelToken, + int delayBeforeInitialQueryMillis, Milliseconds maxDelayBetweenQueries) : _ns(std::move(ns)), _recipientShards(std::move(recipientShards)), _executor(std::move(executor)), _cancelToken(std::move(cancelToken)), _threshold(Milliseconds(gRemainingReshardingOperationTimeThresholdMillis.load())), + _delayBeforeInitialQueryMillis(Milliseconds(delayBeforeInitialQueryMillis)), _maxDelayBetweenQueries(maxDelayBetweenQueries) {} SemiFuture<void> CoordinatorCommitMonitor::waitUntilRecipientsAreWithinCommitThreshold() const { - return _makeFuture() + return _makeFuture(_delayBeforeInitialQueryMillis) .onError([](Status status) { if (ErrorCodes::isCancellationError(status.code()) || ErrorCodes::isInterruption(status.code())) { @@ -195,9 +197,16 @@ CoordinatorCommitMonitor::queryRemainingOperationTimeForRecipients() const { return {minRemainingTime, maxRemainingTime}; } -ExecutorFuture<void> CoordinatorCommitMonitor::_makeFuture() const { +ExecutorFuture<void> CoordinatorCommitMonitor::_makeFuture(Milliseconds delayBetweenQueries) const { return ExecutorFuture<void>(_executor) - .then([this] { return queryRemainingOperationTimeForRecipients(); }) + // Start waiting so that we have a more time to calculate a more realistic remaining time + // estimate. + .then([this, anchor = shared_from_this(), delayBetweenQueries] { + return _executor->sleepFor(delayBetweenQueries, _cancelToken) + .then([this, anchor = std::move(anchor)] { + return queryRemainingOperationTimeForRecipients(); + }); + }) .onError([this](Status status) { if (_cancelToken.isCanceled()) { // Do not retry on cancellation errors. @@ -233,12 +242,10 @@ ExecutorFuture<void> CoordinatorCommitMonitor::_makeFuture() const { // The following ensures that the monitor would never sleep for more than a predefined // maximum delay between querying recipient shards. Thus, it can handle very large, // and potentially inaccurate estimates of the remaining operation time. - auto sleepTime = std::min(remainingTimes.max - _threshold, _maxDelayBetweenQueries); - return _executor->sleepFor(sleepTime, _cancelToken) - .then([this, anchor = std::move(anchor)] { - // We are not canceled yet, so schedule new queries against recipient shards. - return _makeFuture(); - }); + auto delayBetweenQueries = + std::min(remainingTimes.max - _threshold, _maxDelayBetweenQueries); + + return _makeFuture(delayBetweenQueries); }); } diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h index 64544981ae5..be722fb11bd 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h +++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h @@ -72,6 +72,7 @@ public: std::vector<ShardId> recipientShards, TaskExecutorPtr executor, CancellationToken cancelToken, + int delayBeforeInitialQueryMillis, Milliseconds maxDelayBetweenQueries = kMaxDelayBetweenQueries); SemiFuture<void> waitUntilRecipientsAreWithinCommitThreshold() const; @@ -88,7 +89,7 @@ public: RemainingOperationTimes queryRemainingOperationTimeForRecipients() const; private: - ExecutorFuture<void> _makeFuture() const; + ExecutorFuture<void> _makeFuture(Milliseconds delayBetweenQueries) const; static constexpr auto kDiagnosticLogLevel = 0; static constexpr auto kMaxDelayBetweenQueries = Seconds(30); @@ -98,6 +99,8 @@ private: const TaskExecutorPtr _executor; const CancellationToken _cancelToken; const Milliseconds _threshold; + + const Milliseconds _delayBeforeInitialQueryMillis; const Milliseconds _maxDelayBetweenQueries; TaskExecutorPtr _networkExecutor; diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp index 2fe3075f1fc..0804565201c 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp @@ -152,7 +152,7 @@ void CoordinatorCommitMonitorTest::setUp() { _cancellationSource = std::make_unique<CancellationSource>(); _commitMonitor = std::make_shared<CoordinatorCommitMonitor>( - _ns, _recipientShards, _futureExecutor, _cancellationSource->token(), Milliseconds(0)); + _ns, _recipientShards, _futureExecutor, _cancellationSource->token(), 0, Milliseconds(0)); _commitMonitor->setNetworkExecutorForTest(executor()); } diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp index ea3451f1c4a..988360d6799 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp @@ -98,6 +98,7 @@ MONGO_FAIL_POINT_DEFINE(pauseBeforeInsertCoordinatorDoc); MONGO_FAIL_POINT_DEFINE(pauseBeforeCTHolderInitialization); const std::string kReshardingCoordinatorActiveIndexName = "ReshardingCoordinatorActiveIndex"; +const int kReshardingNumInitialChunksDefault = 90; const Backoff kExponentialBackoff(Seconds(1), Milliseconds::max()); const WriteConcernOptions kMajorityWriteConcern{ WriteConcernOptions::kMajority, WriteConcernOptions::SyncMode::UNSET, Seconds(0)}; @@ -879,7 +880,7 @@ ReshardingCoordinatorExternalStateImpl::calculateParticipantShardsAndChunks( } else { int numInitialChunks = coordinatorDoc.getNumInitialChunks() ? *coordinatorDoc.getNumInitialChunks() - : cm.numChunks(); + : kReshardingNumInitialChunksDefault; ShardKeyPattern shardKey(coordinatorDoc.getReshardingKey()); const auto tempNs = coordinatorDoc.getTempReshardingNss(); @@ -1714,7 +1715,8 @@ void ReshardingCoordinatorService::ReshardingCoordinator::_startCommitMonitor( _coordinatorDoc.getSourceNss(), extractShardIdsFromParticipantEntries(_coordinatorDoc.getRecipientShards()), **executor, - _ctHolder->getCommitMonitorToken()); + _ctHolder->getCommitMonitorToken(), + resharding::gReshardingDelayBeforeRemainingOperationTimeQueryMillis.load()); _commitMonitorQuiesced = _commitMonitor->waitUntilRecipientsAreWithinCommitThreshold() .thenRunOn(**executor) diff --git a/src/mongo/db/s/resharding/resharding_data_copy_util.cpp b/src/mongo/db/s/resharding/resharding_data_copy_util.cpp index d0b27f00c3c..8635e389cf4 100644 --- a/src/mongo/db/s/resharding/resharding_data_copy_util.cpp +++ b/src/mongo/db/s/resharding/resharding_data_copy_util.cpp @@ -282,7 +282,8 @@ void updateSessionRecord(OperationContext* opCtx, BSONObj o2Field, std::vector<StmtId> stmtIds, boost::optional<repl::OpTime> preImageOpTime, - boost::optional<repl::OpTime> postImageOpTime) { + boost::optional<repl::OpTime> postImageOpTime, + NamespaceString sourceNss) { invariant(opCtx->getLogicalSessionId()); invariant(opCtx->getTxnNumber()); @@ -296,7 +297,7 @@ void updateSessionRecord(OperationContext* opCtx, oplogEntry.setOpType(repl::OpTypeEnum::kNoop); oplogEntry.setObject(SessionCatalogMigration::kSessionOplogTag); oplogEntry.setObject2(std::move(o2Field)); - oplogEntry.setNss({}); + oplogEntry.setNss(std::move(sourceNss)); oplogEntry.setSessionId(sessionId); oplogEntry.setTxnNumber(txnNumber); oplogEntry.setStatementIds(stmtIds); diff --git a/src/mongo/db/s/resharding/resharding_data_copy_util.h b/src/mongo/db/s/resharding/resharding_data_copy_util.h index b51cfc250a5..af24152aaf2 100644 --- a/src/mongo/db/s/resharding/resharding_data_copy_util.h +++ b/src/mongo/db/s/resharding/resharding_data_copy_util.h @@ -144,7 +144,8 @@ void updateSessionRecord(OperationContext* opCtx, BSONObj o2Field, std::vector<StmtId> stmtIds, boost::optional<repl::OpTime> preImageOpTime, - boost::optional<repl::OpTime> postImageOpTime); + boost::optional<repl::OpTime> postImageOpTime, + NamespaceString sourceNss); /** * Calls and returns the value from the supplied lambda function. diff --git a/src/mongo/db/s/resharding/resharding_donor_service.cpp b/src/mongo/db/s/resharding/resharding_donor_service.cpp index 4594c4018a6..2e75751ba01 100644 --- a/src/mongo/db/s/resharding/resharding_donor_service.cpp +++ b/src/mongo/db/s/resharding/resharding_donor_service.cpp @@ -47,6 +47,7 @@ #include "mongo/db/persistent_task_store.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/wait_for_majority_service.h" +#include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/recoverable_critical_section_service.h" #include "mongo/db/s/resharding/resharding_change_event_o2_field_gen.h" #include "mongo/db/s/resharding/resharding_data_copy_util.h" @@ -179,8 +180,9 @@ public: } } - void clearFilteringMetadata(OperationContext* opCtx) { - resharding::clearFilteringMetadata(opCtx, true /* scheduleAsyncRefresh */); + void refreshCollectionPlacementInfo(OperationContext* opCtx, + const NamespaceString& sourceNss) override { + onShardVersionMismatch(opCtx, sourceNss, boost::none); } }; @@ -369,8 +371,15 @@ ExecutorFuture<void> ReshardingDonorService::DonorStateMachine::_finishReshardin { auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); - - _externalState->clearFilteringMetadata(opCtx.get()); + std::initializer_list<NamespaceString> namespacesToRefresh{ + _metadata.getSourceNss(), _metadata.getTempReshardingNss()}; + + // Clear filtering metadata for the source and temp resharding nss. + for (const auto& nss : namespacesToRefresh) { + AutoGetCollection autoColl(opCtx.get(), nss, MODE_IX); + CollectionShardingRuntime::get(opCtx.get(), nss) + ->clearFilteringMetadata(opCtx.get()); + } RecoverableCriticalSectionService::get(opCtx.get()) ->releaseRecoverableCriticalSection( @@ -380,6 +389,13 @@ ExecutorFuture<void> ReshardingDonorService::DonorStateMachine::_finishReshardin ShardingCatalogClient::kLocalWriteConcern); _metrics()->leaveCriticalSection(getCurrentTime()); + + // We force a refresh to make sure that the placement information is updated in + // cache after abort decision before the donor state document is deleted. + for (const auto& nss : namespacesToRefresh) { + _externalState->refreshCollectionPlacementInfo(opCtx.get(), nss); + _externalState->waitForCollectionFlush(opCtx.get(), nss); + } } auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); diff --git a/src/mongo/db/s/resharding/resharding_donor_service.h b/src/mongo/db/s/resharding/resharding_donor_service.h index b50c88b6af5..7b5331f93ac 100644 --- a/src/mongo/db/s/resharding/resharding_donor_service.h +++ b/src/mongo/db/s/resharding/resharding_donor_service.h @@ -298,7 +298,8 @@ public: const BSONObj& query, const BSONObj& update) = 0; - virtual void clearFilteringMetadata(OperationContext* opCtx) = 0; + virtual void refreshCollectionPlacementInfo(OperationContext* opCtx, + const NamespaceString& sourceNss) = 0; }; } // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp index cb358c3f508..16e1507be1c 100644 --- a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp +++ b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp @@ -84,7 +84,8 @@ public: const BSONObj& query, const BSONObj& update) override {} - void clearFilteringMetadata(OperationContext* opCtx) override {} + void refreshCollectionPlacementInfo(OperationContext* opCtx, + const NamespaceString& sourceNss) override {} }; class DonorOpObserverForTest : public OpObserverForTest { diff --git a/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp b/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp index 80338642587..95c29d0825c 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp @@ -108,6 +108,7 @@ boost::optional<SharedSemiFuture<void>> ReshardingOplogSessionApplication::tryAp invariant(op.getTxnNumber()); invariant(op.get_id()); + auto sourceNss = op.getNss(); auto lsid = *op.getSessionId(); if (isInternalSessionForNonRetryableWrite(lsid)) { // Skip internal sessions for non-retryable writes since they only support transactions @@ -157,7 +158,8 @@ boost::optional<SharedSemiFuture<void>> ReshardingOplogSessionApplication::tryAp std::move(o2Field), std::move(stmtIds), std::move(preImageOpTime), - std::move(postImageOpTime)); + std::move(postImageOpTime), + std::move(sourceNss)); }); } diff --git a/src/mongo/db/s/resharding/resharding_server_parameters.idl b/src/mongo/db/s/resharding/resharding_server_parameters.idl index daaedfc9ea4..7508d30c772 100644 --- a/src/mongo/db/s/resharding/resharding_server_parameters.idl +++ b/src/mongo/db/s/resharding/resharding_server_parameters.idl @@ -151,6 +151,21 @@ server_parameters: validator: gte: 0 + reshardingDelayBeforeRemainingOperationTimeQueryMillis: + description: >- + Initial delay before querying for remaining operation time from recipient shards. + The delay allows for applying more oplog entries before calculating time remaining, giving + a more accurate value. + Note we will have this delay every time we happen to have a failover occur. + set_at: [startup, runtime] + cpp_vartype: AtomicWord<int> + cpp_varname: gReshardingDelayBeforeRemainingOperationTimeQueryMillis + default: + expr: 0 + validator: + gte: 0 + redact: false + reshardingCriticalSectionTimeoutMillis: description: >- The upper limit on how long to wait to hear back from recipient shards reaching strict diff --git a/src/mongo/db/s/resharding/resharding_txn_cloner.cpp b/src/mongo/db/s/resharding/resharding_txn_cloner.cpp index 764a32d68d5..96a08c210fa 100644 --- a/src/mongo/db/s/resharding/resharding_txn_cloner.cpp +++ b/src/mongo/db/s/resharding/resharding_txn_cloner.cpp @@ -190,7 +190,8 @@ boost::optional<SharedSemiFuture<void>> ReshardingTxnCloner::doOneRecord( TransactionParticipant::kDeadEndSentinel, {kIncompleteHistoryStmtId}, boost::none /* preImageOpTime */, - boost::none /* postImageOpTime */); + boost::none /* postImageOpTime */, + {}); }); } diff --git a/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp b/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp index 3a10e01306c..b77db34ef11 100644 --- a/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp +++ b/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp @@ -40,6 +40,7 @@ #include "mongo/db/logical_session_cache_noop.h" #include "mongo/db/persistent_task_store.h" #include "mongo/db/pipeline/process_interface/shardsvr_process_interface.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/db/repl/storage_interface_impl.h" #include "mongo/db/repl/wait_for_majority_service.h" #include "mongo/db/s/resharding/resharding_server_parameters_gen.h" diff --git a/src/mongo/db/s/set_allow_migrations_coordinator.cpp b/src/mongo/db/s/set_allow_migrations_coordinator.cpp index 1cf2edac166..aee3052a8ee 100644 --- a/src/mongo/db/s/set_allow_migrations_coordinator.cpp +++ b/src/mongo/db/s/set_allow_migrations_coordinator.cpp @@ -122,12 +122,12 @@ ExecutorFuture<void> SetAllowMigrationsCoordinator::_runImpl( return updateOp; }()); - updateRequest.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); - - auto response = configShard->runBatchWriteCommand(opCtx, - Shard::kDefaultConfigCommandTimeout, - updateRequest, - Shard::RetryPolicy::kIdempotent); + auto response = + configShard->runBatchWriteCommand(opCtx, + Shard::kDefaultConfigCommandTimeout, + updateRequest, + ShardingCatalogClient::kMajorityWriteConcern, + Shard::RetryPolicy::kIdempotent); uassertStatusOK(response.toStatus()); }) diff --git a/src/mongo/db/s/shard_key_util.cpp b/src/mongo/db/s/shard_key_util.cpp index 34cd5ea7cd3..ebb4497260a 100644 --- a/src/mongo/db/s/shard_key_util.cpp +++ b/src/mongo/db/s/shard_key_util.cpp @@ -292,7 +292,15 @@ void ValidationBehaviorsShardCollection::verifyUsefulNonMultiKeyIndex( "admin", BSON(kCheckShardingIndexCmdName << nss.ns() << kKeyPatternField << proposedKey), res); - uassert(ErrorCodes::InvalidOptions, res["errmsg"].str(), success); + + // checkShardingIndex may return UnknownError if a compatible shard key index cannot be + // found when the command is executed on a node with an old binary. In this case, we should + // return InvalidOptions to correspond with the shardCollection behavior. + const auto status = getStatusFromCommandResult(res); + if (status == ErrorCodes::UnknownError) { + uassert(ErrorCodes::InvalidOptions, res["errmsg"].str(), success); + } + uassertStatusOK(status); } void ValidationBehaviorsShardCollection::verifyCanCreateShardKeyIndex(const NamespaceString& nss, @@ -342,7 +350,7 @@ std::vector<BSONObj> ValidationBehaviorsRefineShardKey::loadIndexes( void ValidationBehaviorsRefineShardKey::verifyUsefulNonMultiKeyIndex( const NamespaceString& nss, const BSONObj& proposedKey) const { - auto checkShardingIndexRes = uassertStatusOK(_indexShard->runCommand( + auto res = uassertStatusOK(_indexShard->runCommand( _opCtx, ReadPreferenceSetting(ReadPreference::PrimaryOnly), "admin", @@ -350,13 +358,14 @@ void ValidationBehaviorsRefineShardKey::verifyUsefulNonMultiKeyIndex( BSON(kCheckShardingIndexCmdName << nss.ns() << kKeyPatternField << proposedKey), _cm.getVersion(_indexShard->getId())), Shard::RetryPolicy::kIdempotent)); - if (checkShardingIndexRes.commandStatus == ErrorCodes::UnknownError) { - // CheckShardingIndex returns UnknownError if a compatible shard key index cannot be found, - // but we return InvalidOptions to correspond with the shardCollection behavior. - uasserted(ErrorCodes::InvalidOptions, checkShardingIndexRes.response["errmsg"].str()); + + // checkShardingIndex may return UnknownError if a compatible shard key index cannot be + // found when the command is executed on a node with an old binary. In this case, we should + // return InvalidOptions to correspond with the shardCollection behavior. + if (res.commandStatus == ErrorCodes::UnknownError) { + uasserted(ErrorCodes::InvalidOptions, res.response["errmsg"].str()); } - // Rethrow any other error to allow retries on retryable errors. - uassertStatusOK(checkShardingIndexRes.commandStatus); + uassertStatusOK(Shard::CommandResponse::getEffectiveStatus(res)); } void ValidationBehaviorsRefineShardKey::verifyCanCreateShardKeyIndex(const NamespaceString& nss, diff --git a/src/mongo/db/s/shard_local.cpp b/src/mongo/db/s/shard_local.cpp index fec64ebd8f2..229a61e8480 100644 --- a/src/mongo/db/s/shard_local.cpp +++ b/src/mongo/db/s/shard_local.cpp @@ -227,4 +227,24 @@ Status ShardLocal::runAggregation( return _rsLocalClient.runAggregation(opCtx, aggRequest, callback); } +BatchedCommandResponse ShardLocal::runBatchWriteCommand(OperationContext* opCtx, + const Milliseconds maxTimeMS, + const BatchedCommandRequest& batchRequest, + const WriteConcernOptions& writeConcern, + RetryPolicy retryPolicy) { + // A request dispatched through a local client is served within the same thread that submits it + // (so that the opCtx needs to be used as the vehicle to pass the WC to the ServiceEntryPoint). + const auto originalWC = opCtx->getWriteConcern(); + ScopeGuard resetWCGuard([&] { opCtx->setWriteConcern(originalWC); }); + opCtx->setWriteConcern(writeConcern); + + const auto dbName = batchRequest.getNS().db(); + const BSONObj cmdObj = [&] { + BSONObjBuilder cmdObjBuilder; + batchRequest.serialize(&cmdObjBuilder); + return cmdObjBuilder.obj(); + }(); + + return _submitBatchWriteCommand(opCtx, cmdObj, dbName, maxTimeMS, retryPolicy); +} } // namespace mongo diff --git a/src/mongo/db/s/shard_local.h b/src/mongo/db/s/shard_local.h index e7c6e74b000..f82e3f7bb96 100644 --- a/src/mongo/db/s/shard_local.h +++ b/src/mongo/db/s/shard_local.h @@ -81,6 +81,12 @@ public: std::function<bool(const std::vector<BSONObj>& batch, const boost::optional<BSONObj>& postBatchResumeToken)> callback); + BatchedCommandResponse runBatchWriteCommand(OperationContext* opCtx, + Milliseconds maxTimeMS, + const BatchedCommandRequest& batchRequest, + const WriteConcernOptions& writeConcern, + RetryPolicy retryPolicy) final; + private: StatusWith<Shard::CommandResponse> _runCommand(OperationContext* opCtx, const ReadPreferenceSetting& unused, diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp index a374159bc87..91aa5b9564b 100644 --- a/src/mongo/db/s/shard_server_op_observer.cpp +++ b/src/mongo/db/s/shard_server_op_observer.cpp @@ -587,9 +587,16 @@ void ShardServerOpObserver::onCreateCollection(OperationContext* opCtx, const BSONObj& idIndex, const OplogSlot& createOpTime, bool fromMigrate) { - // Only the shard primay nodes control the collection creation and secondaries just follow - // Secondaries CSR will be the defaulted one (UNKNOWN in most of the cases) + // Only the shard primary nodes control the collection creation. if (!opCtx->writesAreReplicated()) { + // On secondaries node of sharded cluster we force the cleanup of the filtering metadata in + // order to remove anything that was left from any previous collection instance. This could + // happen by first having an UNSHARDED version for a collection that didn't exist followed + // by a movePrimary to the current shard. + if (ShardingState::get(opCtx)->enabled()) { + CollectionShardingRuntime::get(opCtx, collectionName)->clearFilteringMetadata(opCtx); + } + return; } diff --git a/src/mongo/db/s/sharding_ddl_util.cpp b/src/mongo/db/s/sharding_ddl_util.cpp index 14c18e46923..ba13de4abbd 100644 --- a/src/mongo/db/s/sharding_ddl_util.cpp +++ b/src/mongo/db/s/sharding_ddl_util.cpp @@ -77,11 +77,14 @@ void updateTags(OperationContext* opCtx, }()}); return updateOp; }()); - request.setWriteConcern(writeConcern.toBSON()); auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto response = configShard->runBatchWriteCommand( - opCtx, Milliseconds::max(), request, Shard::RetryPolicy::kIdempotentOrCursorInvalidated); + auto response = + configShard->runBatchWriteCommand(opCtx, + Milliseconds::max(), + request, + writeConcern, + Shard::RetryPolicy::kIdempotentOrCursorInvalidated); uassertStatusOK(response.toStatus()); } @@ -105,11 +108,13 @@ void deleteChunks(OperationContext* opCtx, return deleteOp; }()); - request.setWriteConcern(writeConcern.toBSON()); - auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto response = configShard->runBatchWriteCommand( - opCtx, Milliseconds::max(), request, Shard::RetryPolicy::kIdempotentOrCursorInvalidated); + auto response = + configShard->runBatchWriteCommand(opCtx, + Milliseconds::max(), + request, + writeConcern, + Shard::RetryPolicy::kIdempotentOrCursorInvalidated); uassertStatusOK(response.toStatus()); } @@ -336,11 +341,13 @@ void removeTagsMetadataFromConfig_notIdempotent(OperationContext* opCtx, return deleteOp; }()); - request.setWriteConcern(writeConcern.toBSON()); - auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto response = configShard->runBatchWriteCommand( - opCtx, Milliseconds::max(), request, Shard::RetryPolicy::kIdempotentOrCursorInvalidated); + auto response = + configShard->runBatchWriteCommand(opCtx, + Milliseconds::max(), + request, + writeConcern, + Shard::RetryPolicy::kIdempotentOrCursorInvalidated); uassertStatusOK(response.toStatus()); } diff --git a/src/mongo/db/server_options_helpers.cpp b/src/mongo/db/server_options_helpers.cpp index 9ed7af1f62a..a3dc5ec59df 100644 --- a/src/mongo/db/server_options_helpers.cpp +++ b/src/mongo/db/server_options_helpers.cpp @@ -134,42 +134,43 @@ Status validateBaseOptions(const moe::Environment& params) { } } + std::map<std::string, std::string> parameters; if (params.count("setParameter")) { - const auto parameters = params["setParameter"].as<std::map<std::string, std::string>>(); - - const bool enableTestCommandsValue = ([¶meters] { - const auto etc = parameters.find("enableTestCommands"); - if (etc == parameters.end()) { - return false; - } - const auto& val = etc->second; - return (0 == val.compare("1")) || (0 == val.compare("true")); - })(); + parameters = params["setParameter"].as<std::map<std::string, std::string>>(); + } - if (enableTestCommandsValue) { - // Only register failpoint server parameters if enableTestCommands=1. - globalFailPointRegistry().registerAllFailPointsAsServerParameters(); - } else { - // Deregister test-only parameters. - ServerParameterSet::getNodeParameterSet()->disableTestParameters(); - ServerParameterSet::getClusterParameterSet()->disableTestParameters(); + const bool enableTestCommandsValue = ([¶meters] { + const auto etc = parameters.find("enableTestCommands"); + if (etc == parameters.end()) { + return false; } + const auto& val = etc->second; + return (0 == val.compare("1")) || (0 == val.compare("true")); + })(); + + if (enableTestCommandsValue) { + // Only register failpoint server parameters if enableTestCommands=1. + globalFailPointRegistry().registerAllFailPointsAsServerParameters(); + } else { + // Deregister test-only parameters. + ServerParameterSet::getNodeParameterSet()->disableTestParameters(); + ServerParameterSet::getClusterParameterSet()->disableTestParameters(); + } - // Must come after registerAllFailPointsAsServerParameters() above. - auto* paramSet = ServerParameterSet::getNodeParameterSet(); - for (const auto& setParam : parameters) { - auto* param = paramSet->getIfExists(setParam.first); + // Must come after registerAllFailPointsAsServerParameters() above. + auto* paramSet = ServerParameterSet::getNodeParameterSet(); + for (const auto& setParam : parameters) { + auto* param = paramSet->getIfExists(setParam.first); - if (!param) { - return {ErrorCodes::BadValue, - str::stream() << "Unknown --setParameter '" << setParam.first << "'"}; - } + if (!param) { + return {ErrorCodes::BadValue, + str::stream() << "Unknown --setParameter '" << setParam.first << "'"}; + } - if (!param->isEnabled()) { - return {ErrorCodes::BadValue, - str::stream() << "--setParameter '" << setParam.first - << "' only available when used with 'enableTestCommands'"}; - } + if (!param->isEnabled()) { + return {ErrorCodes::BadValue, + str::stream() << "--setParameter '" << setParam.first + << "' only available when used with 'enableTestCommands'"}; } } diff --git a/src/mongo/db/service_context.cpp b/src/mongo/db/service_context.cpp index 0de0541aa88..9051510e470 100644 --- a/src/mongo/db/service_context.cpp +++ b/src/mongo/db/service_context.cpp @@ -254,7 +254,10 @@ ServiceContext::UniqueOperationContext ServiceContext::makeOperationContext(Clie onCreate(opCtx.get(), _clientObservers); ScopeGuard onCreateGuard([&] { onDestroy(opCtx.get(), _clientObservers); }); - invariant(opCtx->lockState(), ProcessInfo().getProcessName()); + invariant( + opCtx->lockState(), + str::stream() << "No lock state configured. This could be a missing build dependency. " + << ProcessInfo().getProcessName()); if (!opCtx->recoveryUnit()) { opCtx->setRecoveryUnit(std::make_unique<RecoveryUnitNoop>(), diff --git a/src/mongo/db/sessions_collection.cpp b/src/mongo/db/sessions_collection.cpp index a5595f21dbf..6deccfe9ff3 100644 --- a/src/mongo/db/sessions_collection.cpp +++ b/src/mongo/db/sessions_collection.cpp @@ -42,26 +42,13 @@ #include "mongo/db/logical_session_id_helpers.h" #include "mongo/db/ops/write_ops.h" #include "mongo/db/repl/read_concern_args.h" +#include "mongo/db/sessions_server_parameters_gen.h" #include "mongo/rpc/get_status_from_command_result.h" +#include "mongo/util/duration.h" namespace mongo { namespace { -// This batch size is chosen to ensure that we don't form requests larger than the 16mb limit. -// Especially for refreshes, the updates we send include the full user name (user@db), and user -// names can be quite large (we enforce a max 10k limit for usernames used with sessions). -// -// At 1000 elements, a 16mb payload gives us a budget of 16000 bytes per user, which we should -// comfortably be able to stay under, even with 10k user names. -constexpr size_t kMaxBatchSize = 1000; - -// Used to refresh or remove items from the session collection with write -// concern majority -const WriteConcernOptions kMajorityWriteConcern{WriteConcernOptions::kMajority, - WriteConcernOptions::SyncMode::UNSET, - WriteConcernOptions::kWriteConcernTimeoutSystem}; - - BSONObj lsidQuery(const LogicalSessionId& lsid) { return BSON(LogicalSessionRecord::kIdFieldName << lsid.toBSON()); } @@ -104,7 +91,7 @@ void runBulkGeneric(TFactory makeT, AddLineFn addLine, SendFn sendBatch, const C for (const auto& item : items) { addLine(*thing, item); - if (++i >= kMaxBatchSize) { + if (++i >= std::size_t(mongo::gSessionMaxBatchSize.load())) { sendLocalBatch(); setupBatch(); @@ -192,7 +179,14 @@ SessionsCollection::FindBatchFn SessionsCollection::makeFindFnForCommand(const N void SessionsCollection::_doRefresh(const NamespaceString& ns, const std::vector<LogicalSessionRecord>& sessions, SendBatchFn send) { - auto init = [ns](BSONObjBuilder* batch) { + // Used to refresh items from the session collection with write + // concern majority + const WriteConcernOptions kMajorityWriteConcern{ + WriteConcernOptions::kMajority, + WriteConcernOptions::SyncMode::UNSET, + Milliseconds(mongo::gSessionWriteConcernTimeoutSystemMillis.load())}; + + auto init = [ns, kMajorityWriteConcern](BSONObjBuilder* batch) { batch->append("update", ns.coll()); batch->append("ordered", false); batch->append(WriteConcernOptions::kWriteConcernField, kMajorityWriteConcern.toBSON()); @@ -202,14 +196,20 @@ void SessionsCollection::_doRefresh(const NamespaceString& ns, entries->append( BSON("q" << lsidQuery(record) << "u" << updateQuery(record) << "upsert" << true)); }; - runBulkCmd("updates", init, add, send, sessions); } void SessionsCollection::_doRemove(const NamespaceString& ns, const std::vector<LogicalSessionId>& sessions, SendBatchFn send) { - auto init = [ns](BSONObjBuilder* batch) { + // Used to remove items from the session collection with write + // concern majority + const WriteConcernOptions kMajorityWriteConcern{ + WriteConcernOptions::kMajority, + WriteConcernOptions::SyncMode::UNSET, + Milliseconds(mongo::gSessionWriteConcernTimeoutSystemMillis.load())}; + + auto init = [ns, kMajorityWriteConcern](BSONObjBuilder* batch) { batch->append("delete", ns.coll()); batch->append("ordered", false); batch->append(WriteConcernOptions::kWriteConcernField, kMajorityWriteConcern.toBSON()); diff --git a/src/mongo/db/sessions_server_parameters.idl b/src/mongo/db/sessions_server_parameters.idl new file mode 100644 index 00000000000..c8676b03073 --- /dev/null +++ b/src/mongo/db/sessions_server_parameters.idl @@ -0,0 +1,63 @@ +# Copyright (C) 2024-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +# Server parameters for configuring the refresh of the session colelction. + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/idl/basic_types.idl" + +server_parameters: + sessionWriteConcernTimeoutSystemMillis: + description: Controls the write concern timeout (in milliseconds) for the refresh or removal of items from the session collection. + set_at: [startup, runtime] + cpp_vartype: AtomicWord<int> + cpp_varname: gSessionWriteConcernTimeoutSystemMillis + default: 60000 + validator: + gte: 0 + redact: false + + sessionMaxBatchSize: + description: >- + Controls the maximum batch size (number of elements) for the sessions' refresh. + This batch size is chosen to ensure that we don't form requests larger than the 16mb limit. + Especially for refreshes, the updates we send include the full user name (user@db), and user + names can be quite large (we enforce a max 10k limit for usernames used with sessions). + At a default of 1000 elements, a 16mb payload gives us a budget of 16000 bytes per user, which we should + comfortably be able to stay under, even with 10k user names. so we do not form requests larger than the 16mb limit. + set_at: [startup, runtime] + cpp_vartype: AtomicWord<int> + cpp_varname: gSessionMaxBatchSize + default: 1000 + validator: + gte: 100 + lte: 10000 + redact: false diff --git a/src/mongo/db/sorter/sorter.cpp b/src/mongo/db/sorter/sorter.cpp index 93ba45e4d74..ce460b0fc89 100644 --- a/src/mongo/db/sorter/sorter.cpp +++ b/src/mongo/db/sorter/sorter.cpp @@ -64,6 +64,7 @@ #include "mongo/s/is_mongos.h" #include "mongo/util/assert_util.h" #include "mongo/util/destructor_guard.h" +#include "mongo/util/file.h" #include "mongo/util/str.h" // As this file is included in various places we need to handle the case of having the log header @@ -1191,6 +1192,17 @@ Sorter<Key, Value>::File::~File() { } if (_keep) { + if (!_file.is_open()) { + return; + } + DESTRUCTOR_GUARD(_file.flush()); + + mongo::File fileForFsync; + fileForFsync.open(_path.string().c_str()); + if (fileForFsync.is_open()) { + fileForFsync.fsync(); + } + return; } diff --git a/src/mongo/db/sorter/sorter.h b/src/mongo/db/sorter/sorter.h index cec8174bc80..059d7b72f46 100644 --- a/src/mongo/db/sorter/sorter.h +++ b/src/mongo/db/sorter/sorter.h @@ -42,6 +42,7 @@ #include "mongo/bson/util/builder.h" #include "mongo/db/exec/document_value/document.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/db/sorter/sorter_gen.h" #include "mongo/db/sorter/sorter_stats.h" #include "mongo/platform/atomic_word.h" @@ -461,7 +462,7 @@ public: virtual std::pair<Key, Value> next() = 0; // Serialize the bound for explain output - virtual Document serializeBound() const = 0; + virtual Document serializeBound(const SerializationOptions& opts) const = 0; virtual size_t totalDataSizeBytes() const = 0; virtual size_t limit() const = 0; @@ -541,8 +542,8 @@ public: std::pair<Key, Value> next(); // Serialize the bound for explain output - Document serializeBound() const { - return {makeBound.serialize()}; + Document serializeBound(const SerializationOptions& opts) const { + return {makeBound.serialize(opts)}; }; size_t totalDataSizeBytes() const { diff --git a/src/mongo/db/sorter/sorter_test.cpp b/src/mongo/db/sorter/sorter_test.cpp index 531a217a41e..328dcd256c1 100644 --- a/src/mongo/db/sorter/sorter_test.cpp +++ b/src/mongo/db/sorter/sorter_test.cpp @@ -1037,7 +1037,7 @@ public: Key operator()(Key k, const Doc&) const { return k - 10; } - Document serialize() const { + Document serialize(const SerializationOptions& opts = {}) const { MONGO_UNREACHABLE; } }; @@ -1045,7 +1045,7 @@ public: Key operator()(Key k, const Doc&) const { return k + 10; } - Document serialize() const { + Document serialize(const SerializationOptions& opts = {}) const { MONGO_UNREACHABLE; } }; diff --git a/src/mongo/db/stats/SConscript b/src/mongo/db/stats/SConscript index f913dc6abf2..dfaf4a6a533 100644 --- a/src/mongo/db/stats/SConscript +++ b/src/mongo/db/stats/SConscript @@ -22,6 +22,7 @@ env.Library( 'operation_latency_histogram.cpp' ], LIBDEPS=[ + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/service_context', ], diff --git a/src/mongo/db/stats/counters.cpp b/src/mongo/db/stats/counters.cpp index e344141052d..5f571640b5d 100644 --- a/src/mongo/db/stats/counters.cpp +++ b/src/mongo/db/stats/counters.cpp @@ -351,4 +351,16 @@ Counter64 deleteManyCount; ServerStatusMetricField<Counter64> displayDeleteManyCount("query.deleteManyCount", &deleteManyCount); +Counter64 internalRetryableWriteCount; +ServerStatusMetricField<Counter64> displayInternalRetryableWriteCount( + "query.internalRetryableWriteCount", &internalRetryableWriteCount); + +Counter64 externalRetryableWriteCount; +ServerStatusMetricField<Counter64> displayExternalRetryableWriteCount( + "query.externalRetryableWriteCount", &externalRetryableWriteCount); + +Counter64 retryableInternalTransactionCount; +ServerStatusMetricField<Counter64> displayRetryableInternalTransactionCount( + "query.retryableInternalTransactionCount", &retryableInternalTransactionCount); + } // namespace mongo diff --git a/src/mongo/db/stats/counters.h b/src/mongo/db/stats/counters.h index e41de1e31b0..d314ec49302 100644 --- a/src/mongo/db/stats/counters.h +++ b/src/mongo/db/stats/counters.h @@ -503,5 +503,11 @@ extern OperatorCounters operatorCountersWindowAccumulatorExpressions; extern Counter64 updateManyCount; // Track the number of deleteMany calls. extern Counter64 deleteManyCount; +// Track the number of internal retryable writes +extern Counter64 internalRetryableWriteCount; +// Track the number of external retryable writes +extern Counter64 externalRetryableWriteCount; +// Track the number of internal transactions for retryable writes +extern Counter64 retryableInternalTransactionCount; } // namespace mongo diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index 144cfa386bb..616e7c09185 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -90,6 +90,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', ], ) @@ -389,6 +390,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/bson/dotted_path_support', '$BUILD_DIR/mongo/db/server_options_core', ], @@ -486,6 +488,7 @@ env.Library( env.CppUnitTest( target='db_storage_test', source=[ + 'backup_block_test.cpp', 'flow_control_test.cpp', 'historical_ident_tracker_test.cpp', 'index_entry_comparison_test.cpp', @@ -521,6 +524,7 @@ env.CppUnitTest( '$BUILD_DIR/mongo/executor/network_interface_factory', '$BUILD_DIR/mongo/executor/network_interface_mock', '$BUILD_DIR/mongo/util/periodic_runner_factory', + 'backup_block', 'flow_control', 'flow_control_parameters', 'historical_ident_tracker', @@ -645,6 +649,7 @@ env.CppLibfuzzerTest( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', 'key_string', ], ) diff --git a/src/mongo/db/storage/backup_block.cpp b/src/mongo/db/storage/backup_block.cpp index 3cc2b0b0cf3..9b8dc29f209 100644 --- a/src/mongo/db/storage/backup_block.cpp +++ b/src/mongo/db/storage/backup_block.cpp @@ -49,22 +49,38 @@ const std::set<std::string> kRequiredMDBFiles = {"_mdb_catalog.wt", "sizeStorer. } // namespace +namespace details { + +std::string extractIdentFromPath(const boost::filesystem::path& dbpath, + const boost::filesystem::path& identAbsolutePath) { + // Remove the dbpath prefix to the identAbsolutePath. + boost::filesystem::path identWithExtension = boost::filesystem::relative( + identAbsolutePath, boost::filesystem::path(storageGlobalParams.dbpath)); + + // Remove the file extension and convert to generic form (i.e. replace "\" with "/" + // on windows, no-op on unix). + return boost::filesystem::change_extension(identWithExtension, "").generic_string(); +} + +} // namespace details + BackupBlock::BackupBlock(OperationContext* opCtx, - std::string filePath, + std::string fileAbsolutePath, const IdentToNamespaceAndUUIDMap& identToNamespaceAndUUIDMap, boost::optional<Timestamp> checkpointTimestamp, std::uint64_t offset, std::uint64_t length, std::uint64_t fileSize) - : _filePath(filePath), _offset(offset), _length(length), _fileSize(fileSize) { - boost::filesystem::path path(filePath); - _filenameStem = path.stem().string(); + : _fileAbsolutePath(fileAbsolutePath), _offset(offset), _length(length), _fileSize(fileSize) { + boost::filesystem::path absolutePath(fileAbsolutePath); + _ident = details::extractIdentFromPath(boost::filesystem::path(storageGlobalParams.dbpath), + absolutePath); _initialize(opCtx, identToNamespaceAndUUIDMap, checkpointTimestamp); } bool BackupBlock::isRequired() const { // Extract the filename from the path. - boost::filesystem::path path(_filePath); + boost::filesystem::path path(_fileAbsolutePath); const std::string filename = path.filename().string(); // Check whether this is a required WiredTiger file. @@ -121,7 +137,7 @@ void BackupBlock::_initialize(OperationContext* opCtx, } // Fetch the latest values for the ident. - auto it = identToNamespaceAndUUIDMap.find(_filenameStem); + auto it = identToNamespaceAndUUIDMap.find(_ident); if (it != identToNamespaceAndUUIDMap.end()) { _uuid = it->second.second; _setNamespaceString(it->second.first); @@ -134,7 +150,7 @@ void BackupBlock::_initialize(OperationContext* opCtx, // Check if the ident had a different value at the checkpoint timestamp. If so, we want to use // that instead as that will be the ident's value when restoring from the backup. boost::optional<std::pair<NamespaceString, UUID>> historicalEntry = - HistoricalIdentTracker::get(opCtx).lookup(_filenameStem, checkpointTimestamp.get()); + HistoricalIdentTracker::get(opCtx).lookup(_ident, checkpointTimestamp.value()); if (historicalEntry) { _uuid = historicalEntry->second; _setNamespaceString(historicalEntry->first); diff --git a/src/mongo/db/storage/backup_block.h b/src/mongo/db/storage/backup_block.h index fd96b8e5e2e..30a640190a6 100644 --- a/src/mongo/db/storage/backup_block.h +++ b/src/mongo/db/storage/backup_block.h @@ -39,6 +39,11 @@ namespace mongo { +namespace details { +std::string extractIdentFromPath(const boost::filesystem::path& dbpath, + const boost::filesystem::path& identAbsolutePath); +} + /** * Represents the file blocks returned by the storage engine during both full and incremental * backups. In the case of a full backup, each block is an entire file with offset=0 and @@ -57,7 +62,7 @@ public: stdx::unordered_map<std::string, std::pair<NamespaceString, UUID>>; explicit BackupBlock(OperationContext* opCtx, - std::string filePath, + std::string fileAbsolutePath, const IdentToNamespaceAndUUIDMap& identToNamespaceAndUUIDMap, boost::optional<Timestamp> checkpointTimestamp, std::uint64_t offset = 0, @@ -67,7 +72,7 @@ public: ~BackupBlock() = default; std::string filePath() const { - return _filePath; + return _fileAbsolutePath; } std::string ns() const { @@ -110,12 +115,12 @@ private: boost::optional<Timestamp> checkpointTimestamp); void _setNamespaceString(const NamespaceString& nss); - const std::string _filePath; + const std::string _fileAbsolutePath; const std::uint64_t _offset; const std::uint64_t _length; const std::uint64_t _fileSize; - std::string _filenameStem; + std::string _ident; NamespaceString _nss; boost::optional<UUID> _uuid; }; diff --git a/src/mongo/db/storage/backup_block_test.cpp b/src/mongo/db/storage/backup_block_test.cpp new file mode 100644 index 00000000000..d98a1e63650 --- /dev/null +++ b/src/mongo/db/storage/backup_block_test.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include <boost/filesystem/path.hpp> + +#include "mongo/db/storage/backup_block.h" +#include "mongo/unittest/unittest.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest + + +namespace mongo { +namespace details { +namespace { + +TEST(BackupBlockTest, ExtractIdentFromPath) { + boost::filesystem::path dbpath = "/data/db"; + boost::filesystem::path identAbsolutePathDefault = + "/data/db/collection-9-11733751379908443489.wt"; + std::string identDefault = "collection-9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath(dbpath, identAbsolutePathDefault), identDefault); + + boost::filesystem::path identAbsolutePathDirectoryPerDb = + "/data/db/test/collection-9-11733751379908443489.wt"; + std::string identDirectoryPerDb = "test/collection-9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath(dbpath, identAbsolutePathDirectoryPerDb), + identDirectoryPerDb); + + boost::filesystem::path identAbsolutePathWiredTigerDirectoryForIndexes = + "/data/db/collection/9-11733751379908443489.wt"; + std::string identWiredTigerDirectoryForIndexes = "collection/9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath(dbpath, identAbsolutePathWiredTigerDirectoryForIndexes), + identWiredTigerDirectoryForIndexes); + + boost::filesystem::path identAbsolutePathDirectoryPerDbAndWiredTigerDirectoryForIndexes = + "/data/db/test/collection/9-11733751379908443489.wt"; + std::string identDirectoryPerDbWiredTigerDirectoryForIndexes = + "test/collection/9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath( + dbpath, identAbsolutePathDirectoryPerDbAndWiredTigerDirectoryForIndexes), + identDirectoryPerDbWiredTigerDirectoryForIndexes); +} + +} // namespace +} // namespace details +} // namespace mongo diff --git a/src/mongo/db/storage/control/storage_control.cpp b/src/mongo/db/storage/control/storage_control.cpp index 11222448eba..3f5af49c729 100644 --- a/src/mongo/db/storage/control/storage_control.cpp +++ b/src/mongo/db/storage/control/storage_control.cpp @@ -79,8 +79,8 @@ void startStorageControls(ServiceContext* serviceContext, bool forTestOnly) { std::unique_ptr<JournalFlusher> journalFlusher = std::make_unique<JournalFlusher>( /*disablePeriodicFlushes*/ forTestOnly || (!storageEngine->isDurable() && !storageEngine->isEphemeral())); - journalFlusher->go(); JournalFlusher::set(serviceContext, std::move(journalFlusher)); + JournalFlusher::get(serviceContext)->go(); } if (!storageEngine->isEphemeral() && !storageGlobalParams.readOnly) { diff --git a/src/mongo/db/storage/kv/SConscript b/src/mongo/db/storage/kv/SConscript index 67f6904300a..8e53d62525f 100644 --- a/src/mongo/db/storage/kv/SConscript +++ b/src/mongo/db/storage/kv/SConscript @@ -11,7 +11,7 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/curop', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/storage/write_unit_of_work', ], ) diff --git a/src/mongo/db/storage/kv/storage_engine_test.cpp b/src/mongo/db/storage/kv/storage_engine_test.cpp index fe85863c7b5..0a7fb957003 100644 --- a/src/mongo/db/storage/kv/storage_engine_test.cpp +++ b/src/mongo/db/storage/kv/storage_engine_test.cpp @@ -649,15 +649,17 @@ TEST_F(TimestampKVEngineTest, TimestampMonitorNotifiesListeners) { _storageEngine->getTimestampMonitor()->addListener(&fourth); // Wait until all 4 listeners get notified at least once. - stdx::unique_lock<Latch> lk(mutex); - cv.wait(lk, [&] { - for (auto const& change : changes) { - if (!change) { - return false; + { + stdx::unique_lock<Latch> lk(mutex); + cv.wait(lk, [&] { + for (auto const& change : changes) { + if (!change) { + return false; + } } - } - return true; - }); + return true; + }); + }; _storageEngine->getTimestampMonitor()->clearListeners(); } diff --git a/src/mongo/db/storage/record_store.h b/src/mongo/db/storage/record_store.h index 1742cddba5e..ccbc8d582b9 100644 --- a/src/mongo/db/storage/record_store.h +++ b/src/mongo/db/storage/record_store.h @@ -477,15 +477,6 @@ public: } /** - * If compact() supports online compaction. - * - * Only called if compactSupported() returns true. - */ - virtual bool supportsOnlineCompaction() const { - MONGO_UNREACHABLE; - } - - /** * Attempt to reduce the storage space used by this RecordStore. * * Only called if compactSupported() returns true. diff --git a/src/mongo/db/storage/recovery_unit.h b/src/mongo/db/storage/recovery_unit.h index eb032f73d4e..7b867906535 100644 --- a/src/mongo/db/storage/recovery_unit.h +++ b/src/mongo/db/storage/recovery_unit.h @@ -77,6 +77,22 @@ enum class PrepareConflictBehavior { }; /** + * DataCorruptionDetectionMode determines how we handle the discovery of evidence of data + * corruption. + */ +enum class DataCorruptionDetectionMode { + /** + * Always throw a DataCorruptionDetected error when evidence of data corruption is detected. + */ + kThrow, + /** + * When evidence of data corruption is decected, log an entry to the health log and the server + * logs, but do not throw an error. Continue attempting to return results. + */ + kLogAndContinue, +}; + +/** * A RecoveryUnit is responsible for ensuring that data is persisted. * All on-disk information must be mutated through this interface. */ @@ -711,6 +727,14 @@ public: return _noEvictionAfterRollback; } + void setDataCorruptionDetectionMode(DataCorruptionDetectionMode mode) { + _dataCorruptionDetectionMode = mode; + } + + DataCorruptionDetectionMode getDataCorruptionDetectionMode() const { + return _dataCorruptionDetectionMode; + } + /** * Returns true if this is an instance of RecoveryUnitNoop. */ @@ -718,6 +742,13 @@ public: return false; } + /** + * Sets a maximum timeout that the storage engine will block an operation when the cache is + * under pressure. + * If not set (default 0) then the storage engine will block indefinitely. + */ + virtual void setCacheMaxWaitTimeout(Milliseconds) {} + protected: RecoveryUnit(); @@ -770,6 +801,8 @@ protected: AbandonSnapshotMode _abandonSnapshotMode = AbandonSnapshotMode::kAbort; + DataCorruptionDetectionMode _dataCorruptionDetectionMode = DataCorruptionDetectionMode::kThrow; + private: // Sets the snapshot associated with this RecoveryUnit to a new globally unique id number. void assignNextSnapshotId(); diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp index 26a7fc9dae0..21f04d4445a 100644 --- a/src/mongo/db/storage/storage_engine_impl.cpp +++ b/src/mongo/db/storage/storage_engine_impl.cpp @@ -41,6 +41,7 @@ #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/multitenancy.h" #include "mongo/db/operation_context.h" @@ -1235,12 +1236,12 @@ void StorageEngineImpl::TimestampMonitor::_startup() { } try { - auto opCtx = client->getOperationContext(); - mongo::ServiceContext::UniqueOperationContext uOpCtx; - if (!opCtx) { - uOpCtx = client->makeOperationContext(); - opCtx = uOpCtx.get(); - } + auto uniqueOpCtx = client->makeOperationContext(); + auto opCtx = uniqueOpCtx.get(); + + // The TimestampMonitor is an important background cleanup task for the storage + // engine and needs to be able to make progress to free up resources. + SkipTicketAcquisitionForLock skipTicketAcquisition(opCtx); Timestamp checkpoint; Timestamp oldest; diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index 8ce4c135cfc..1b7790e61c6 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -53,11 +53,11 @@ wtEnv.Library( '$BUILD_DIR/mongo/db/catalog/collection', '$BUILD_DIR/mongo/db/catalog/collection_options', '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/global_settings', '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/namespace_string', '$BUILD_DIR/mongo/db/prepare_conflict_tracker', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/repl_settings', @@ -80,6 +80,7 @@ wtEnv.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog/database_holder', + '$BUILD_DIR/mongo/db/catalog/health_log_interface', '$BUILD_DIR/mongo/db/commands/server_status', '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/db_raii', diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 09b1603361e..0a498ead248 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -38,12 +38,16 @@ #include <set> #include "mongo/base/checked_cast.h" +#include "mongo/base/string_data.h" +#include "mongo/db/catalog/health_log.h" +#include "mongo/db/catalog/health_log_gen.h" #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/catalog/validate_results.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/global_settings.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/json.h" +#include "mongo/db/namespace_string.h" #include "mongo/db/repl/repl_settings.h" #include "mongo/db/service_context.h" #include "mongo/db/stats/resource_consumption_metrics.h" @@ -59,6 +63,7 @@ #include "mongo/util/assert_util.h" #include "mongo/util/fail_point.h" #include "mongo/util/hex.h" +#include "mongo/util/stacktrace.h" #include "mongo/util/str.h" #include "mongo/util/testing_proctor.h" @@ -84,8 +89,53 @@ namespace { MONGO_FAIL_POINT_DEFINE(WTCompactIndexEBUSY); MONGO_FAIL_POINT_DEFINE(WTIndexPauseAfterSearchNear); MONGO_FAIL_POINT_DEFINE(WTValidateIndexStructuralDamage); +MONGO_FAIL_POINT_DEFINE(WTIndexUassertDuplicateRecordForKeyOnIdUnindex); static const WiredTigerItem emptyItem(nullptr, 0); + +/** + * Add a data corruption entry to the health log. + */ +void addDataCorruptionEntryToHealthLog(OperationContext* opCtx, + const NamespaceString& nss, + StringData operation, + StringData message, + const BSONObj& key, + StringData indexName, + StringData uri) { + HealthLogEntry entry; + entry.setNss(nss); + entry.setTimestamp(Date_t::now()); + entry.setSeverity(SeverityEnum::Error); + entry.setScope(ScopeEnum::Index); + entry.setOperation(operation); + entry.setMsg(message); + + BSONObjBuilder bob; + bob.append("key", key); + bob.append("indexName", indexName); + bob.append("uri", uri); + bob.appendElements(getStackTrace().getBSONRepresentation()); + entry.setData(bob.obj()); + + HealthLog::get(opCtx)->log(entry); +} + +/** + * Returns the logv2::LogOptions controlling the behaviour after logging a data corruption + * error. When the TestingProctor is enabled we will fatally assert. When the testing proctor is + * disabled or when 'forceUassert' is specified (for instance because a failpoint is enabled), + * we should log and throw DataCorruptionDetected. + */ +logv2::LogOptions getLogOptionsForDataCorruption(RecoveryUnit& ru, bool forceUassert = false) { + if (ru.getDataCorruptionDetectionMode() == DataCorruptionDetectionMode::kThrow || + MONGO_unlikely(forceUassert)) { + return logv2::LogOptions{logv2::UserAssertAfterLog(ErrorCodes::DataCorruptionDetected)}; + } else { + return logv2::LogOptions(logv2::LogComponent::kAutomaticDetermination); + } +} + } // namespace void WiredTigerIndex::setKey(WT_CURSOR* cursor, const WT_ITEM* item) { @@ -240,7 +290,7 @@ WiredTigerIndex::WiredTigerIndex(OperationContext* ctx, bool isLogged, bool isReadOnly) : SortedDataInterface(ident, - _handleVersionInfo(ctx, uri, desc, isLogged, isReadOnly), + _handleVersionInfo(ctx, uri, ident, desc, isLogged, isReadOnly), Ordering::make(desc->keyPattern()), rsKeyFormat), _uri(uri), @@ -711,8 +761,45 @@ StatusWith<bool> WiredTigerIndex::_checkDups(OperationContext* opCtx, _collation); } +void WiredTigerIndex::_repairDataFormatVersion(OperationContext* opCtx, + const std::string& uri, + StringData ident, + const IndexDescriptor* desc) { + auto indexVersion = desc->version(); + auto isIndexVersion1 = indexVersion == IndexDescriptor::IndexVersion::kV1; + auto isIndexVersion2 = indexVersion == IndexDescriptor::IndexVersion::kV2; + auto isDataFormat6 = _dataFormatVersion == kDataFormatV1KeyStringV0IndexVersionV1; + auto isDataFormat8 = _dataFormatVersion == kDataFormatV2KeyStringV1IndexVersionV2; + auto isDataFormat13 = _dataFormatVersion == kDataFormatV5KeyStringV0UniqueIndexVersionV1; + auto isDataFormat14 = _dataFormatVersion == kDataFormatV6KeyStringV1UniqueIndexVersionV2; + // Only fixes the index data format when it could be from an edge case when converting the + // uniqueness of the index. Specifically: + // * The index is a secondary unique index, but the data format version is 6 (v1) or 8 (v2). + // * The index is a non-unique index, but the data format version is 13 (v1) or 14 (v2). + if ((!desc->isIdIndex() && desc->unique() && + ((isIndexVersion1 && isDataFormat6) || (isIndexVersion2 && isDataFormat8))) || + (!desc->unique() && + ((isIndexVersion1 && isDataFormat13) || (isIndexVersion2 && isDataFormat14)))) { + auto engine = opCtx->getServiceContext()->getStorageEngine(); + engine->getEngine()->alterIdentMetadata( + opCtx, ident, desc, /* isForceUpdateMetadata */ false); + auto prevVersion = _dataFormatVersion; + // The updated data format is guaranteed to be within the supported version range. + _dataFormatVersion = WiredTigerUtil::checkApplicationMetadataFormatVersion( + opCtx, uri, kMinimumIndexVersion, kMaximumIndexVersion) + .getValue(); + LOGV2_WARNING(6818600, + "Fixing index metadata data format version", + "namespace"_attr = desc->getEntry()->getNSSFromCatalog(opCtx), + "indexName"_attr = desc->indexName(), + "prevVersion"_attr = prevVersion, + "newVersion"_attr = _dataFormatVersion); + } +} + KeyString::Version WiredTigerIndex::_handleVersionInfo(OperationContext* ctx, const std::string& uri, + StringData ident, const IndexDescriptor* desc, bool isLogged, bool isReadOnly) { @@ -730,6 +817,8 @@ KeyString::Version WiredTigerIndex::_handleVersionInfo(OperationContext* ctx, } _dataFormatVersion = version.getValue(); + _repairDataFormatVersion(ctx, uri, ident, desc); + if (!desc->isIdIndex() && desc->unique() && (_dataFormatVersion < kDataFormatV3KeyStringV0UniqueIndexVersionV1 || _dataFormatVersion > kDataFormatV6KeyStringV1UniqueIndexVersionV2)) { @@ -1510,14 +1599,24 @@ private: _typeBits.resetFromBuffer(&br); if (!br.atEof()) { - LOGV2_FATAL(28608, - "Unique index cursor seeing multiple records for key {key} in index " - "{index} ({uri}) belonging to collection {collection}", - "Unique index cursor seeing multiple records for key in index", - "key"_attr = redact(curr(kWantKey)->key), - "index"_attr = _idx.indexName(), - "uri"_attr = _idx.uri(), - "collection"_attr = _idx.getCollectionNamespace(_opCtx)); + const auto bsonKey = redact(curr(kWantKey)->key); + const auto collectionNamespace = _idx.getCollectionNamespace(_opCtx); + addDataCorruptionEntryToHealthLog( + _opCtx, + collectionNamespace, + "WiredTigerIndexUniqueCursor::_updateIdAndTypeBitsFromValue", + "Unique index cursor seeing multiple records for key in index", + bsonKey, + _idx.indexName(), + _idx.uri()); + + LOGV2_ERROR_OPTIONS(7623202, + getLogOptionsForDataCorruption(*_opCtx->recoveryUnit()), + "Unique index cursor seeing multiple records for key in index", + "key"_attr = bsonKey, + "index"_attr = _idx.indexName(), + "uri"_attr = _idx.uri(), + logAttrs(collectionNamespace)); } } }; @@ -1545,12 +1644,25 @@ public: _typeBits.resetFromBuffer(&br); if (!br.atEof()) { - LOGV2_FATAL(5176200, - "Index cursor seeing multiple records for key in _id index", - "key"_attr = redact(curr(kWantKey)->key), - "index"_attr = _idx.indexName(), - "uri"_attr = _idx.uri(), - "collection"_attr = _idx.getCollectionNamespace(_opCtx)); + const auto bsonKey = redact(curr(kWantKey)->key); + const auto collectionNamespace = _idx.getCollectionNamespace(_opCtx); + + addDataCorruptionEntryToHealthLog( + _opCtx, + collectionNamespace, + "WiredTigerIdIndexCursor::updateIdAndTypeBits", + "Index cursor seeing multiple records for key in _id index", + bsonKey, + _idx.indexName(), + _idx.uri()); + + LOGV2_ERROR_OPTIONS(5176200, + getLogOptionsForDataCorruption(*_opCtx->recoveryUnit()), + "Index cursor seeing multiple records for key in _id index", + "key"_attr = bsonKey, + "index"_attr = _idx.indexName(), + "uri"_attr = _idx.uri(), + logAttrs(collectionNamespace)); } } }; @@ -1773,9 +1885,11 @@ void WiredTigerIdIndex::_unindex(OperationContext* opCtx, WiredTigerItem keyItem(keyString.getBuffer(), sizeWithoutRecordId); setKey(c, keyItem.Get()); + const auto failWithDataCorruptionForTest = + WTIndexUassertDuplicateRecordForKeyOnIdUnindex.shouldFail(); // On the _id index, the RecordId is stored in the value of the index entry. If the dupsAllowed // flag is not set, we blindly delete using only the key without checking the RecordId. - if (!dupsAllowed) { + if (!dupsAllowed && MONGO_likely(!failWithDataCorruptionForTest)) { int ret = WT_OP_CHECK(wiredTigerCursorRemove(opCtx, c)); if (ret == WT_NOTFOUND) { return; @@ -1807,14 +1921,26 @@ void WiredTigerIdIndex::_unindex(OperationContext* opCtx, RecordId idInIndex = KeyString::decodeRecordIdLong(&br); KeyString::TypeBits typeBits = KeyString::TypeBits::fromBuffer(getKeyStringVersion(), &br); - if (!br.atEof()) { + if (!br.atEof() || MONGO_unlikely(failWithDataCorruptionForTest)) { auto bsonKey = KeyString::toBson(keyString, _ordering); - LOGV2_FATAL(5176201, - "Un-index seeing multiple records for key", - "key"_attr = bsonKey, - "index"_attr = _desc->indexName(), - "uri"_attr = _uri, - "collection"_attr = getCollectionNamespace(opCtx)); + const auto collectionNamespace = getCollectionNamespace(opCtx); + + addDataCorruptionEntryToHealthLog(opCtx, + collectionNamespace, + "WiredTigerIdIndex::_unindex", + "Un-index seeing multiple records for key", + bsonKey, + _indexName, + _uri); + + LOGV2_ERROR_OPTIONS( + 5176201, + getLogOptionsForDataCorruption(*opCtx->recoveryUnit(), failWithDataCorruptionForTest), + "Un-index seeing multiple records for key", + "key"_attr = bsonKey, + "index"_attr = _indexName, + "uri"_attr = _uri, + logAttrs(collectionNamespace)); } // The RecordId matches, so remove the entry. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index 8fccc7d8c24..faea4ca6cac 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -234,10 +234,20 @@ protected: */ KeyString::Version _handleVersionInfo(OperationContext* ctx, const std::string& uri, + StringData ident, const IndexDescriptor* desc, bool isLogged, bool isReadOnly); + /* + * Attempts to repair the data format version in the index table metadata if there is a mismatch + * to the index type during startup. + */ + void _repairDataFormatVersion(OperationContext* opCtx, + const std::string& uri, + StringData ident, + const IndexDescriptor* desc); + RecordId _decodeRecordIdAtEnd(const void* buffer, size_t size); class BulkBuilder; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 97bf1fc8557..0de1c43284f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -1173,11 +1173,22 @@ private: int wtRet; bool fileUnchangedFlag = false; if (!_wtBackup->dupCursor) { - wtRet = (_session)->open_cursor( - _session, nullptr, _wtBackup->cursor, config.c_str(), &_wtBackup->dupCursor); - if (wtRet != 0) { - return wtRCToStatus(wtRet, _session); - } + size_t attempt = 0; + do { + wtRet = _session->open_cursor( + _session, nullptr, _wtBackup->cursor, config.c_str(), &_wtBackup->dupCursor); + + if (wtRet == EBUSY) { + logAndBackoff(8927900, + ::mongo::logv2::LogComponent::kStorage, + logv2::LogSeverity::Debug(1), + ++attempt, + "Opening duplicate backup cursor returned EBUSY, retrying", + "config"_attr = config); + } else if (wtRet != 0) { + return wtRCToStatus(wtRet, _session); + } + } while (wtRet == EBUSY); fileUnchangedFlag = true; } @@ -1310,12 +1321,15 @@ WiredTigerKVEngine::beginNonBlockingBackup(OperationContext* opCtx, for (const DurableCatalog::Entry& e : catalogEntries) { // Populate the collection ident with its namespace and UUID. UUID uuid = catalog->getMetaData(opCtx, e.catalogId)->options.uuid.get(); - _wtBackup.identToNamespaceAndUUIDMap.emplace(e.ident, std::make_pair(e.nss, uuid)); + std::string collectionIdent = e.ident; + _wtBackup.identToNamespaceAndUUIDMap.emplace(collectionIdent, + std::make_pair(e.nss, uuid)); // Populate the collection's index idents with the collection's namespace and UUID. std::vector<std::string> idxIdents = catalog->getIndexIdents(opCtx, e.catalogId); - for (const std::string& idxIdent : idxIdents) { - _wtBackup.identToNamespaceAndUUIDMap.emplace(idxIdent, std::make_pair(e.nss, uuid)); + for (const std::string& idxIdentFull : idxIdents) { + _wtBackup.identToNamespaceAndUUIDMap.emplace(idxIdentFull, + std::make_pair(e.nss, uuid)); } } } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 24af566c560..891c9c91bea 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -60,6 +60,26 @@ class WiredTigerSessionCache; class WiredTigerSizeStorer; class WiredTigerEngineRuntimeConfigParameter; +/** + * With the absolute path to an ident and the parent dbpath, return the ident. + * + * Note that the ident can have 4 different forms depending on the combination + * of server parameters present (directoryperdb / wiredTigerDirectoryForIndexes). + * With any one of these server parameters enabled, a directory could be included + * in the returned ident. + * See the unit test WiredTigerKVEngineTest::ExtractIdentFromPath for example usage. + * + * Note (2) idents use unix-style separators (always, see + * durable_catalog.cpp:generateUniqueIdent) but ident paths are platform-dependant. + * This method returns the unix-style "/" separators always. + */ +std::string extractIdentFromPath(const boost::filesystem::path& dbpath, + const boost::filesystem::path& identAbsolutePath); + + +Status validateExtraDiagnostics(const std::vector<std::string>& value, + const boost::optional<TenantId>& tenantId); + struct WiredTigerFileVersion { // MongoDB 4.4+ will not open on datafiles left behind by 4.2.5 and earlier. MongoDB 4.4 // shutting down in FCV 4.2 will leave data files that 4.2.6+ will understand diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index ba8861d15a6..547753bebf9 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -677,6 +677,15 @@ public: // On destruction, we must always handle freeing the underlying raw WT_CURSOR pointer. _saveStorageCursorOnDetachFromOperationContext = false; + // Shutdown does not wait for any threads running queries to be interrupted and exit. + // In addition, the RandomCursor destructor doesn't hold any global lock so we need to + // check if the server is shutting down to avoid calling into the storage engine, whose + // connection may have already been closed. + Status interruptStatus = _opCtx->checkForInterruptNoAssert(); + if (interruptStatus.code() == ErrorCodes::InterruptedAtShutdown) { + return; + } + detachFromOperationContext(); } } @@ -1769,6 +1778,8 @@ Status WiredTigerRecordStore::doCompact(OperationContext* opCtx) { dassert(opCtx->lockState()->isWriteLocked()); WiredTigerSessionCache* cache = WiredTigerRecoveryUnit::get(opCtx)->getSessionCache(); + const std::string uri(getURI()); + if (!cache->isEphemeral()) { WT_SESSION* s = WiredTigerRecoveryUnit::get(opCtx)->getSession()->getSession(); opCtx->recoveryUnit()->abandonSnapshot(); @@ -1779,7 +1790,7 @@ Status WiredTigerRecordStore::doCompact(OperationContext* opCtx) { if (ret == EBUSY) { return Status(ErrorCodes::Interrupted, - str::stream() << "Compaction interrupted on " << getURI().c_str() + str::stream() << "Compaction interrupted on " << uri << " due to cache eviction pressure"); } invariantWTOK(ret, s); @@ -2232,9 +2243,18 @@ boost::optional<Record> WiredTigerRecordStoreCursorBase::next() { invariant(!TestingProctor::instance().isEnabled(), "cursor returned out-of-order keys"); } - // uassert with 'DataCorruptionDetected' after logging. + auto options = [&] { + if (_opCtx->recoveryUnit()->getDataCorruptionDetectionMode() == + DataCorruptionDetectionMode::kThrow) { + // uassert with 'DataCorruptionDetected' after logging. + return logv2::LogOptions{ + logv2::UserAssertAfterLog(ErrorCodes::DataCorruptionDetected)}; + } else { + return logv2::LogOptions(logv2::LogComponent::kAutomaticDetermination); + } + }(); LOGV2_ERROR_OPTIONS(22406, - {logv2::UserAssertAfterLog(ErrorCodes::DataCorruptionDetected)}, + options, "WT_Cursor::next -- returned out-of-order keys", "forward"_attr = _forward, "next"_attr = id, diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h index 119b907f7a7..1afb4919fef 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h @@ -183,9 +183,6 @@ public: virtual bool compactSupported() const { return !_isEphemeral; } - virtual bool supportsOnlineCompaction() const { - return true; - } virtual Timestamp getPinnedOplog() const final; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp index 3da6bb97782..f8d9c410ef3 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp @@ -91,6 +91,13 @@ WiredTigerRecoveryUnit::WiredTigerRecoveryUnit(WiredTigerSessionCache* sc, WiredTigerRecoveryUnit::~WiredTigerRecoveryUnit() { invariant(!_inUnitOfWork(), toString(_getState())); _abort(); + + // If the session has non zero timeout then reset it back to 0 before returning the session back + // to the cache. + if (durationCount<Milliseconds>(_cacheMaxWaitTimeout)) { + auto wtSession = getSessionNoTxn()->getSession(); + invariantWTOK(wtSession->reconfigure(wtSession, "cache_max_wait_ms=0"), wtSession); + } } void WiredTigerRecoveryUnit::_commit() { @@ -421,6 +428,12 @@ void WiredTigerRecoveryUnit::_txnClose(bool commit) { _isOplogReader = false; _oplogVisibleTs = boost::none; _orderedCommit = true; // Default value is true; we assume all writes are ordered. + // Reset the kLastApplied read source back to the default of kNoTimestamp. Any reader requiring + // kLastApplied will set the read source again before reading. Resetting this read source + // simplifies the handling when stepup happens concurrently with read operations. + if (_timestampReadSource == ReadSource::kLastApplied) { + _timestampReadSource = ReadSource::kNoTimestamp; + } } Status WiredTigerRecoveryUnit::majorityCommittedSnapshotAvailable() const { @@ -598,11 +611,12 @@ void WiredTigerRecoveryUnit::_beginTransactionAtLastAppliedTimestamp(WT_SESSION* RoundUpReadTimestamp::kRound); auto status = txnOpen.setReadSnapshot(_readAtTimestamp); fassert(4847501, status); - txnOpen.done(); // We might have rounded to oldest between calling getLastApplied and setReadSnapshot. We // need to get the actual read timestamp we used. - _readAtTimestamp = _getTransactionReadTimestamp(session); + auto actualTimestamp = _getTransactionReadTimestamp(session); + txnOpen.done(); + _readAtTimestamp = actualTimestamp; } Timestamp WiredTigerRecoveryUnit::_beginTransactionAtNoOverlapTimestamp(WT_SESSION* session) { @@ -659,11 +673,11 @@ Timestamp WiredTigerRecoveryUnit::_beginTransactionAtNoOverlapTimestamp(WT_SESSI RoundUpReadTimestamp::kRound); auto status = txnOpen.setReadSnapshot(readTimestamp); fassert(51066, status); - txnOpen.done(); // We might have rounded to oldest between calling getAllDurable and setReadSnapshot. We // need to get the actual read timestamp we used. readTimestamp = _getTransactionReadTimestamp(session); + txnOpen.done(); return readTimestamp; } @@ -925,4 +939,15 @@ void WiredTigerRecoveryUnit::storeWriteContextForDebugging(const BSONObj& info) _writeContextForDebugging.push_back(info); } +void WiredTigerRecoveryUnit::setCacheMaxWaitTimeout(Milliseconds timeout) { + _cacheMaxWaitTimeout = timeout; + + auto wtSession = getSessionNoTxn()->getSession(); + invariantWTOK( + wtSession->reconfigure( + wtSession, + fmt::format("cache_max_wait_ms={}", durationCount<Milliseconds>(_cacheMaxWaitTimeout)) + .c_str()), + wtSession); +} } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h index e0b26282779..f1f692043b4 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h @@ -147,6 +147,8 @@ public: _multiTimestampConstraintTracker.ignoreAllMultiTimestampConstraints = true; } + void setCacheMaxWaitTimeout(Milliseconds) override; + // ---- WT STUFF WiredTigerSession* getSession(); @@ -283,6 +285,8 @@ private: std::vector<BSONObj> _writeContextForDebugging; WiredTigerStats _sessionStatsAfterLastOperation; + + Milliseconds _cacheMaxWaitTimeout{0}; }; } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp index 26e9824cc57..19e749bb445 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp @@ -461,7 +461,7 @@ bool WiredTigerSessionCache::isEphemeral() { UniqueWiredTigerSession WiredTigerSessionCache::getSession() { // We should never be able to get here after _shuttingDown is set, because no new // operations should be allowed to start. - invariant(!(_shuttingDown.loadRelaxed() & kShuttingDownMask)); + invariant(!(_shuttingDown.load() & kShuttingDownMask)); { stdx::lock_guard<Latch> lock(_cacheLock); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp index 942d6ce8592..de55e539056 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp @@ -27,7 +27,12 @@ * it in the license file. */ +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kWiredTiger + #include "mongo/db/storage/wiredtiger/wiredtiger_stats.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_util.h" +#include "mongo/logv2/log.h" +#include "mongo/unittest/log_test.h" #include "mongo/unittest/temp_dir.h" #include "mongo/unittest/unittest.h" #include <memory> @@ -164,10 +169,23 @@ protected: }; TEST_F(WiredTigerStatsTest, EmptySession) { + // Increase log component verbosity for WiredTiger + auto verbosityGuard = unittest::MinimumLoggedSeverityGuard{logv2::LogComponent::kWiredTiger, + logv2::LogSeverity::Debug(5)}; + auto verboseConfig = WiredTigerUtil::generateWTVerboseConfiguration(); + ASSERT_OK(wtRCToStatus(_conn->reconfigure(_conn, verboseConfig.c_str()), nullptr)); + // Read and write statistics should be empty. Check "data" field does not exist. "wait" fields // such as the schemaLock might have some value. auto statsBson = WiredTigerStats{_session}.toBSON(); - ASSERT_FALSE(statsBson.hasField("data")); + + { + BSONObjBuilder bob; + ASSERT_OK(WiredTigerUtil::exportTableToBSON(_session, "statistics:", "", &bob)); + LOGV2(9032000, "Connection statistics", "stats"_attr = bob.obj()); + } + + ASSERT_FALSE(statsBson.hasField("data")) << statsBson; } TEST_F(WiredTigerStatsTest, SessionWithWrite) { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index 7fa7416d58f..1afdb79babc 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -602,9 +602,12 @@ logv2::LogSeverity getWTLOGV2SeverityLevel(const BSONObj& obj) { return logv2::LogSeverity::Info(); case WT_VERBOSE_INFO: return logv2::LogSeverity::Log(); - case WT_VERBOSE_DEBUG: - return logv2::LogSeverity::Debug(1); default: + // MongoDB enables some WT debug compnonents by default. If performed a 1:1 + // translation from WT log severity levels, MongoDB would not log anything + // below default level Log, even if a Debug message came through the message + // handler. To solve this, we upgrade all Debug messages to the Log level + // to ensure they are seen. return logv2::LogSeverity::Log(); } } @@ -1191,6 +1194,30 @@ std::string WiredTigerUtil::generateWTVerboseConfiguration() { return cfg; } +// static +boost::optional<std::string> WiredTigerUtil::getConfigStringFromStorageOptions( + const BSONObj& options) { + if (auto wtElem = options[kWiredTigerEngineName]) { + BSONObj wtObj = wtElem.Obj(); + if (auto configStringElem = wtObj.getField(kConfigStringField)) { + return configStringElem.String(); + } + } + + return boost::none; +} + +// static +BSONObj WiredTigerUtil::setConfigStringToStorageOptions(const BSONObj& options, + const std::string& configString) { + // Storage options may contain settings for non-WiredTiger storage engines (e.g. inMemory). + // We should leave these settings intact. + auto wtElem = options[kWiredTigerEngineName]; + auto wtObj = wtElem ? wtElem.Obj() : BSONObj(); + return options.addFields( + BSON(kWiredTigerEngineName << wtObj.addFields(BSON(kConfigStringField << configString)))); +} + void WiredTigerUtil::removeEncryptionFromConfigString(std::string* configString) { static const StaticImmortal<pcrecpp::RE> encryptionOptsRegex(R"re(encryption=\([^\)]*\),?)re"); encryptionOptsRegex->GlobalReplace("", configString); @@ -1198,20 +1225,13 @@ void WiredTigerUtil::removeEncryptionFromConfigString(std::string* configString) // static BSONObj WiredTigerUtil::getSanitizedStorageOptionsForSecondaryReplication(const BSONObj& options) { - // Storage options may contain settings for non-WiredTiger storage engines (e.g. inMemory). - // We should leave these settings intact. - if (auto wtElem = options[kWiredTigerEngineName]) { - BSONObj wtObj = wtElem.Obj(); - if (auto configStringElem = wtObj.getField(kConfigStringField)) { - auto configString = configStringElem.String(); - removeEncryptionFromConfigString(&configString); - // Return a new BSONObj with the configString field sanitized. - return options.addFields(BSON(kWiredTigerEngineName << wtObj.addFields( - BSON(kConfigStringField << configString)))); - } + auto configString = getConfigStringFromStorageOptions(options); + if (!configString) { + return options; } - return options; + removeEncryptionFromConfigString(configString.get_ptr()); + return setConfigStringToStorageOptions(options, *configString); } } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index 57f8113e3c9..a70c050b2ea 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -335,6 +335,17 @@ public: static T castStatisticsValue(uint64_t statisticsValue); /** + * Gets the WiredTiger configuration string from storage engine collection options. + */ + static boost::optional<std::string> getConfigStringFromStorageOptions(const BSONObj& options); + + /** + * Sets the WiredTiger configuration string to storage engine collection options. + */ + static BSONObj setConfigStringToStorageOptions(const BSONObj& options, + const std::string& configString); + + /** * Removes encryption configuration from a config string. Should only be applied on custom * config strings on secondaries. Fixes an issue where encryption configuration might be * replicated to non-encrypted nodes, or nodes with different encryption options, causing diff --git a/src/mongo/db/timeseries/SConscript b/src/mongo/db/timeseries/SConscript index 25162b9d2c7..ed1674218e7 100644 --- a/src/mongo/db/timeseries/SConscript +++ b/src/mongo/db/timeseries/SConscript @@ -13,6 +13,7 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', '$BUILD_DIR/mongo/idl/idl_parser', '$BUILD_DIR/mongo/idl/server_parameter', '$BUILD_DIR/mongo/util/processinfo' @@ -145,6 +146,7 @@ env.CppUnitTest( source=[ 'bucket_catalog_test.cpp', 'minmax_test.cpp', + 'schema_test.cpp', 'timeseries_dotted_path_support_test.cpp', 'timeseries_extended_range_test.cpp', 'timeseries_index_schema_conversion_functions_test.cpp', diff --git a/src/mongo/db/timeseries/bucket_catalog.cpp b/src/mongo/db/timeseries/bucket_catalog.cpp index b41d20962f1..c9d3bc2f76a 100644 --- a/src/mongo/db/timeseries/bucket_catalog.cpp +++ b/src/mongo/db/timeseries/bucket_catalog.cpp @@ -899,8 +899,9 @@ const BSONObj& BucketCatalog::BucketMetadata::toBSON() const { return _metadata; } -StringData BucketCatalog::BucketMetadata::getMetaField() const { - return StringData(_metadataElement.fieldName()); +boost::optional<StringData> BucketCatalog::BucketMetadata::getMetaField() const { + return _metadataElement ? boost::make_optional(_metadataElement.fieldNameStringData()) + : boost::none; } const StringData::ComparatorInterface* BucketCatalog::BucketMetadata::getComparator() const { diff --git a/src/mongo/db/timeseries/bucket_catalog.h b/src/mongo/db/timeseries/bucket_catalog.h index 8b175a3f9b1..64b65dfe753 100644 --- a/src/mongo/db/timeseries/bucket_catalog.h +++ b/src/mongo/db/timeseries/bucket_catalog.h @@ -343,7 +343,7 @@ private: const BSONObj& toBSON() const; - StringData getMetaField() const; + boost::optional<StringData> getMetaField() const; const StringData::ComparatorInterface* getComparator() const; diff --git a/src/mongo/db/timeseries/bucket_compression.cpp b/src/mongo/db/timeseries/bucket_compression.cpp index 7d6e6961687..414a8f46b64 100644 --- a/src/mongo/db/timeseries/bucket_compression.cpp +++ b/src/mongo/db/timeseries/bucket_compression.cpp @@ -165,7 +165,7 @@ CompressionResult compressBucket(const BSONObj& bucketDoc, std::sort(measurements.begin(), measurements.end(), [](const Measurement& lhs, const Measurement& rhs) { - return lhs.timeField.timestamp() < rhs.timeField.timestamp(); + return lhs.timeField.date() < rhs.timeField.date(); }); // Write _id unless EOO which it can be in some unittests diff --git a/src/mongo/db/timeseries/flat_bson.cpp b/src/mongo/db/timeseries/flat_bson.cpp index e5b15133b1e..772d87ecdae 100644 --- a/src/mongo/db/timeseries/flat_bson.cpp +++ b/src/mongo/db/timeseries/flat_bson.cpp @@ -896,7 +896,8 @@ Schema::UpdateStatus Schema::_maybeUpdateValue( if (data.type() == SchemaStore::Type::kUnset) { data.setValue(elem); - } else if (typeComp(elem, data.value().type()) != 0) { + } else if (data.type() != SchemaStore::Type::kValue || + typeComp(elem, data.value().type()) != 0) { // Type mismatch status = UpdateStatus::Failed; } diff --git a/src/mongo/db/timeseries/schema_test.cpp b/src/mongo/db/timeseries/schema_test.cpp new file mode 100644 index 00000000000..25bd8385859 --- /dev/null +++ b/src/mongo/db/timeseries/schema_test.cpp @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/timeseries/flat_bson.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::timeseries::bucket_catalog { +namespace { + +TEST(Schema, UpdateValue) { + Schema schema; + ASSERT(schema.update(BSON("a" << 1), boost::none, nullptr) == Schema::UpdateStatus::Updated); + ASSERT(schema.update(BSON("a" << 2), boost::none, nullptr) == Schema::UpdateStatus::Updated); + ASSERT(schema.update(BSON("a" << 3 << "b" << 1), boost::none, nullptr) == + Schema::UpdateStatus::Updated); + ASSERT(schema.update(BSON("b" << 2), boost::none, nullptr) == Schema::UpdateStatus::Updated); + ASSERT(schema.update(BSON("a" << BSONUndefined), boost::none, nullptr) == + Schema::UpdateStatus::Failed); + ASSERT(schema.update(BSON("a" << BSON("b" << 1)), boost::none, nullptr) == + Schema::UpdateStatus::Failed); + ASSERT(schema.update(BSON("a" << BSON_ARRAY(1)), boost::none, nullptr) == + Schema::UpdateStatus::Failed); +} + +TEST(Schema, UpdateArray) { + Schema schema; + ASSERT(schema.update(BSON("a" << BSON_ARRAY(1)), boost::none, nullptr) == + Schema::UpdateStatus::Updated); + ASSERT(schema.update(BSON("a" << BSON_ARRAY(2)), boost::none, nullptr) == + Schema::UpdateStatus::Updated); + ASSERT(schema.update(BSON("a" << BSON_ARRAY(1 << 2)), boost::none, nullptr) == + Schema::UpdateStatus::Updated); + ASSERT(schema.update(BSON("a" << BSONUndefined), boost::none, nullptr) == + Schema::UpdateStatus::Failed); + ASSERT(schema.update(BSON("a" << 1), boost::none, nullptr) == Schema::UpdateStatus::Failed); + ASSERT(schema.update(BSON("a" << BSON("b" << 1)), boost::none, nullptr) == + Schema::UpdateStatus::Failed); +} + +TEST(Schema, UpdateObject) { + Schema schema; + ASSERT(schema.update(BSON("a" << BSON("b" << 1)), boost::none, nullptr) == + Schema::UpdateStatus::Updated); + ASSERT(schema.update(BSON("a" << BSON("b" << 2)), boost::none, nullptr) == + Schema::UpdateStatus::Updated); + ASSERT(schema.update(BSON("a" << 1), boost::none, nullptr) == Schema::UpdateStatus::Failed); + ASSERT(schema.update(BSON("a" << BSONUndefined), boost::none, nullptr) == + Schema::UpdateStatus::Failed); + ASSERT(schema.update(BSON("a" << BSON_ARRAY(1)), boost::none, nullptr) == + Schema::UpdateStatus::Failed); +} + +} // namespace +} // namespace mongo::timeseries::bucket_catalog diff --git a/src/mongo/db/timeseries/timeseries_collmod.cpp b/src/mongo/db/timeseries/timeseries_collmod.cpp index 1e8ad8e186d..49183fbac75 100644 --- a/src/mongo/db/timeseries/timeseries_collmod.cpp +++ b/src/mongo/db/timeseries/timeseries_collmod.cpp @@ -80,6 +80,8 @@ std::unique_ptr<CollMod> makeTimeseriesBucketsCollModCommand(OperationContext* o request.setChangeStreamPreAndPostImages(origCmd.getChangeStreamPreAndPostImages()); request.setExpireAfterSeconds(origCmd.getExpireAfterSeconds()); request.setTimeseries(origCmd.getTimeseries()); + request.setTimeseriesBucketsMayHaveMixedSchemaData( + origCmd.getTimeseriesBucketsMayHaveMixedSchemaData()); request.setDryRun(origCmd.getDryRun()); auto cmd = std::make_unique<CollMod>(ns); cmd->setCollModRequest(request); diff --git a/src/mongo/db/timeseries/timeseries_extended_range.cpp b/src/mongo/db/timeseries/timeseries_extended_range.cpp index 44a7ff060d9..a2a96866245 100644 --- a/src/mongo/db/timeseries/timeseries_extended_range.cpp +++ b/src/mongo/db/timeseries/timeseries_extended_range.cpp @@ -34,9 +34,14 @@ namespace mongo::timeseries { bool dateOutsideStandardRange(Date_t date) { - constexpr long long kMaxNormalRangeTimestamp = ((1LL << 31) - 1); - long long timeSeconds = durationCount<Seconds>(date.toDurationSinceEpoch()); - return timeSeconds < 0 || timeSeconds > kMaxNormalRangeTimestamp; + // The latest timestamp in the standard range is when the bucket OID cannot fit into a 32 bit + // integer, because we cannot reliably scan the documents by OID and must indicate the bucket + // requires extended range support. Since OID are rounded to seconds, this maximum value is the + // largest 32 bit integer number of seconds since the epoch. We convert this value to + // milliseconds, since query routing and user specified dates have millisecond precision. + constexpr long long kMaxNormalRangeTimestamp = ((1LL << 31) - 1) * 1000; + long long timeMilliseconds = date.toMillisSinceEpoch(); + return timeMilliseconds < 0 || timeMilliseconds > kMaxNormalRangeTimestamp; } bool bucketsHaveDateOutsideStandardRange(const TimeseriesOptions& options, diff --git a/src/mongo/db/timeseries/timeseries_extended_range_test.cpp b/src/mongo/db/timeseries/timeseries_extended_range_test.cpp index a6392aabd1b..e1f1cd6d496 100644 --- a/src/mongo/db/timeseries/timeseries_extended_range_test.cpp +++ b/src/mongo/db/timeseries/timeseries_extended_range_test.cpp @@ -36,10 +36,10 @@ namespace mongo { namespace { TEST(TimeseriesExtendedRangeSupport, DateOutsideStandardRange) { - Date_t minStandard = Date_t::fromDurationSinceEpoch(Seconds(0)); + Date_t minStandard = Date_t::fromDurationSinceEpoch(Milliseconds(0)); Date_t maxStandard = Date_t::fromDurationSinceEpoch(Seconds((1LL << 31) - 1)); - Date_t extendedLow = Date_t::fromDurationSinceEpoch(Seconds(-1)); + Date_t extendedLow = Date_t::fromDurationSinceEpoch(Milliseconds(-1)); Date_t extendedHigh = Date_t::fromDurationSinceEpoch(Seconds(1LL << 31)); ASSERT_FALSE(timeseries::dateOutsideStandardRange(minStandard)); @@ -82,16 +82,35 @@ TEST(TimeseriesExtendedRangeSupport, BucketsHaveDateOutsideStandardRange) { R"({"control": {"min": {"time": {"$date": "2110-01-01T00:00:00.000Z"}}}})")}, }; + std::vector<InsertStatement> extendedRangeMillisecondsLow = { + {7, mongo::fromjson(R"({"control": {"min": {"time": {"$date": -999}}}})")}, + }; + + // This date is one millisecond after the maximum (the largest 32 bit integer) + // number of seconds since the epoch. + std::vector<InsertStatement> extendedRangeMillisecondsHigh = { + {8, + mongo::fromjson( + R"({"control": {"min": {"time": {"$date": "2038-01-19T03:14:07.001Z"}}}})")}, + }; + ASSERT_FALSE(timeseries::bucketsHaveDateOutsideStandardRange( options, standardRange.begin(), standardRange.end())); ASSERT_TRUE(timeseries::bucketsHaveDateOutsideStandardRange( options, extendedRangeLow.begin(), extendedRangeLow.end())); ASSERT_TRUE(timeseries::bucketsHaveDateOutsideStandardRange( options, extendedRangeHigh.begin(), extendedRangeHigh.end())); + ASSERT_TRUE(timeseries::bucketsHaveDateOutsideStandardRange( + options, extendedRangeMillisecondsHigh.begin(), extendedRangeMillisecondsHigh.end())); std::vector<InsertStatement> mixed = {standardRange[0], standardRange[1], extendedRangeLow[0]}; ASSERT_TRUE( timeseries::bucketsHaveDateOutsideStandardRange(options, mixed.begin(), mixed.end())); + + std::vector<InsertStatement> mixedWithMilliseconds = {standardRange[0], + extendedRangeMillisecondsLow[0]}; + ASSERT_TRUE(timeseries::bucketsHaveDateOutsideStandardRange( + options, mixedWithMilliseconds.begin(), mixedWithMilliseconds.end())); } } // namespace diff --git a/src/mongo/db/timeseries/timeseries_options.cpp b/src/mongo/db/timeseries/timeseries_options.cpp index 217e3b07a3b..5281a799210 100644 --- a/src/mongo/db/timeseries/timeseries_options.cpp +++ b/src/mongo/db/timeseries/timeseries_options.cpp @@ -167,6 +167,16 @@ Date_t roundTimestampToGranularity(const Date_t& time, BucketGranularityEnum gra int roundingSeconds = getBucketRoundingSecondsFromGranularity(granularity); long long timeSeconds = durationCount<Seconds>(time.toDurationSinceEpoch()); long long roundedTimeSeconds = (timeSeconds - (timeSeconds % roundingSeconds)); + // Make sure we always round down and not towards epoch, even for dates prior to 1970 with a + // negative duration since epoch. + if (roundedTimeSeconds > timeSeconds) { + roundedTimeSeconds -= roundingSeconds; + // It is not possible that we underflowed when performing the subtraction above. Because + // we've converted the dates in milliseconds to seconds there is tons of integer space left + // for the subtraction. We'd need to have a gigantic amount of rounding seconds to be able + // to overflow here. Therefore we invariant over uasserting. + invariant(roundedTimeSeconds <= timeSeconds); + } return Date_t::fromDurationSinceEpoch(Seconds{roundedTimeSeconds}); } } // namespace timeseries diff --git a/src/mongo/db/timeseries/timeseries_options_test.cpp b/src/mongo/db/timeseries/timeseries_options_test.cpp index e3106407cea..db3d8e4a46b 100644 --- a/src/mongo/db/timeseries/timeseries_options_test.cpp +++ b/src/mongo/db/timeseries/timeseries_options_test.cpp @@ -29,6 +29,11 @@ #include "mongo/platform/basic.h" +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/date_time/posix_time/time_parsers.hpp> +#include <boost/move/utility_core.hpp> + +#include "mongo/base/string_data.h" #include "mongo/db/timeseries/timeseries_options.h" #include "mongo/unittest/unittest.h" #include "mongo/util/time_support.h" @@ -36,6 +41,21 @@ namespace mongo { +int roundingSecondsFromGranularity(BucketGranularityEnum granularity) { + switch (granularity) { + case BucketGranularityEnum::Seconds: + // Round down to nearest minute. + return 60; + case BucketGranularityEnum::Minutes: + // Round down to nearest hour. + return 60 * 60; + case BucketGranularityEnum::Hours: + // Round down to nearest day. + return 60 * 60 * 24; + } + MONGO_UNREACHABLE; +} + TEST(TimeseriesOptionsTest, RoundTimestampToGranularity) { std::vector<std::tuple<BucketGranularityEnum, std::string, std::string>> testCases{ {BucketGranularityEnum::Seconds, "2021-01-01T00:00:15.555Z", "2021-01-01T00:00:00.000Z"}, @@ -60,4 +80,123 @@ TEST(TimeseriesOptionsTest, RoundTimestampToGranularity) { } } +TEST(TimeseriesOptionsTest, RoundTimestampBySeconds) { + std::vector<std::tuple<BucketGranularityEnum, std::string, std::string>> testCases{ + {BucketGranularityEnum::Seconds, "2024-08-08T00:00:00.000Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T00:00:00.001Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T00:00:15.555Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T00:00:30.555Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T00:00:45.555Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T00:00:59.999Z", "2024-08-08T00:00:00.000Z"}, + + {BucketGranularityEnum::Seconds, "2024-08-08T05:04:00.000Z", "2024-08-08T05:04:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T05:04:00.001Z", "2024-08-08T05:04:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T05:04:15.555Z", "2024-08-08T05:04:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T05:04:30.555Z", "2024-08-08T05:04:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T05:04:45.555Z", "2024-08-08T05:04:00.000Z"}, + {BucketGranularityEnum::Seconds, "2024-08-08T05:04:59.999Z", "2024-08-08T05:04:00.000Z"}, + + {BucketGranularityEnum::Minutes, "2024-08-08T00:00:00.000Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Minutes, "2024-08-08T00:00:00.001Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Minutes, "2024-08-08T00:15:00.000Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Minutes, "2024-08-08T00:30:00.000Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Minutes, "2024-08-08T00:45:00.000Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Minutes, "2024-08-08T00:59:59.999Z", "2024-08-08T00:00:00.000Z"}, + + {BucketGranularityEnum::Hours, "2024-08-08T00:00:00.000Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Hours, "2024-08-08T00:00:00.001Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Hours, "2024-08-08T06:00:00.000Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Hours, "2024-08-08T12:00:00.000Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Hours, "2024-08-08T18:00:00.000Z", "2024-08-08T00:00:00.000Z"}, + {BucketGranularityEnum::Hours, "2024-08-08T23:59:59.999Z", "2024-08-08T00:00:00.000Z"}, + }; + + for (const auto& [roundingGranularity, input, expectedOutput] : testCases) { + auto inputDate = dateFromISOString(input); + ASSERT_OK(inputDate); + auto roundedDate = + timeseries::roundTimestampToGranularity(inputDate.getValue(), roundingGranularity); + ASSERT_EQ(dateToISOStringUTC(roundedDate), expectedOutput); + } +} + +TEST(TimeseriesOptionsTest, ExtendedRangeRoundTimestamp) { + std::vector<std::tuple<BucketGranularityEnum, std::string, std::string>> testCases{ + {BucketGranularityEnum::Seconds, "1901-01-01T00:00:12.345", "1901-01-01T00:00:00"}, + {BucketGranularityEnum::Seconds, "1901-01-01T00:04:12.345", "1901-01-01T00:04:00"}, + {BucketGranularityEnum::Seconds, "1901-01-01T02:04:12.345", "1901-01-01T02:04:00"}, + + {BucketGranularityEnum::Seconds, "1969-01-01T00:00:12.345", "1969-01-01T00:00:00"}, + {BucketGranularityEnum::Seconds, "1969-01-01T00:04:12.345", "1969-01-01T00:04:00"}, + {BucketGranularityEnum::Seconds, "1969-01-01T02:04:12.345", "1969-01-01T02:04:00"}, + + {BucketGranularityEnum::Seconds, "2040-01-01T00:00:12.345", "2040-01-01T00:00:00"}, + {BucketGranularityEnum::Seconds, "2040-01-01T00:04:12.345", "2040-01-01T00:04:00"}, + {BucketGranularityEnum::Seconds, "2040-01-01T02:04:12.345", "2040-01-01T02:04:00"}, + + {BucketGranularityEnum::Seconds, "2108-01-01T00:00:12.345", "2108-01-01T00:00:00"}, + {BucketGranularityEnum::Seconds, "2108-01-01T00:04:12.345", "2108-01-01T00:04:00"}, + {BucketGranularityEnum::Seconds, "2108-01-01T02:04:12.345", "2108-01-01T02:04:00"}, + + {BucketGranularityEnum::Minutes, "1901-01-01T00:00:12.345", "1901-01-01T00:00:00"}, + {BucketGranularityEnum::Minutes, "1901-01-01T00:04:12.345", "1901-01-01T00:00:00"}, + {BucketGranularityEnum::Minutes, "1901-01-01T02:04:12.345", "1901-01-01T02:00:00"}, + + {BucketGranularityEnum::Minutes, "1969-01-01T00:00:12.345", "1969-01-01T00:00:00"}, + {BucketGranularityEnum::Minutes, "1969-01-01T00:04:12.345", "1969-01-01T00:00:00"}, + {BucketGranularityEnum::Minutes, "1969-01-01T02:04:12.345", "1969-01-01T02:00:00"}, + + {BucketGranularityEnum::Minutes, "2040-01-01T00:00:12.345", "2040-01-01T00:00:00"}, + {BucketGranularityEnum::Minutes, "2040-01-01T00:04:12.345", "2040-01-01T00:00:00"}, + {BucketGranularityEnum::Minutes, "2040-01-01T02:04:12.345", "2040-01-01T02:00:00"}, + + {BucketGranularityEnum::Minutes, "2108-01-01T00:00:12.345", "2108-01-01T00:00:00"}, + {BucketGranularityEnum::Minutes, "2108-01-01T00:04:12.345", "2108-01-01T00:00:00"}, + {BucketGranularityEnum::Minutes, "2108-01-01T02:04:12.345", "2108-01-01T02:00:00"}, + + {BucketGranularityEnum::Hours, "1901-01-01T00:00:12.345", "1901-01-01T00:00:00"}, + {BucketGranularityEnum::Hours, "1901-01-01T00:04:12.345", "1901-01-01T00:00:00"}, + {BucketGranularityEnum::Hours, "1901-01-01T02:04:12.345", "1901-01-01T00:00:00"}, + + {BucketGranularityEnum::Hours, "1969-01-01T00:00:12.345", "1969-01-01T00:00:00"}, + {BucketGranularityEnum::Hours, "1969-01-01T00:04:12.345", "1969-01-01T00:00:00"}, + {BucketGranularityEnum::Hours, "1969-01-01T02:04:12.345", "1969-01-01T00:00:00"}, + + {BucketGranularityEnum::Hours, "2040-01-01T00:00:12.345", "2040-01-01T00:00:00"}, + {BucketGranularityEnum::Hours, "2040-01-01T00:04:12.345", "2040-01-01T00:00:00"}, + {BucketGranularityEnum::Hours, "2040-01-01T02:04:12.345", "2040-01-01T00:00:00"}, + + {BucketGranularityEnum::Hours, "2108-01-01T00:00:12.345", "2108-01-01T00:00:00"}, + {BucketGranularityEnum::Hours, "2108-01-01T00:04:12.345", "2108-01-01T00:00:00"}, + {BucketGranularityEnum::Hours, "2108-01-01T02:04:12.345", "2108-01-01T00:00:00"}, + }; + + // TODO SERVER-94228: Support ISO 8601 date parsing and formatting of dates prior to 1970. + static constexpr auto epoch = boost::posix_time::ptime(boost::gregorian::date(1970, 1, 1)); + auto parse = [](const std::string& input) { + auto ptime = boost::posix_time::from_iso_extended_string(input); + return Date_t::fromMillisSinceEpoch((ptime - epoch).total_milliseconds()); + }; + auto format = [](Date_t date) { + boost::posix_time::milliseconds ms(date.toMillisSinceEpoch()); + return boost::posix_time::to_iso_extended_string(epoch + ms); + }; + + for (const auto& [roundingGranularity, input, expectedOutput] : testCases) { + Date_t inputDate = parse(input); + auto roundedDate = timeseries::roundTimestampToGranularity(inputDate, roundingGranularity); + // We should always round down + ASSERT_LTE(roundedDate, inputDate); + // The rounding amount should be less than the rounding seconds + ASSERT_LT((inputDate - roundedDate).count(), + roundingSecondsFromGranularity(roundingGranularity) * 1000); + // Ensure that we've rounded to an even number according to our rounding seconds + ASSERT_EQ(durationCount<Seconds>(roundedDate.toDurationSinceEpoch()) % + roundingSecondsFromGranularity(roundingGranularity), + 0); + // Validate the expected output + ASSERT_EQ(format(roundedDate), expectedOutput); + } +} + } // namespace mongo diff --git a/src/mongo/db/update/addtoset_node.cpp b/src/mongo/db/update/addtoset_node.cpp index f7b821815d1..824f6943c8e 100644 --- a/src/mongo/db/update/addtoset_node.cpp +++ b/src/mongo/db/update/addtoset_node.cpp @@ -85,7 +85,10 @@ Status AddToSetNode::init(BSONElement modExpr, str::stream() << "Found unexpected fields after $each in $addToSet: " << modExpr.Obj()); } - _elements = firstElement.Array(); + + // We call 'ArrayVerifyIndexes' to uassert in the event that 'firstElement' is a + // BSONArray with invalid indexes. + _elements = firstElement.ArrayVerifyIndexes(); } } diff --git a/src/mongo/db/update/addtoset_node_test.cpp b/src/mongo/db/update/addtoset_node_test.cpp index d2b7f90be65..f590c7140dc 100644 --- a/src/mongo/db/update/addtoset_node_test.cpp +++ b/src/mongo/db/update/addtoset_node_test.cpp @@ -99,6 +99,23 @@ TEST(AddToSetNodeTest, InitSucceedsWithArray) { ASSERT_OK(node.init(update["$addToSet"]["a"], expCtx)); } +TEST(AddToSetNodeTest, InitFailsWhenArgumentIsInvalidBSONArray) { + // Create our invalid array by creating a BSONObj with non contiguous array indexes that is then + // passed to the BSONArray ctor. + BSONObj updateArrAsObj = BSON("0" + << "foo" + << "2" + << "bar"); + BSONArray updateArr(updateArrAsObj); + + auto update = BSON("$addToSet" << BSON("fieldName" << BSON("$each" << updateArr))); + boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); + AddToSetNode node; + + ASSERT_THROWS(node.init(update["$addToSet"]["fieldName"], expCtx), + ExceptionFor<ErrorCodes::BadValue>); +} + TEST(AddToSetNodeTest, InitSucceedsWithScaler) { auto update = fromjson("{$addToSet: {a: 1}}"); boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest()); diff --git a/src/mongo/db/update/document_diff_applier.cpp b/src/mongo/db/update/document_diff_applier.cpp index ec592ef3363..7620a756064 100644 --- a/src/mongo/db/update/document_diff_applier.cpp +++ b/src/mongo/db/update/document_diff_applier.cpp @@ -455,7 +455,7 @@ int32_t computeDamageOnArray(const BSONObj& preImageRoot, nextMod = reader->next(); } else { // This field is not mentioned in the diff so we pad the post image with null. - auto idxAsStr = std::to_string(idx); + const auto idxAsStr = std::to_string(idx); // The end of 'arrayPreImage' with the offset from the updates made already. auto targetOffset = targetOffsetInPostImage( arrayPreImage.end()->rawdata(), preImageRoot.objdata(), offsetRoot, diffSize); @@ -464,7 +464,7 @@ int32_t computeDamageOnArray(const BSONObj& preImageRoot, appendDamage(damages, bufBuilder->len(), sourceSize, targetOffset, 0); diffSize += sourceSize; appendTypeByte(bufBuilder, BSONType::jstNULL); - bufBuilder->appendStr(idxAsStr); + bufBuilder->appendCStr(idxAsStr); } } diff --git a/src/mongo/db/update/document_diff_calculator_test.cpp b/src/mongo/db/update/document_diff_calculator_test.cpp index 55a05051021..5c83ac027c8 100644 --- a/src/mongo/db/update/document_diff_calculator_test.cpp +++ b/src/mongo/db/update/document_diff_calculator_test.cpp @@ -32,6 +32,7 @@ #include <functional> #include "mongo/bson/bson_depth.h" +#include "mongo/bson/bson_validate.h" #include "mongo/bson/json.h" #include "mongo/db/update/document_diff_calculator.h" #include "mongo/unittest/unittest.h" @@ -295,7 +296,7 @@ TEST(DocumentDiffCalculatorTest, DeeplyNestObjectGenerateDiff) { preBob.append("largeField", largeValue); buildDeepObj(&preBob, "subObj", 0, maxDepth, functionToApply); auto preObj = preBob.done(); - ASSERT(preObj.valid()); + ASSERT(validateBSON(preObj).isOK()); BSONObjBuilder postBob; postBob.append("largeField", largeValue); @@ -312,7 +313,7 @@ TEST(DocumentDiffCalculatorTest, DeeplyNestObjectGenerateDiff) { // Deleting the deepest field should give the post object. diffOutput = doc_diff::computeDiff(preObj, postBob2.done(), 0, nullptr); ASSERT(diffOutput); - ASSERT(diffOutput->diff.valid()); + ASSERT_OK(validateBSON(diffOutput->diff)); BSONObjBuilder expectedOutputBuilder; buildDeepObj(&expectedOutputBuilder, @@ -341,17 +342,17 @@ TEST(DocumentDiffCalculatorTest, DeepestObjectSubDiff) { value = 1; buildDeepObj(&bob1, "subObj", 0, BSONDepth::getMaxDepthForUserStorage(), functionToApply); auto preObj = bob1.done(); - ASSERT(preObj.valid()); + ASSERT_OK(validateBSON(preObj)); BSONObjBuilder postBob; value = 2; buildDeepObj(&postBob, "subObj", 0, BSONDepth::getMaxDepthForUserStorage(), functionToApply); auto postObj = postBob.done(); - ASSERT(postObj.valid()); + ASSERT_OK(validateBSON(postObj)); auto diffOutput = doc_diff::computeDiff(preObj, postObj, 0, nullptr); ASSERT(diffOutput); - ASSERT(diffOutput->diff.valid()); + ASSERT_OK(validateBSON(diffOutput->diff)); BSONObjBuilder expectedOutputBuilder; buildDeepObj(&expectedOutputBuilder, diff --git a/src/mongo/db/update/object_replace_executor.cpp b/src/mongo/db/update/object_replace_executor.cpp index e2d9262e001..205b270a8ca 100644 --- a/src/mongo/db/update/object_replace_executor.cpp +++ b/src/mongo/db/update/object_replace_executor.cpp @@ -45,19 +45,24 @@ namespace { constexpr StringData kIdFieldName = "_id"_sd; } // namespace -ObjectReplaceExecutor::ObjectReplaceExecutor(BSONObj replacement) - : _replacementDoc(replacement.getOwned()), _containsId(false) { - - // Replace all zero-valued timestamps with the current time and check for the existence of _id. +ObjectReplaceExecutor::ObjectReplaceExecutor(BSONObj replacement, bool bypassEmptyTsReplacement) + : _replacementDoc(replacement.getOwned()), + _containsId(false), + _bypassEmptyTsReplacement(bypassEmptyTsReplacement) { + // Check for the existence of the "_id" field, and if approrpriate replace all zero-valued + // timestamps with the current time. for (auto&& elem : _replacementDoc) { - // Do not change the _id field. if (elem.fieldNameStringData() == kIdFieldName) { _containsId = true; continue; } - if (elem.type() == BSONType::bsonTimestamp) { + // For updates that originated from the oplog, we're required to apply the update + // exactly as it was recorded (even if it contains zero-valued timestamps). Therefore, + // we should only replace zero-valued timestamps with the current time when + // '_bypassEmptyTsReplacement' is false. + if (!_bypassEmptyTsReplacement && elem.type() == BSONType::bsonTimestamp) { auto timestampView = DataView(const_cast<char*>(elem.value())); // We don't need to do an endian-safe read here, because 0 is 0 either way. diff --git a/src/mongo/db/update/object_replace_executor.h b/src/mongo/db/update/object_replace_executor.h index f9c70bde919..3f4c787e588 100644 --- a/src/mongo/db/update/object_replace_executor.h +++ b/src/mongo/db/update/object_replace_executor.h @@ -65,10 +65,11 @@ public: bool allowTopLevelDollarPrefixedFields = false); /** - * Initializes the node with the document to replace with. Any zero-valued timestamps (except - * for the _id) are updated to the current time. + * Initializes the node with the document to replace with. If 'bypassEmptyTsReplacement' is + * false, any zero-valued timestamps (except for the _id) will be replaced with the current + * time. */ - explicit ObjectReplaceExecutor(BSONObj replacement); + explicit ObjectReplaceExecutor(BSONObj replacement, bool bypassEmptyTsReplacement = false); /** * Replaces the document that 'applyParams.element' belongs to with 'val'. If 'val' does not @@ -95,6 +96,8 @@ private: // True if '_replacementDoc' contains an _id. bool _containsId; + + bool _bypassEmptyTsReplacement = false; }; } // namespace mongo diff --git a/src/mongo/db/update/update_driver.cpp b/src/mongo/db/update/update_driver.cpp index 5ce9dc8aa84..9839487da9d 100644 --- a/src/mongo/db/update/update_driver.cpp +++ b/src/mongo/db/update/update_driver.cpp @@ -159,7 +159,14 @@ void UpdateDriver::parse( "multi update is not supported for replacement-style update", !multi); - _updateExecutor = std::make_unique<ObjectReplaceExecutor>(updateMod.getUpdateReplacement()); + // For updates that originated from the oplog, we're required to apply the update + // exactly as it was recorded (even if it contains zero-valued timestamps). Therefore, + // we should only replace zero-valued timestamps with the current time when both + // '_bypassEmptyTsReplacement' and '_fromOplogApplication' are false. + const bool bypassEmptyTsReplacement = _bypassEmptyTsReplacement || _fromOplogApplication; + + _updateExecutor = std::make_unique<ObjectReplaceExecutor>(updateMod.getUpdateReplacement(), + bypassEmptyTsReplacement); // Register the fact that this driver will only do full object replacements. _updateType = UpdateType::kReplacement; diff --git a/src/mongo/db/update/update_driver.h b/src/mongo/db/update/update_driver.h index 94d1f73c8d4..9949f43787c 100644 --- a/src/mongo/db/update/update_driver.h +++ b/src/mongo/db/update/update_driver.h @@ -195,6 +195,13 @@ public: _containsDotsAndDollarsField = containsDotsAndDollarsField; } + bool bypassEmptyTsReplacement() const { + return _bypassEmptyTsReplacement; + } + void setBypassEmptyTsReplacement(bool bypassEmptyTsReplacement) { + _bypassEmptyTsReplacement = bypassEmptyTsReplacement; + } + /** * Serialize the update expression to Value. Output of this method is expected to, when parsed, * produce a logically equivalent update expression. @@ -238,6 +245,8 @@ private: // True if this update comes from an oplog application. bool _fromOplogApplication = false; + bool _bypassEmptyTsReplacement = false; + // True if this update is guaranteed not to contain dots or dollars fields and should skip the // check. bool _skipDotsDollarsCheck = false; diff --git a/src/mongo/db/update/update_driver_test.cpp b/src/mongo/db/update/update_driver_test.cpp index 2d03e4b7886..86f6ae492bf 100644 --- a/src/mongo/db/update/update_driver_test.cpp +++ b/src/mongo/db/update/update_driver_test.cpp @@ -46,15 +46,6 @@ #include "mongo/db/update_index_data.h" #include "mongo/unittest/unittest.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - ::mongo::str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString(); \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace mongo { namespace { diff --git a/src/mongo/db/views/resolved_view.h b/src/mongo/db/views/resolved_view.h index bdceb93ac8e..a68ed1d5b1e 100644 --- a/src/mongo/db/views/resolved_view.h +++ b/src/mongo/db/views/resolved_view.h @@ -80,6 +80,10 @@ public: return _defaultCollation; } + bool timeseries() const { + return _timeseriesOptions.has_value(); + } + // ErrorExtraInfo API static constexpr auto code = ErrorCodes::CommandOnShardedViewNotSupportedOnMongod; static constexpr StringData kTimeseriesMayContainMixedData = "timeseriesMayContainMixedData"_sd; diff --git a/src/mongo/dbtests/cursor_manager_test.cpp b/src/mongo/dbtests/cursor_manager_test.cpp index f71e45fe2ec..52d543e9157 100644 --- a/src/mongo/dbtests/cursor_manager_test.cpp +++ b/src/mongo/dbtests/cursor_manager_test.cpp @@ -708,6 +708,33 @@ TEST_F(CursorManagerTestCustomOpCtx, MultipleCursorsMultipleOperationKeys) { ASSERT(cursors.find(cursor2) != cursors.end()); } +TEST_F(CursorManagerTestCustomOpCtx, MultipleCursorsSameOperationKey) { + auto opKey = UUID::gen(); + + auto opCtx = _queryServiceContext->makeOperationContext(); + opCtx->setOperationKey(opKey); + auto cursor1 = makeCursor(opCtx.get()).getCursor()->cursorid(); + auto cursor2 = makeCursor(opCtx.get()).getCursor()->cursorid(); + + // Retrieve cursors for operation key - should be both cursors. + auto cursors = useCursorManager()->getCursorsForOpKeys({opKey}); + ASSERT_EQ(cursors.size(), size_t(2)); + ASSERT(cursors.find(cursor1) != cursors.end()); + ASSERT(cursors.find(cursor2) != cursors.end()); + + // Now delete first one. The other should remain. + ASSERT_OK(useCursorManager()->killCursor(opCtx.get(), cursor1)); + cursors = useCursorManager()->getCursorsForOpKeys({opKey}); + ASSERT_EQ(cursors.size(), size_t(1)); + ASSERT(cursors.find(cursor1) == cursors.end()); + ASSERT(cursors.find(cursor2) != cursors.end()); + + // Now delete the other. None should remain. + ASSERT_OK(useCursorManager()->killCursor(opCtx.get(), cursor2)); + cursors = useCursorManager()->getCursorsForOpKeys({opKey}); + ASSERT_EQ(cursors.size(), size_t(0)); +} + TEST_F(CursorManagerTestCustomOpCtx, TimedOutCursorShouldNotBeReturnedForOpKeyLookup) { auto opKey = UUID::gen(); auto opCtx = _queryServiceContext->makeOperationContext(); diff --git a/src/mongo/dbtests/documentsourcetests.cpp b/src/mongo/dbtests/documentsourcetests.cpp index ca5d8b519d1..08349d40c15 100644 --- a/src/mongo/dbtests/documentsourcetests.cpp +++ b/src/mongo/dbtests/documentsourcetests.cpp @@ -240,7 +240,7 @@ TEST_F(DocumentSourceCursorTest, SerializationQueryPlannerExplainLevel) { ctx()->explain = verb; createSource(); - auto explainResult = source()->serialize(verb); + auto explainResult = source()->serialize(SerializationOptions{boost::make_optional(verb)}); ASSERT_FALSE(explainResult["$cursor"]["queryPlanner"].missing()); ASSERT_TRUE(explainResult["$cursor"]["executionStats"].missing()); @@ -255,7 +255,7 @@ TEST_F(DocumentSourceCursorTest, SerializationExecStatsExplainLevel) { // Execute the plan so that the source populates its internal execution stats. exhaustCursor(); - auto explainResult = source()->serialize(verb); + auto explainResult = source()->serialize(SerializationOptions{boost::make_optional(verb)}); ASSERT_FALSE(explainResult["$cursor"]["queryPlanner"].missing()); ASSERT_FALSE(explainResult["$cursor"]["executionStats"].missing()); ASSERT_TRUE(explainResult["$cursor"]["executionStats"]["allPlansExecution"].missing()); @@ -271,7 +271,8 @@ TEST_F(DocumentSourceCursorTest, SerializationExecAllPlansExplainLevel) { // Execute the plan so that the source populates its internal executionStats. exhaustCursor(); - auto explainResult = source()->serialize(verb).getDocument(); + auto explainResult = + source()->serialize(SerializationOptions{boost::make_optional(verb)}).getDocument(); ASSERT_FALSE(explainResult["$cursor"]["queryPlanner"].missing()); ASSERT_FALSE(explainResult["$cursor"]["executionStats"].missing()); ASSERT_FALSE(explainResult["$cursor"]["executionStats"]["allPlansExecution"].missing()); @@ -288,7 +289,8 @@ TEST_F(DocumentSourceCursorTest, ExpressionContextAndSerializeVerbosityMismatch) // Execute the plan so that the source populates its internal executionStats. exhaustCursor(); - ASSERT_THROWS_CODE(source()->serialize(verb2), DBException, 50660); + ASSERT_THROWS_CODE( + source()->serialize(SerializationOptions{boost::make_optional(verb2)}), DBException, 50660); } TEST_F(DocumentSourceCursorTest, TailableAwaitDataCursorShouldErrorAfterTimeout) { diff --git a/src/mongo/dbtests/extensions_callback_real_test.cpp b/src/mongo/dbtests/extensions_callback_real_test.cpp index a2d88117969..9475d0e2779 100644 --- a/src/mongo/dbtests/extensions_callback_real_test.cpp +++ b/src/mongo/dbtests/extensions_callback_real_test.cpp @@ -255,13 +255,10 @@ TEST_F(ExtensionsCallbackRealTest, WhereExpressionDesugarsToExprAndInternalJs) { auto expr1 = unittest::assertGet( ExtensionsCallbackReal(&_opCtx, &_nss).parseWhere(expCtx, query1.firstElement())); - BSONObjBuilder gotMatch; - expr1->serialize(&gotMatch); - auto expectedMatch = fromjson( "{$expr: {$function: {'body': 'function() { return this.x == 10; }', 'args': " "['$$CURRENT'], 'lang': 'js', '_internalSetObjToThis': true}}}"); - ASSERT_BSONOBJ_EQ(gotMatch.obj(), expectedMatch); + ASSERT_BSONOBJ_EQ(expr1->serialize(), expectedMatch); } } diff --git a/src/mongo/dbtests/indexcatalogtests.cpp b/src/mongo/dbtests/indexcatalogtests.cpp index 7a648f654f0..d80fd3a471d 100644 --- a/src/mongo/dbtests/indexcatalogtests.cpp +++ b/src/mongo/dbtests/indexcatalogtests.cpp @@ -229,6 +229,129 @@ public: } }; +class PrepareUniqueIndexRecords : IndexCatalogTestBase { +public: + ~PrepareUniqueIndexRecords() { + auto opCtx = cc().makeOperationContext(); + AutoGetDb db{opCtx.get(), _nss.db(), LockMode::MODE_X}; + WriteUnitOfWork wuow{opCtx.get()}; + ASSERT_OK(db.getDb()->dropCollection(opCtx.get(), _nss)); + wuow.commit(); + } + + void run() { + auto opCtx = cc().makeOperationContext(); + dbtests::WriteContextForTests ctx{opCtx.get(), _nss.ns()}; + + ASSERT_OK(dbtests::createIndexFromSpec( + opCtx.get(), + _nss.ns(), + BSON(IndexDescriptor::kIndexVersionFieldName + << static_cast<int>(kIndexVersion) << IndexDescriptor::kIndexNameFieldName << "a_1" + << IndexDescriptor::kKeyPatternFieldName << BSON("a" << 1) + << IndexDescriptor::kPrepareUniqueFieldName << true))); + + AutoGetCollection coll{opCtx.get(), _nss, LockMode::MODE_X}; + auto doc1 = BSON("_id" << 1 << "a" << 1); + auto doc2 = BSON("_id" << 2 << "a" << 1); + + { + WriteUnitOfWork wuow{opCtx.get()}; + ASSERT_OK(indexCatalog(opCtx.get()) + ->indexRecords(opCtx.get(), *coll, {{RecordId{1}, {}, &doc1}}, nullptr)); + wuow.commit(); + } + + { + WriteUnitOfWork wuow{opCtx.get()}; + ASSERT_NOT_OK( + indexCatalog(opCtx.get()) + ->indexRecords(opCtx.get(), *coll, {{RecordId{2}, {}, &doc2}}, nullptr)); + } + + opCtx->setEnforceConstraints(false); + + { + WriteUnitOfWork wuow{opCtx.get()}; + ASSERT_OK(indexCatalog(opCtx.get()) + ->indexRecords(opCtx.get(), *coll, {{RecordId{2}, {}, &doc2}}, nullptr)); + wuow.commit(); + } + } +}; + +class PrepareUniqueUpdateRecord : IndexCatalogTestBase { +public: + ~PrepareUniqueUpdateRecord() { + auto opCtx = cc().makeOperationContext(); + AutoGetDb db{opCtx.get(), _nss.db(), LockMode::MODE_X}; + WriteUnitOfWork wuow{opCtx.get()}; + ASSERT_OK(db.getDb()->dropCollection(opCtx.get(), _nss)); + wuow.commit(); + } + + void run() { + auto opCtx = cc().makeOperationContext(); + dbtests::WriteContextForTests ctx{opCtx.get(), _nss.ns()}; + + ASSERT_OK(dbtests::createIndexFromSpec( + opCtx.get(), + _nss.ns(), + BSON(IndexDescriptor::kIndexVersionFieldName + << static_cast<int>(kIndexVersion) << IndexDescriptor::kIndexNameFieldName << "a_1" + << IndexDescriptor::kKeyPatternFieldName << BSON("a" << 1) + << IndexDescriptor::kPrepareUniqueFieldName << true))); + + AutoGetCollection coll{opCtx.get(), _nss, LockMode::MODE_X}; + auto doc1 = BSON("_id" << 1 << "a" << 1); + auto doc2 = BSON("_id" << 2 << "a" << 2); + auto updatedDoc2 = BSON("_id" << 2 << "a" << 1); + + { + WriteUnitOfWork wuow{opCtx.get()}; + ASSERT_OK(indexCatalog(opCtx.get()) + ->indexRecords(opCtx.get(), + *coll, + {{RecordId{1}, {}, &doc1}, {RecordId{2}, {}, &doc2}}, + nullptr)); + wuow.commit(); + } + + { + WriteUnitOfWork wuow{opCtx.get()}; + int64_t keysInsertedOut, keysDeletedOut; + ASSERT_NOT_OK(indexCatalog(opCtx.get()) + ->updateRecord(opCtx.get(), + *coll, + doc2, + updatedDoc2, + RecordId{2}, + &keysInsertedOut, + &keysDeletedOut)); + ASSERT_EQ(keysInsertedOut, 0); + ASSERT_EQ(keysDeletedOut, 0); + } + + opCtx->setEnforceConstraints(false); + + { + WriteUnitOfWork wuow{opCtx.get()}; + int64_t keysInsertedOut, keysDeletedOut; + ASSERT_OK(indexCatalog(opCtx.get()) + ->updateRecord(opCtx.get(), + *coll, + doc2, + updatedDoc2, + RecordId{2}, + &keysInsertedOut, + &keysDeletedOut)); + ASSERT_EQ(keysInsertedOut, 1); + ASSERT_EQ(keysDeletedOut, 1); + wuow.commit(); + } + } +}; + class IndexCatalogTests : public OldStyleSuiteSpecification { public: IndexCatalogTests() : OldStyleSuiteSpecification("indexcatalogtests") {} @@ -236,6 +359,8 @@ public: add<IndexIteratorTests>(); add<IndexCatalogEntryDroppedTest>(); add<RefreshEntry>(); + add<PrepareUniqueIndexRecords>(); + add<PrepareUniqueUpdateRecord>(); } }; diff --git a/src/mongo/dbtests/jsobjtests.cpp b/src/mongo/dbtests/jsobjtests.cpp index 0c20c6f0f9b..16baa542c18 100644 --- a/src/mongo/dbtests/jsobjtests.cpp +++ b/src/mongo/dbtests/jsobjtests.cpp @@ -179,13 +179,13 @@ public: void run() { { BufBuilder b(0); - b.appendStr("foo"); + b.appendCStr("foo"); ASSERT_EQUALS(4, b.len()); ASSERT(strcmp("foo", b.buf()) == 0); } { mongo::StackBufBuilder b; - b.appendStr("foo"); + b.appendCStr("foo"); ASSERT_EQUALS(4, b.len()); ASSERT(strcmp("foo", b.buf()) == 0); } @@ -200,7 +200,7 @@ public: try { for (; written <= 64 * 1024 * 1024 + 1; ++written) // (re)alloc past the buffer 64mb limit - b.appendStr("a"); + b.appendCStr("a"); } catch (const AssertionException&) { } // assert half of max buffer size was allocated before exception is thrown @@ -505,7 +505,7 @@ public: bb << "a" << 1; BSONObj tmp = bb.asTempObj(); ASSERT(tmp.objsize() == 4 + (1 + 2 + 4) + 1); - ASSERT(tmp.valid()); + ASSERT_OK(mongo::validateBSON(tmp)); ASSERT(tmp.hasField("a")); ASSERT(!tmp.hasField("b")); ASSERT_BSONOBJ_EQ(tmp, BSON("a" << 1)); @@ -513,7 +513,7 @@ public: bb << "b" << 2; BSONObj obj = bb.obj(); ASSERT_EQUALS(obj.objsize(), 4 + (1 + 2 + 4) + (1 + 2 + 4) + 1); - ASSERT(obj.valid()); + ASSERT_OK(mongo::validateBSON(obj)); ASSERT(obj.hasField("a")); ASSERT(obj.hasField("b")); ASSERT_BSONOBJ_EQ(obj, BSON("a" << 1 << "b" << 2)); @@ -523,7 +523,7 @@ public: bb << "a" << GT << 1; BSONObj tmp = bb.asTempObj(); ASSERT(tmp.objsize() == 4 + (1 + 2 + (4 + 1 + 4 + 4 + 1)) + 1); - ASSERT(tmp.valid()); + ASSERT_OK(mongo::validateBSON(tmp)); ASSERT(tmp.hasField("a")); ASSERT(!tmp.hasField("b")); ASSERT_BSONOBJ_EQ(tmp, BSON("a" << BSON("$gt" << 1))); @@ -532,7 +532,7 @@ public: BSONObj obj = bb.obj(); ASSERT(obj.objsize() == 4 + (1 + 2 + (4 + 1 + 4 + 4 + 1)) + (1 + 2 + (4 + 1 + 4 + 4 + 1)) + 1); - ASSERT(obj.valid()); + ASSERT_OK(mongo::validateBSON(obj)); ASSERT(obj.hasField("a")); ASSERT(obj.hasField("b")); ASSERT_BSONOBJ_EQ(obj, BSON("a" << BSON("$gt" << 1) << "b" << BSON("$lt" << 2))); @@ -542,7 +542,7 @@ public: bb << "a" << 1; BSONObj tmp = bb.asTempObj(); ASSERT(tmp.objsize() == 4 + (1 + 2 + 4) + 1); - ASSERT(tmp.valid()); + ASSERT_OK(mongo::validateBSON(tmp)); ASSERT(tmp.hasField("a")); ASSERT(!tmp.hasField("b")); ASSERT_BSONOBJ_EQ(tmp, BSON("a" << 1)); @@ -554,7 +554,7 @@ public: } bb << "b" << arr.arr(); BSONObj obj = bb.obj(); - ASSERT(obj.valid()); + ASSERT_OK(mongo::validateBSON(obj)); ASSERT(obj.hasField("a")); ASSERT(obj.hasField("b")); } @@ -756,8 +756,8 @@ class Base { public: virtual ~Base() {} void run() { - ASSERT(valid().valid()); - ASSERT(!invalid().valid()); + ASSERT_OK(mongo::validateBSON(valid())); + ASSERT(!mongo::validateBSON(invalid()).isOK()); } protected: @@ -806,7 +806,7 @@ public: b.appendNull("a"); BSONObj o = b.done(); set(o, 4, mongo::Undefined); - ASSERT(o.valid()); + ASSERT_OK(mongo::validateBSON(o)); } }; @@ -990,7 +990,7 @@ public: void run() { const char data[] = {0x07, 0x00, 0x00, 0x00, char(type_), 'a', 0x00}; BSONObj o(data); - ASSERT(!o.valid()); + ASSERT(!mongo::validateBSON(o).isOK()); } private: @@ -1329,7 +1329,7 @@ public: b2.done(); b1.append("f", 10.0); BSONObj ret = b1.done(); - ASSERT(ret.valid()); + ASSERT_OK(mongo::validateBSON(ret)); ASSERT(ret.woCompare(fromjson("{a:'bcd',foo:{ggg:44},f:10}")) == 0); } }; @@ -1350,7 +1350,7 @@ public: BSONObj o = BSON("now" << DATENOW); Date_t after = jsTime(); - ASSERT(o.valid()); + ASSERT_OK(mongo::validateBSON(o)); BSONElement e = o["now"]; ASSERT(e.type() == Date); @@ -1368,7 +1368,7 @@ public: b.appendTimeT("now", aTime); BSONObj o = b.obj(); - ASSERT(o.valid()); + ASSERT_OK(mongo::validateBSON(o)); BSONElement e = o["now"]; ASSERT_EQUALS(Date, e.type()); @@ -1382,8 +1382,8 @@ public: BSONObj min = BSON("a" << MINKEY); BSONObj max = BSON("b" << MAXKEY); - ASSERT(min.valid()); - ASSERT(max.valid()); + ASSERT_OK(mongo::validateBSON(min)); + ASSERT_OK(mongo::validateBSON(max)); BSONElement minElement = min["a"]; BSONElement maxElement = max["b"]; diff --git a/src/mongo/dbtests/jsontests.cpp b/src/mongo/dbtests/jsontests.cpp index e175afbf22d..c4410a8c45e 100644 --- a/src/mongo/dbtests/jsontests.cpp +++ b/src/mongo/dbtests/jsontests.cpp @@ -625,7 +625,7 @@ void assertEquals(const std::string& json, } void checkEquivalence(const std::string& json, const BSONObj& bson) { - ASSERT(fromjson(json).valid()); + ASSERT_OK(mongo::validateBSON(fromjson(json))); assertEquals(json, bson, fromjson(json), "mode: json-to-bson"); assertEquals(json, bson, fromjson(tojson(bson)), "mode: <default>"); assertEquals(json, bson, fromjson(tojson(bson, LegacyStrict)), "mode: strict"); @@ -836,6 +836,7 @@ TEST(FromJsonTest, BinDataTypes) { {0x05, MD5Type}, {0x06, Encrypt}, {0x07, Column}, + {0x08, Sensitive}, {0x80, bdtCustom}, }; for (const auto& ts : specs) { diff --git a/src/mongo/dbtests/jstests.cpp b/src/mongo/dbtests/jstests.cpp index 9e48d81b23e..9a396d240f5 100644 --- a/src/mongo/dbtests/jstests.cpp +++ b/src/mongo/dbtests/jstests.cpp @@ -785,7 +785,7 @@ public: { BSONObjBuilder b; b.bb().appendNum(static_cast<char>(bsonTimestamp)); - b.bb().appendStr("a"); + b.bb().appendCStr("a"); b.bb().appendNum(std::numeric_limits<unsigned long long>::max()); in = b.obj(); diff --git a/src/mongo/dbtests/query_stage_update.cpp b/src/mongo/dbtests/query_stage_update.cpp index 2ae8011e9c8..1c6d2302260 100644 --- a/src/mongo/dbtests/query_stage_update.cpp +++ b/src/mongo/dbtests/query_stage_update.cpp @@ -55,15 +55,6 @@ #include "mongo/db/update/update_driver.h" #include "mongo/dbtests/dbtests.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - ::str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString(); \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace QueryStageUpdate { using std::make_unique; diff --git a/src/mongo/dbtests/querytests.cpp b/src/mongo/dbtests/querytests.cpp index f7aefecaf55..2fe9e828355 100644 --- a/src/mongo/dbtests/querytests.cpp +++ b/src/mongo/dbtests/querytests.cpp @@ -1205,7 +1205,7 @@ public: std::unique_ptr<DBClientCursor> cursor = _client.find(std::move(findRequest)); while (cursor->more()) { BSONObj o = cursor->next(); - verify(o.valid()); + verify(validateBSON(o).isOK()); } } void run() { diff --git a/src/mongo/dbtests/validate_tests.cpp b/src/mongo/dbtests/validate_tests.cpp index f4e81c264e2..f85755ac87e 100644 --- a/src/mongo/dbtests/validate_tests.cpp +++ b/src/mongo/dbtests/validate_tests.cpp @@ -184,6 +184,23 @@ protected: dumpOnErrorGuard.dismiss(); } + void ensureValidateWarned() { + ValidateResults results = runValidate(); + + ScopeGuard dumpOnErrorGuard([&] { + StorageDebugUtil::printValidateResults(results); + StorageDebugUtil::printCollectionAndIndexTableEntries(&_opCtx, _nss); + }); + + ASSERT_TRUE(results.valid) << "Validation failed when it should've worked."; + ASSERT_TRUE(results.errors.empty()) + << "Validation reported errors when it should not have."; + ASSERT_FALSE(results.warnings.empty()) + << "Validation did not report a warning when it should have."; + + dumpOnErrorGuard.dismiss(); + } + void ensureValidateFailed() { ValidateResults results = runValidate(); @@ -4243,6 +4260,205 @@ public: } }; +/** + * Validate detects duplicate keys in a secondary unique index {a: 1} when the index is + * on a clustered collection. + * Two cases are tested: + * 1. The false negative case: when validate says there isn't a uniqueness + * violation even though there is one. + * 2. The false positive case: when validate says there is a uniqueness + * violation even though there isn't one. + * + * False negative case: + * Suppose we have two documents {_id: "1000000000", a: 1} and {_id: "1000000000", a: 1} + * that live in a collection. Since they have the same value for field 'a', they violate + * the uniqueness constraint of the index. + * The key strings for index {a: 1} for the two docs look something like this. + * They map from the value of 'a' in the document to the recordId. + * Buffer for keystring1: 1,1000000000 + * Buffer for keystring2: 1,2000000000 + * + * When we compareWithoutRecordIdLong(), we chop off only the number of + * bytes used in a long before making the comparison in the buffer. Since a long + * is 8 bytes, we cut 8 characters off. + * Truncated buffer 1: 1,10 + * Truncated buffer 2: 1,20 + * + * And we can see that the two truncated buffers above still aren't equal. But instead, + * if we used compareWithoutRecordIdStr(), we first figure out how many bytes we need + * to chop to exclude the recordId, and that way only the index entry value is compared. + * Now the unique index violation can be detected, as both the truncated buffers are + * equal. + * Truncated buffer 1: 1 + * Truncated buffer 2: 1 + * + * False positive case: + * Suppose we have two documents {_id: "1", a: 10000001} and {_id: "2", a: 10000002}. + * Clearly they don't violate any constraints. However it is possible, if we truncate + * more bytes than necessary, that we will end up truncating some of the bytes of the + * field 'a'. For example, + * Pre-truncation: + * Buffer for keystring1: 10000001,1 + * Buffer for keystring2: 10000002,2 + * Post-truncation: + * Buffer for keystring1: 10000 + * Buffer for keystring2: 10000 + * This can lead to a false positive uniqueness violation. + */ +template <bool falsePositiveCase> +class ValidateDuplicateKeyOnClusteredCollection : public ValidateBase { +public: + ValidateDuplicateKeyOnClusteredCollection() + : ValidateBase(/*full=*/true, /*background=*/false, /*clustered=*/true) {} + + void run() { + // Cannot run validate with {background:true} if the storage engine does not support + // checkpoints. + if (_background && !_supportsBackgroundValidation) { + return; + } + + SharedBufferFragmentBuilder pooledBuilder( + KeyString::HeapBuilder::kHeapAllocatorDefaultBytes); + + lockDb(MODE_X); + ASSERT(coll()); + + // Create a unique index on {a: 1} + const auto indexName = "a"; + const auto indexKey = BSON("a" << 1); + auto status = dbtests::createIndexFromSpec( + &_opCtx, + coll()->ns().ns(), + BSON("name" << indexName << "key" << indexKey << "v" << static_cast<int>(kIndexVersion) + << "unique" << true)); + ASSERT_OK(status); + + + // Insert documents. + auto firstDoc = BSON("_id" + << "1000000000000" + << "a" << 1); + auto secondDoc = BSON("_id" + << "2000000000000" + << "a" << 1); + if (falsePositiveCase) { + firstDoc = BSON("_id" + << "1" + << "a" << 10000001); + secondDoc = BSON("_id" + << "2" + << "a" << 10000002); + } + OpDebug* const nullOpDebug = nullptr; + lockDb(MODE_X); + { + WriteUnitOfWork wunit(&_opCtx); + ASSERT_OK( + coll()->insertDocument(&_opCtx, InsertStatement(firstDoc), nullOpDebug, true)); + if (falsePositiveCase) { + ASSERT_OK( + coll()->insertDocument(&_opCtx, InsertStatement(secondDoc), nullOpDebug, true)); + } + wunit.commit(); + } + releaseDb(); + ensureValidateWorked(); + + // Insert a document with a duplicate key for "a". + if (!falsePositiveCase) { + lockDb(MODE_X); + + const IndexCatalog* indexCatalog = coll()->getIndexCatalog(); + + InsertDeleteOptions options; + options.dupsAllowed = true; + + WriteUnitOfWork wunit(&_opCtx); + + // Insert a record and its keys separately. We do this to bypass duplicate constraint + // checking. Inserting a record and all of its keys ensures that validation fails + // because there are duplicate keys, and not just because there are keys without + // corresponding records. + auto swRecordId = + coll()->getRecordStore()->insertRecord(&_opCtx, + record_id_helpers::keyForObj(secondDoc), + secondDoc.objdata(), + secondDoc.objsize(), + Timestamp()); + ASSERT_OK(swRecordId); + wunit.commit(); + + // Insert the key on "a". + { + auto descriptor = indexCatalog->findIndexByName(&_opCtx, indexName); + auto entry = const_cast<IndexCatalogEntry*>(indexCatalog->getEntry(descriptor)); + auto iam = entry->accessMethod()->asSortedData(); + auto interceptor = std::make_unique<IndexBuildInterceptor>(&_opCtx, entry); + + KeyStringSet keys; + iam->getKeys(&_opCtx, + coll(), + pooledBuilder, + secondDoc, + InsertDeleteOptions::ConstraintEnforcementMode::kRelaxConstraints, + SortedDataIndexAccessMethod::GetKeysContext::kAddingKeys, + &keys, + nullptr, + nullptr, + swRecordId.getValue()); + ASSERT_EQ(1, keys.size()); + + { + WriteUnitOfWork wunit(&_opCtx); + + int64_t numInserted; + auto insertStatus = iam->insertKeysAndUpdateMultikeyPaths( + &_opCtx, + coll(), + {keys.begin(), keys.end()}, + {}, + MultikeyPaths{}, + options, + [this, &interceptor](const KeyString::Value& duplicateKey) { + return interceptor->recordDuplicateKey(&_opCtx, duplicateKey); + }, + &numInserted); + + ASSERT_EQUALS(numInserted, 1); + ASSERT_OK(insertStatus); + + wunit.commit(); + } + + ASSERT_NOT_OK(interceptor->checkDuplicateKeyConstraints(&_opCtx)); + } + + releaseDb(); + } + + ValidateResults results = runValidate(); + + ScopeGuard dumpOnErrorGuard([&] { + StorageDebugUtil::printValidateResults(results); + StorageDebugUtil::printCollectionAndIndexTableEntries(&_opCtx, coll()->ns()); + }); + + if (falsePositiveCase) { + ASSERT(results.valid) << "Validation failed when it should have worked."; + ASSERT_EQ(static_cast<size_t>(0), results.errors.size()); + } else { + ASSERT_FALSE(results.valid) << "Validation worked when it should have failed."; + ASSERT_EQ(static_cast<size_t>(1), results.errors.size()); + } + ASSERT_EQ(static_cast<size_t>(0), omitTransientWarningsFromCount(results)); + ASSERT_EQ(static_cast<size_t>(0), results.extraIndexEntries.size()); + ASSERT_EQ(static_cast<size_t>(0), results.missingIndexEntries.size()); + + dumpOnErrorGuard.dismiss(); + } +}; + class ValidateRepairOnClusteredCollection : public ValidateBase { public: ValidateRepairOnClusteredCollection() @@ -4568,6 +4784,8 @@ public: add<ValidateInvalidBSONOnClusteredCollection<true>>(); add<ValidateReportInfoOnClusteredCollection<false>>(); add<ValidateReportInfoOnClusteredCollection<true>>(); + add<ValidateDuplicateKeyOnClusteredCollection<true /*falsePositiveCase*/>>(); + add<ValidateDuplicateKeyOnClusteredCollection<false /*falsePositiveCase*/>>(); add<ValidateRepairOnClusteredCollection>(); add<ValidateInvalidRecordIdOnClusteredCollection<false>>(false /*withSecondaryIndex*/); diff --git a/src/mongo/embedded/mongo_embedded/SConscript b/src/mongo/embedded/mongo_embedded/SConscript index 8f84ee065d5..7a5f2043d6b 100644 --- a/src/mongo/embedded/mongo_embedded/SConscript +++ b/src/mongo/embedded/mongo_embedded/SConscript @@ -94,6 +94,7 @@ if get_option('link-model') != 'dynamic-sdk': ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/commands/test_commands_enabled', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/rpc/message', diff --git a/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp b/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp index 0e5c1a9dadf..4a9945875f9 100644 --- a/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp +++ b/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp @@ -35,6 +35,7 @@ #include <yaml-cpp/yaml.h> #include "mongo/base/initializer.h" +#include "mongo/bson/bson_validate.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/commands/test_commands_enabled.h" #include "mongo/db/json.h" @@ -194,7 +195,7 @@ protected: // convert the message into an OpMessage to examine its BSON auto outputOpMsg = mongo::OpMsg::parseOwned(outputMessage); - ASSERT(outputOpMsg.body.valid()); + ASSERT_OK(mongo::validateBSON(outputOpMsg.body)); return outputOpMsg.body; } @@ -407,7 +408,7 @@ TEST_F(MongodbCAPITest, ReadDB) { auto outputBSON = performRpc(client, findMsg); - ASSERT(outputBSON.valid()); + ASSERT_OK(mongo::validateBSON(outputBSON)); ASSERT(outputBSON.hasField("cursor")); ASSERT(outputBSON.getField("cursor").embeddedObject().hasField("firstBatch")); mongo::BSONObj arrObj = @@ -432,7 +433,7 @@ TEST_F(MongodbCAPITest, InsertAndRead) { "{insert: 'collection_name', documents: [{firstName: 'Mongo', lastName: 'DB', age: 10}]}"); auto insertOpMsg = mongo::OpMsgRequest::fromDBAndBody("db_name", insertObj); auto outputBSON1 = performRpc(client, insertOpMsg); - ASSERT(outputBSON1.valid()); + ASSERT_OK(mongo::validateBSON(outputBSON1)); ASSERT(outputBSON1.hasField("n")); ASSERT(outputBSON1.getIntField("n") == 1); ASSERT(outputBSON1.hasField("ok")); @@ -441,7 +442,7 @@ TEST_F(MongodbCAPITest, InsertAndRead) { mongo::BSONObj findObj = mongo::fromjson("{find: 'collection_name', limit: 1}"); auto findMsg = mongo::OpMsgRequest::fromDBAndBody("db_name", findObj); auto outputBSON2 = performRpc(client, findMsg); - ASSERT(outputBSON2.valid()); + ASSERT_OK(mongo::validateBSON(outputBSON2)); ASSERT(outputBSON2.hasField("cursor")); ASSERT(outputBSON2.getField("cursor").embeddedObject().hasField("firstBatch")); mongo::BSONObj arrObj = @@ -467,7 +468,7 @@ TEST_F(MongodbCAPITest, InsertAndReadDifferentClients) { "{insert: 'collection_name', documents: [{firstName: 'Mongo', lastName: 'DB', age: 10}]}"); auto insertOpMsg = mongo::OpMsgRequest::fromDBAndBody("db_name", insertObj); auto outputBSON1 = performRpc(client1, insertOpMsg); - ASSERT(outputBSON1.valid()); + ASSERT_OK(mongo::validateBSON(outputBSON1)); ASSERT(outputBSON1.hasField("n")); ASSERT(outputBSON1.getIntField("n") == 1); ASSERT(outputBSON1.hasField("ok")); @@ -476,7 +477,7 @@ TEST_F(MongodbCAPITest, InsertAndReadDifferentClients) { mongo::BSONObj findObj = mongo::fromjson("{find: 'collection_name', limit: 1}"); auto findMsg = mongo::OpMsgRequest::fromDBAndBody("db_name", findObj); auto outputBSON2 = performRpc(client2, findMsg); - ASSERT(outputBSON2.valid()); + ASSERT_OK(mongo::validateBSON(outputBSON2)); ASSERT(outputBSON2.hasField("cursor")); ASSERT(outputBSON2.getField("cursor").embeddedObject().hasField("firstBatch")); mongo::BSONObj arrObj = @@ -501,7 +502,7 @@ TEST_F(MongodbCAPITest, InsertAndDelete) { "age: 10}]}"); auto insertOpMsg = mongo::OpMsgRequest::fromDBAndBody("db_name", insertObj); auto outputBSON1 = performRpc(client, insertOpMsg); - ASSERT(outputBSON1.valid()); + ASSERT_OK(mongo::validateBSON(outputBSON1)); ASSERT(outputBSON1.hasField("n")); ASSERT(outputBSON1.getIntField("n") == 1); ASSERT(outputBSON1.hasField("ok")); @@ -514,7 +515,7 @@ TEST_F(MongodbCAPITest, InsertAndDelete) { "1}]}"); auto deleteOpMsg = mongo::OpMsgRequest::fromDBAndBody("db_name", deleteObj); auto outputBSON2 = performRpc(client, deleteOpMsg); - ASSERT(outputBSON2.valid()); + ASSERT_OK(mongo::validateBSON(outputBSON2)); ASSERT(outputBSON2.hasField("n")); ASSERT(outputBSON2.getIntField("n") == 1); ASSERT(outputBSON2.hasField("ok")); @@ -530,7 +531,7 @@ TEST_F(MongodbCAPITest, InsertAndUpdate) { "age: 10}]}"); auto insertOpMsg = mongo::OpMsgRequest::fromDBAndBody("db_name", insertObj); auto outputBSON1 = performRpc(client, insertOpMsg); - ASSERT(outputBSON1.valid()); + ASSERT_OK(mongo::validateBSON(outputBSON1)); ASSERT(outputBSON1.hasField("n")); ASSERT(outputBSON1.getIntField("n") == 1); ASSERT(outputBSON1.hasField("ok")); @@ -543,7 +544,7 @@ TEST_F(MongodbCAPITest, InsertAndUpdate) { "{age: 5}}}]}"); auto updateOpMsg = mongo::OpMsgRequest::fromDBAndBody("db_name", updateObj); auto outputBSON2 = performRpc(client, updateOpMsg); - ASSERT(outputBSON2.valid()); + ASSERT_OK(mongo::validateBSON(outputBSON2)); ASSERT(outputBSON2.hasField("ok")); ASSERT(outputBSON2.getField("ok").numberDouble() == 1.0); ASSERT(outputBSON2.hasField("nModified")); diff --git a/src/mongo/executor/network_interface_tl.cpp b/src/mongo/executor/network_interface_tl.cpp index 0f23b2f61d7..9d7a4129ca1 100644 --- a/src/mongo/executor/network_interface_tl.cpp +++ b/src/mongo/executor/network_interface_tl.cpp @@ -1021,10 +1021,15 @@ void NetworkInterfaceTL::ExhaustCommandState::continueExhaustRequest( } auto onAnyResponse = RemoteCommandOnAnyResponse(requestState->host, response); - if (!catchingInvoke([&] { doMetadataHook(onAnyResponse); }, - [&](Status& err) { finalResponsePromise.setError(err); }, - "Exhaust command metadata hook readReplyMetadata")) + try { + doMetadataHook(onAnyResponse); + } catch (const DBException& ex) { + Status err = ex.toStatus(); + LOGV2( + 9183100, "Exhaust command metadata hook readReplyMetadata failed", "error"_attr = err); + finalResponsePromise.setError(err); return; + } // If the command failed, we will call 'onReply' as a part of the future chain paired with // the promise. This is to be sure that all error paths will run 'onReply' only once upon @@ -1037,20 +1042,28 @@ void NetworkInterfaceTL::ExhaustCommandState::continueExhaustRequest( return; } - if (!catchingInvoke([&] { onReplyFn(onAnyResponse); }, - [&](Status& err) { finalResponsePromise.setError(err); }, - "Exhaust command onReplyFn")) + try { + onReplyFn(onAnyResponse); + } catch (const DBException& ex) { + Status err = ex.toStatus(); + LOGV2(9183101, "Exhaust command onReplyFn failed", "error"_attr = err); + finalResponsePromise.setError(err); return; + } // Reset the stopwatch to measure the correct duration for the following reply stopwatch.restart(); if (deadline != kNoExpirationDate) { deadline = stopwatch.start() + requestOnAny.timeout; } - if (!catchingInvoke([&] { setTimer(); }, - [&](Status& err) { finalResponsePromise.setError(err); }, - "Exhaust command setTimer")) + try { + setTimer(); + } catch (const DBException& ex) { + Status err = ex.toStatus(); + LOGV2(9183102, "Exhaust command setTimer failed", "error"_attr = err); + finalResponsePromise.setError(err); return; + } requestState->getClient(requestState->conn) ->awaitExhaustCommand(baton) diff --git a/src/mongo/idl/basic_types.h b/src/mongo/idl/basic_types.h index 4b77f57461d..4f93115bf0c 100644 --- a/src/mongo/idl/basic_types.h +++ b/src/mongo/idl/basic_types.h @@ -127,6 +127,11 @@ private: boost::optional<bool> _value; }; +template <typename H> +H AbslHashValue(H h, const OptionalBool& optBool) { + return H::combine(std::move(h), optBool.has_value(), bool(optBool)); +} + /** * Class to represent a BSON element with any type from IDL. The caller must ensure that the backing * BSON stays alive while this type is in use. diff --git a/src/mongo/idl/basic_types.idl b/src/mongo/idl/basic_types.idl index 634b05d9539..e41c18b7714 100644 --- a/src/mongo/idl/basic_types.idl +++ b/src/mongo/idl/basic_types.idl @@ -163,6 +163,13 @@ types: cpp_type: "std::vector<std::uint8_t>" deserializer: "mongo::BSONElement::_binDataVector" + bindata_sensitive: + bson_serialization_type: bindata + bindata_subtype: sensitive + description: "A BSON bindata of sensitive sub type" + cpp_type: "std::vector<std::uint8_t>" + deserializer: "mongo::BSONElement::_binDataVector" + uuid: bson_serialization_type: bindata bindata_subtype: uuid diff --git a/src/mongo/idl/server_parameter_test_util.h b/src/mongo/idl/server_parameter_test_util.h index 566489dd1a9..f7e2ae31834 100644 --- a/src/mongo/idl/server_parameter_test_util.h +++ b/src/mongo/idl/server_parameter_test_util.h @@ -36,31 +36,28 @@ namespace mongo { /** - * Test-only RAII type that allows to set a server parameter value during the execution of a - * unit test, or part of a unit test, and resets it to the original value on destruction. + * Test-only class that sets a server parameter to the specified value and allows + * resetting after the test completes. */ -class RAIIServerParameterControllerForTest { +class ServerParameterControllerForTest { public: /** * Constructor setting the server parameter to the specified value. */ template <typename T> - RAIIServerParameterControllerForTest(const std::string& name, T value) + ServerParameterControllerForTest(const std::string& name, T value) : _serverParam(ServerParameterSet::getNodeParameterSet()->get(name)) { - // Save the old value + // Save the old value. BSONObjBuilder bob; _serverParam->appendSupportingRoundtrip(nullptr, bob, name); _oldValue = bob.obj(); - // Set to the new value + // Set server param to the new value. uassertStatusOK(_serverParam->set(BSON(name << value).firstElement())); } - /** - * Destructor resetting the server parameter to the original value. - */ - ~RAIIServerParameterControllerForTest() { - // Reset to the old value + void reset() { + // Reset to the old value. auto elem = _oldValue.firstElement(); uassertStatusOK(_serverParam->set(elem)); } @@ -70,4 +67,28 @@ private: BSONObj _oldValue; }; +/** + * Test-only RAII type that wraps ServerParameterControllerForTest. Upon destruction, the server + * parameter will be set to its original value. + */ +class RAIIServerParameterControllerForTest { +public: + /** + * Constructor setting the server parameter to the specified value. + */ + template <typename T> + RAIIServerParameterControllerForTest(const std::string& name, T value) + : _serverParamController(ServerParameterControllerForTest(name, value)) {} + + /** + * Destructor resetting the server parameter to the original value. + */ + ~RAIIServerParameterControllerForTest() { + _serverParamController.reset(); + } + +private: + ServerParameterControllerForTest _serverParamController; +}; + } // namespace mongo diff --git a/src/mongo/logv2/log_capture_backend.h b/src/mongo/logv2/log_capture_backend.h index 701ec62a9d4..4078934fb4c 100644 --- a/src/mongo/logv2/log_capture_backend.h +++ b/src/mongo/logv2/log_capture_backend.h @@ -37,31 +37,44 @@ #include <string> #include <vector> +#include "mongo/platform/atomic_word.h" + namespace mongo::logv2 { + +/* + * LogLineListener is a wrapper class used in the LogCaptureBackend that defines what to do with + * log lines upon consumption. + */ +class LogLineListener { +public: + virtual ~LogLineListener() = default; + virtual void accept(const std::string& line) = 0; +}; + class LogCaptureBackend : public boost::log::sinks:: - basic_formatted_sink_backend<char, boost::log::sinks::synchronized_feeding> { + basic_formatted_sink_backend<char, boost::log::sinks::concurrent_feeding> { public: - LogCaptureBackend(std::vector<std::string>& lines, bool stripEol) - : _stripEol(stripEol), _logLines(lines) {} + LogCaptureBackend(std::unique_ptr<LogLineListener> logListener, bool stripEol) + : _logListener{std::move(logListener)}, _stripEol(stripEol) {} - static boost::shared_ptr<boost::log::sinks::synchronous_sink<LogCaptureBackend>> create( - std::vector<std::string>& lines, bool stripEol) { - return boost::make_shared<boost::log::sinks::synchronous_sink<LogCaptureBackend>>( - boost::make_shared<LogCaptureBackend>(lines, stripEol)); + static boost::shared_ptr<boost::log::sinks::unlocked_sink<LogCaptureBackend>> create( + std::unique_ptr<LogLineListener> logListener, bool stripEol) { + return boost::make_shared<boost::log::sinks::unlocked_sink<LogCaptureBackend>>( + boost::make_shared<LogCaptureBackend>(std::move(logListener), stripEol)); } void consume(boost::log::record_view const& rec, string_type const& formatted_string) { - if (_stripEol && !formatted_string.empty() && + if (_stripEol.load() && !formatted_string.empty() && formatted_string[formatted_string.size() - 1] == '\n') { - _logLines.push_back(formatted_string.substr(0, formatted_string.size() - 1)); + _logListener->accept(formatted_string.substr(0, formatted_string.size() - 1)); } else { - _logLines.push_back(formatted_string); + _logListener->accept(formatted_string); } } private: - bool _stripEol; - std::vector<std::string>& _logLines; + std::unique_ptr<LogLineListener> _logListener; + AtomicWord<bool> _stripEol; }; } // namespace mongo::logv2 diff --git a/src/mongo/logv2/log_component.h b/src/mongo/logv2/log_component.h index 2b1985756b9..90aced1c07a 100644 --- a/src/mongo/logv2/log_component.h +++ b/src/mongo/logv2/log_component.h @@ -61,6 +61,7 @@ namespace mongo::logv2 { X(kNetwork, , "network" , "NETWORK" , kDefault) \ X(kProcessHealth, , "processHealth" , "HEALTH" , kDefault) \ X(kQuery, , "query" , "QUERY" , kDefault) \ + X(kQueryStats, , "queryStats" , "QRYSTATS", kDefault) \ X(kReplication, , "replication" , "REPL" , kDefault) \ X(kReplicationElection, , "election" , "ELECTION", kReplication) \ X(kReplicationHeartbeats, , "heartbeats" , "REPL_HB" , kReplication) \ diff --git a/src/mongo/logv2/logv2_test.cpp b/src/mongo/logv2/logv2_test.cpp index 2477d27033b..899440b4fba 100644 --- a/src/mongo/logv2/logv2_test.cpp +++ b/src/mongo/logv2/logv2_test.cpp @@ -64,6 +64,7 @@ #include "mongo/unittest/death_test.h" #include "mongo/unittest/temp_dir.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/shared_buffer.h" #include "mongo/util/str_escape.h" #include "mongo/util/string_map.h" #include "mongo/util/uuid.h" @@ -192,10 +193,21 @@ void applyDefaultFilterToSink(SinkPtr&& sink) { sink->set_filter(ComponentSettingsFilter(mgr().getGlobalDomain(), mgr().getGlobalSettings())); } +class Listener : public logv2::LogLineListener { +public: + explicit Listener(synchronized_value<std::vector<std::string>>* sv) : _sv(sv) {} + void accept(const std::string& line) override { + (***_sv).push_back(line); + } + +private: + synchronized_value<std::vector<std::string>>* _sv; +}; + class LogDuringInitShutdownTester { public: LogDuringInitShutdownTester() { - auto sink = LogCaptureBackend::create(lines, true); + auto sink = LogCaptureBackend::create(std::make_unique<Listener>(&syncedLines), true); applyDefaultFilterToSink(sink); // We have to leave this sink installed as it is not allowed to install sinks during // shutdown. Add a filter so it is only used during this test. @@ -205,15 +217,15 @@ public: ScopeGuard enabledGuard([this] { enabled = false; }); LOGV2(20001, "log during init"); - ASSERT_EQUALS(lines.back(), "log during init"); + ASSERT_EQUALS((**syncedLines).back(), "log during init"); } ~LogDuringInitShutdownTester() { enabled = true; LOGV2(4600800, "log during shutdown"); - ASSERT_EQUALS(lines.back(), "log during shutdown"); + ASSERT_EQUALS((**syncedLines).back(), "log during shutdown"); } - std::vector<std::string> lines; + synchronized_value<std::vector<std::string>> syncedLines; bool enabled = true; }; @@ -225,28 +237,30 @@ public: public: LineCapture() = delete; LineCapture(bool stripEol) - : _lines{std::make_unique<std::vector<std::string>>()}, - _sink{LogCaptureBackend::create(*_lines, stripEol)} {} - auto& lines() { - return *_lines; + : _syncedLines{synchronized_value<std::vector<std::string>>()}, + _sink{ + LogCaptureBackend::create(std::make_unique<Listener>(&_syncedLines), stripEol)} {} + auto lines() { + return **_syncedLines; } auto& sink() { return _sink; } - const std::string& back() const { - ASSERT_GT(_lines->size(), 0); - return _lines->back(); + std::string back() const { + auto logLinesLockGuard = *_syncedLines; + ASSERT_GT(logLinesLockGuard->size(), 0); + return logLinesLockGuard->back(); } void clear() { - return _lines->clear(); + return (**_syncedLines).clear(); } size_t size() const { - return _lines->size(); + return (**_syncedLines).size(); } private: - std::unique_ptr<std::vector<std::string>> _lines; - boost::shared_ptr<boost::log::sinks::synchronous_sink<LogCaptureBackend>> _sink; + synchronized_value<std::vector<std::string>> _syncedLines; + boost::shared_ptr<boost::log::sinks::unlocked_sink<LogCaptureBackend>> _sink; }; LogV2Test() { @@ -287,9 +301,9 @@ public: } template <typename Fmt> - LineCapture makeLineCapture(Fmt&& formatter, bool stripEol = true) { - LineCapture ret(stripEol); - auto& s = ret.sink(); + std::unique_ptr<LineCapture> makeLineCapture(Fmt&& formatter, bool stripEol = true) { + auto ret = std::make_unique<LineCapture>(stripEol); + auto& s = ret->sink(); applyDefaultFilterToSink(s); s->set_formatter(std::forward<Fmt>(formatter)); attachSink(s); @@ -307,60 +321,60 @@ TEST_F(LogV2Test, Basic) { fmt::memory_buffer buffer; LOGV2(20002, "test"); - ASSERT_EQUALS(lines.back(), "test"); + ASSERT_EQUALS(lines->back(), "test"); LOGV2_DEBUG(20063, -2, "test debug"); - ASSERT_EQUALS(lines.back(), "test debug"); + ASSERT_EQUALS(lines->back(), "test debug"); LOGV2(20003, "test {name}", "name"_attr = 1); - ASSERT_EQUALS(lines.back(), "test 1"); + ASSERT_EQUALS(lines->back(), "test 1"); LOGV2(20004, "test {name:d}", "name"_attr = 2); - ASSERT_EQUALS(lines.back(), "test 2"); + ASSERT_EQUALS(lines->back(), "test 2"); LOGV2(20005, "test {name}", "name"_attr = "char*"); - ASSERT_EQUALS(lines.back(), "test char*"); + ASSERT_EQUALS(lines->back(), "test char*"); LOGV2(20006, "test {name}", "name"_attr = std::string("std::string")); - ASSERT_EQUALS(lines.back(), "test std::string"); + ASSERT_EQUALS(lines->back(), "test std::string"); LOGV2(20007, "test {name}", "name"_attr = "StringData"_sd); - ASSERT_EQUALS(lines.back(), "test StringData"); + ASSERT_EQUALS(lines->back(), "test StringData"); LOGV2_OPTIONS(20064, {LogTag::kStartupWarnings}, "test"); - ASSERT_EQUALS(lines.back(), "test"); + ASSERT_EQUALS(lines->back(), "test"); TypeWithBSON t(1.0, 2.0); LOGV2(20008, "{name} custom formatting", "name"_attr = t); - ASSERT_EQUALS(lines.back(), t.toString() + " custom formatting"); + ASSERT_EQUALS(lines->back(), t.toString() + " custom formatting"); TypeWithoutBSON t2(1.0, 2.0); LOGV2(20009, "{name} custom formatting, no bson", "name"_attr = t2); - ASSERT_EQUALS(lines.back(), t.toString() + " custom formatting, no bson"); + ASSERT_EQUALS(lines->back(), t.toString() + " custom formatting, no bson"); TypeWithOnlyStringSerialize t3(1.0, 2.0); LOGV2(20010, "{name}", "name"_attr = t3); buffer.clear(); t3.serialize(buffer); - ASSERT_EQUALS(lines.back(), fmt::to_string(buffer)); + ASSERT_EQUALS(lines->back(), fmt::to_string(buffer)); // Serialize should be preferred when both are available TypeWithBothStringFormatters t4; LOGV2(20011, "{name}", "name"_attr = t4); buffer.clear(); t4.serialize(buffer); - ASSERT_EQUALS(lines.back(), fmt::to_string(buffer)); + ASSERT_EQUALS(lines->back(), fmt::to_string(buffer)); // Message string is selected when using API that also take a format string LOGV2(20084, "fmtstr {name}", "msgstr", "name"_attr = 1); - ASSERT_EQUALS(lines.back(), "msgstr"); + ASSERT_EQUALS(lines->back(), "msgstr"); // Test that logging exceptions does not propagate out to user code in release builds if (!kDebugBuild) { LOGV2(4638203, "mismatch {name}", "not_name"_attr = 1); - ASSERT(StringData(lines.back()).startsWith("Exception during log"_sd)); + ASSERT(StringData(lines->back()).startsWith("Exception during log"_sd)); } -} +} // namespace namespace bl_sinks = boost::log::sinks; // Sink backend which will grab a mutex, then immediately segfault. @@ -414,7 +428,7 @@ public: template <typename T> void validateJSON(T expected) { namespace pt = boost::property_tree; - std::istringstream json_stream(json.back()); + std::istringstream json_stream(json->back()); pt::ptree ptree; pt::json_parser::read_json(json_stream, ptree); ASSERT_EQUALS(ptree.get<std::string>(std::string(kTenantFieldName)), tenant.toString()); @@ -422,15 +436,20 @@ public: } auto lastBSONElement() { - ASSERT_EQUALS(BSONObj(bson.back().data()).getField(kTenantFieldName).str(), - tenant.toString()); - return BSONObj(bson.back().data()).getField(kAttributesFieldName).Obj().getField("name"_sd); + auto str = bson->back(); + buf.realloc(str.size()); + str.copy(buf.get(), str.size()); + BSONObj obj(buf); + + ASSERT_EQUALS(obj.getField(kTenantFieldName).str(), tenant.toString()); + return obj.getField(kAttributesFieldName).Obj().getField("name"_sd); } TenantId tenant = TenantId(OID::gen()); - LineCapture text = makeLineCapture(PlainFormatter()); - LineCapture json = makeLineCapture(JSONFormatter()); - LineCapture bson = makeLineCapture(BSONFormatter()); + std::unique_ptr<LineCapture> text = makeLineCapture(PlainFormatter()); + std::unique_ptr<LineCapture> json = makeLineCapture(JSONFormatter()); + std::unique_ptr<LineCapture> bson = makeLineCapture(BSONFormatter()); + SharedBuffer buf; }; TEST_F(LogV2TypesTest, Numeric) { @@ -438,9 +457,9 @@ TEST_F(LogV2TypesTest, Numeric) { using T = decltype(dummy); auto test = [&](auto value) { - text.clear(); + text->clear(); LOGV2(20012, "{name}", "name"_attr = value); - ASSERT_EQUALS(text.back(), fmt::format("{}", value)); + ASSERT_EQUALS(text->back(), fmt::format("{}", value)); validateJSON(value); // TODO: We should have been able to use std::make_signed here but it is broken on @@ -471,10 +490,10 @@ TEST_F(LogV2TypesTest, Numeric) { using T = decltype(dummy); auto test = [&](auto value) { - text.clear(); + text->clear(); LOGV2(20013, "{name}", "name"_attr = value); // Floats are formatted as double - ASSERT_EQUALS(text.back(), fmt::format("{}", static_cast<double>(value))); + ASSERT_EQUALS(text->back(), fmt::format("{}", static_cast<double>(value))); validateJSON(value); ASSERT_EQUALS(lastBSONElement().Number(), value); }; @@ -493,13 +512,13 @@ TEST_F(LogV2TypesTest, Numeric) { bool b = true; LOGV2(20014, "bool {name}", "name"_attr = b); - ASSERT_EQUALS(text.back(), "bool true"); + ASSERT_EQUALS(text->back(), "bool true"); validateJSON(b); ASSERT(lastBSONElement().Bool() == b); char c = 1; LOGV2(20015, "char {name}", "name"_attr = c); - ASSERT_EQUALS(text.back(), "char 1"); + ASSERT_EQUALS(text->back(), "char 1"); validateJSON(static_cast<uint8_t>(c)); // cast to prevent property_tree ASCII parse. ASSERT(lastBSONElement().Number() == c); @@ -526,19 +545,19 @@ TEST_F(LogV2TypesTest, Enums) { enum UnscopedEnum { UnscopedEntry }; LOGV2(20076, "{name}", "name"_attr = UnscopedEntry); auto expectedUnscoped = static_cast<std::underlying_type_t<UnscopedEnum>>(UnscopedEntry); - ASSERT_EQUALS(text.back(), std::to_string(expectedUnscoped)); + ASSERT_EQUALS(text->back(), std::to_string(expectedUnscoped)); validateJSON(expectedUnscoped); ASSERT_EQUALS(lastBSONElement().Number(), expectedUnscoped); enum class ScopedEnum { Entry = -1 }; LOGV2(20077, "{name}", "name"_attr = ScopedEnum::Entry); auto expectedScoped = static_cast<std::underlying_type_t<ScopedEnum>>(ScopedEnum::Entry); - ASSERT_EQUALS(text.back(), std::to_string(expectedScoped)); + ASSERT_EQUALS(text->back(), std::to_string(expectedScoped)); validateJSON(expectedScoped); ASSERT_EQUALS(lastBSONElement().Number(), expectedScoped); LOGV2(20078, "{name}", "name"_attr = UnscopedEntryWithToString); - ASSERT_EQUALS(text.back(), toString(UnscopedEntryWithToString)); + ASSERT_EQUALS(text->back(), toString(UnscopedEntryWithToString)); validateJSON(toString(UnscopedEntryWithToString)); ASSERT_EQUALS(lastBSONElement().String(), toString(UnscopedEntryWithToString)); } @@ -546,32 +565,32 @@ TEST_F(LogV2TypesTest, Enums) { TEST_F(LogV2TypesTest, Stringlike) { const char* c_str = "a c string"; LOGV2(20016, "c string {name}", "name"_attr = c_str); - ASSERT_EQUALS(text.back(), "c string a c string"); + ASSERT_EQUALS(text->back(), "c string a c string"); validateJSON(std::string(c_str)); ASSERT_EQUALS(lastBSONElement().String(), c_str); char* c_str2 = const_cast<char*>("non-const"); LOGV2(20017, "c string {name}", "name"_attr = c_str2); - ASSERT_EQUALS(text.back(), "c string non-const"); + ASSERT_EQUALS(text->back(), "c string non-const"); validateJSON(std::string(c_str2)); ASSERT_EQUALS(lastBSONElement().String(), c_str2); std::string str = "a std::string"; LOGV2(20018, "std::string {name}", "name"_attr = str); - ASSERT_EQUALS(text.back(), "std::string a std::string"); + ASSERT_EQUALS(text->back(), "std::string a std::string"); validateJSON(str); ASSERT_EQUALS(lastBSONElement().String(), str); StringData str_data = "a StringData"_sd; LOGV2(20019, "StringData {name}", "name"_attr = str_data); - ASSERT_EQUALS(text.back(), "StringData a StringData"); + ASSERT_EQUALS(text->back(), "StringData a StringData"); validateJSON(str_data.toString()); ASSERT_EQUALS(lastBSONElement().String(), str_data); { std::string_view s = "a std::string_view"; LOGV2(4329200, "std::string_view {name}", "name"_attr = s); - ASSERT_EQUALS(text.back(), "std::string_view a std::string_view"); + ASSERT_EQUALS(text->back(), "std::string_view a std::string_view"); validateJSON(std::string{s}); ASSERT_EQUALS(lastBSONElement().String(), s); } @@ -585,9 +604,9 @@ TEST_F(LogV2TypesTest, BSONObj) { .append("str"_sd, "a StringData"_sd) .obj(); LOGV2(20020, "bson {name}", "name"_attr = bsonObj); - ASSERT(text.back() == + ASSERT(text->back() == std::string("bson ") + bsonObj.jsonString(JsonStringFormat::ExtendedRelaxedV2_0_0)); - ASSERT(mongo::fromjson(json.back()) + ASSERT(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name") @@ -600,9 +619,9 @@ TEST_F(LogV2TypesTest, BSONArray) { BSONArray bsonArr = BSONArrayBuilder().append("first"_sd).append("second"_sd).append("third"_sd).arr(); LOGV2(20021, "{name}", "name"_attr = bsonArr); - ASSERT_EQUALS(text.back(), + ASSERT_EQUALS(text->back(), bsonArr.jsonString(JsonStringFormat::ExtendedRelaxedV2_0_0, 0, true)); - ASSERT(mongo::fromjson(json.back()) + ASSERT(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name") @@ -619,8 +638,8 @@ TEST_F(LogV2TypesTest, BSONElement) { .append("str"_sd, "a StringData"_sd) .obj(); LOGV2(20022, "bson element {name}", "name"_attr = bsonObj.getField("int32"_sd)); - ASSERT(text.back() == std::string("bson element ") + bsonObj.getField("int32"_sd).toString()); - ASSERT(mongo::fromjson(json.back()) + ASSERT(text->back() == std::string("bson element ") + bsonObj.getField("int32"_sd).toString()); + ASSERT(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name"_sd) @@ -637,8 +656,8 @@ TEST_F(LogV2TypesTest, DateT) { setDateFormatIsLocalTimezone(localTimezone); Date_t date = Date_t::now(); LOGV2(20023, "Date_t {name}", "name"_attr = date); - ASSERT_EQUALS(text.back(), std::string("Date_t ") + date.toString()); - ASSERT_EQUALS(mongo::fromjson(json.back()) + ASSERT_EQUALS(text->back(), std::string("Date_t ") + date.toString()); + ASSERT_EQUALS(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name") @@ -652,8 +671,8 @@ TEST_F(LogV2TypesTest, DateT) { TEST_F(LogV2TypesTest, Decimal128) { LOGV2(20024, "Decimal128 {name}", "name"_attr = Decimal128::kPi); - ASSERT_EQUALS(text.back(), std::string("Decimal128 ") + Decimal128::kPi.toString()); - ASSERT(mongo::fromjson(json.back()) + ASSERT_EQUALS(text->back(), std::string("Decimal128 ") + Decimal128::kPi.toString()); + ASSERT(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name") @@ -665,9 +684,9 @@ TEST_F(LogV2TypesTest, Decimal128) { TEST_F(LogV2TypesTest, OID) { OID oid = OID::gen(); LOGV2(20025, "OID {name}", "name"_attr = oid); - ASSERT_EQUALS(text.back(), std::string("OID ") + oid.toString()); + ASSERT_EQUALS(text->back(), std::string("OID ") + oid.toString()); ASSERT_EQUALS( - mongo::fromjson(json.back()).getField(kAttributesFieldName).Obj().getField("name").OID(), + mongo::fromjson(json->back()).getField(kAttributesFieldName).Obj().getField("name").OID(), oid); ASSERT_EQUALS(lastBSONElement().OID(), oid); } @@ -675,8 +694,8 @@ TEST_F(LogV2TypesTest, OID) { TEST_F(LogV2TypesTest, Timestamp) { Timestamp ts = Timestamp::max(); LOGV2(20026, "Timestamp {name}", "name"_attr = ts); - ASSERT_EQUALS(text.back(), std::string("Timestamp ") + ts.toString()); - ASSERT_EQUALS(mongo::fromjson(json.back()) + ASSERT_EQUALS(text->back(), std::string("Timestamp ") + ts.toString()); + ASSERT_EQUALS(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name") @@ -688,8 +707,8 @@ TEST_F(LogV2TypesTest, Timestamp) { TEST_F(LogV2TypesTest, UUID) { UUID uuid = UUID::gen(); LOGV2(20027, "UUID {name}", "name"_attr = uuid); - ASSERT_EQUALS(text.back(), std::string("UUID ") + uuid.toString()); - ASSERT_EQUALS(UUID::parse(mongo::fromjson(json.back()) + ASSERT_EQUALS(text->back(), std::string("UUID ") + uuid.toString()); + ASSERT_EQUALS(UUID::parse(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name") @@ -700,10 +719,10 @@ TEST_F(LogV2TypesTest, UUID) { TEST_F(LogV2TypesTest, BoostOptional) { LOGV2(20028, "boost::optional empty {name}", "name"_attr = boost::optional<bool>()); - ASSERT_EQUALS(text.back(), + ASSERT_EQUALS(text->back(), std::string("boost::optional empty ") + constants::kNullOptionalString.toString()); - ASSERT(mongo::fromjson(json.back()) + ASSERT(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name") @@ -711,18 +730,18 @@ TEST_F(LogV2TypesTest, BoostOptional) { ASSERT(lastBSONElement().isNull()); LOGV2(20029, "boost::optional<bool> {name}", "name"_attr = boost::optional<bool>(true)); - ASSERT_EQUALS(text.back(), std::string("boost::optional<bool> true")); + ASSERT_EQUALS(text->back(), std::string("boost::optional<bool> true")); ASSERT_EQUALS( - mongo::fromjson(json.back()).getField(kAttributesFieldName).Obj().getField("name").Bool(), + mongo::fromjson(json->back()).getField(kAttributesFieldName).Obj().getField("name").Bool(), true); ASSERT_EQUALS(lastBSONElement().Bool(), true); LOGV2(20030, "boost::optional<boost::optional<bool>> {name}", "name"_attr = boost::optional<boost::optional<bool>>(boost::optional<bool>(true))); - ASSERT_EQUALS(text.back(), std::string("boost::optional<boost::optional<bool>> true")); + ASSERT_EQUALS(text->back(), std::string("boost::optional<boost::optional<bool>> true")); ASSERT_EQUALS( - mongo::fromjson(json.back()).getField(kAttributesFieldName).Obj().getField("name").Bool(), + mongo::fromjson(json->back()).getField(kAttributesFieldName).Obj().getField("name").Bool(), true); ASSERT_EQUALS(lastBSONElement().Bool(), true); @@ -730,8 +749,9 @@ TEST_F(LogV2TypesTest, BoostOptional) { LOGV2(20031, "boost::optional<TypeWithBSON> {name}", "name"_attr = boost::optional<TypeWithBSON>(withBSON)); - ASSERT_EQUALS(text.back(), std::string("boost::optional<TypeWithBSON> ") + withBSON.toString()); - ASSERT(mongo::fromjson(json.back()) + ASSERT_EQUALS(text->back(), + std::string("boost::optional<TypeWithBSON> ") + withBSON.toString()); + ASSERT(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name") @@ -743,25 +763,28 @@ TEST_F(LogV2TypesTest, BoostOptional) { LOGV2(20032, "boost::optional<TypeWithBSON> {name}", "name"_attr = boost::optional<TypeWithoutBSON>(withoutBSON)); - ASSERT_EQUALS(text.back(), + ASSERT_EQUALS(text->back(), std::string("boost::optional<TypeWithBSON> ") + withoutBSON.toString()); - ASSERT_EQUALS( - mongo::fromjson(json.back()).getField(kAttributesFieldName).Obj().getField("name").String(), - withoutBSON.toString()); + ASSERT_EQUALS(mongo::fromjson(json->back()) + .getField(kAttributesFieldName) + .Obj() + .getField("name") + .String(), + withoutBSON.toString()); ASSERT_EQUALS(lastBSONElement().String(), withoutBSON.toString()); } TEST_F(LogV2TypesTest, Duration) { Milliseconds ms{12345}; LOGV2(20033, "Duration {name}", "name"_attr = ms); - ASSERT_EQUALS(text.back(), std::string("Duration ") + ms.toString()); - ASSERT_EQUALS(mongo::fromjson(json.back()) + ASSERT_EQUALS(text->back(), std::string("Duration ") + ms.toString()); + ASSERT_EQUALS(mongo::fromjson(json->back()) .getField(kAttributesFieldName) .Obj() .getField("name" + ms.mongoUnitSuffix()) .Int(), ms.count()); - ASSERT_EQUALS(BSONObj(bson.back().data()) + ASSERT_EQUALS(BSONObj(bson->back().data()) .getField(kAttributesFieldName) .Obj() .getField("name" + ms.mongoUnitSuffix()) @@ -773,31 +796,31 @@ TEST_F(LogV2Test, TextFormat) { auto lines = makeLineCapture(TextFormatter()); LOGV2_OPTIONS(20065, {LogTag::kNone}, "warning"); - ASSERT(lines.back().rfind("** WARNING: warning") == std::string::npos); + ASSERT(lines->back().rfind("** WARNING: warning") == std::string::npos); LOGV2_OPTIONS(20066, {LogTag::kStartupWarnings}, "warning"); - ASSERT(lines.back().rfind("** WARNING: warning") != std::string::npos); + ASSERT(lines->back().rfind("** WARNING: warning") != std::string::npos); LOGV2_OPTIONS(20067, {static_cast<LogTag::Value>(LogTag::kStartupWarnings | LogTag::kPlainShell)}, "warning"); - ASSERT(lines.back().rfind("** WARNING: warning") != std::string::npos); + ASSERT(lines->back().rfind("** WARNING: warning") != std::string::npos); TypeWithBSON t(1.0, 2.0); LOGV2(20034, "{name} custom formatting", "name"_attr = t); - ASSERT(lines.back().rfind(t.toString() + " custom formatting") != std::string::npos); + ASSERT(lines->back().rfind(t.toString() + " custom formatting") != std::string::npos); LOGV2(20035, "{name} bson", "name"_attr = t.toBSON()); - ASSERT(lines.back().rfind(t.toBSON().jsonString(JsonStringFormat::ExtendedRelaxedV2_0_0) + - " bson") != std::string::npos); + ASSERT(lines->back().rfind(t.toBSON().jsonString(JsonStringFormat::ExtendedRelaxedV2_0_0) + + " bson") != std::string::npos); TypeWithoutBSON t2(1.0, 2.0); LOGV2(20036, "{name} custom formatting, no bson", "name"_attr = t2); - ASSERT(lines.back().rfind(t.toString() + " custom formatting, no bson") != std::string::npos); + ASSERT(lines->back().rfind(t.toString() + " custom formatting, no bson") != std::string::npos); TypeWithNonMemberFormatting t3; LOGV2(20079, "{name}", "name"_attr = t3); - ASSERT(lines.back().rfind(toString(t3)) != std::string::npos); + ASSERT(lines->back().rfind(toString(t3)) != std::string::npos); } std::string hello() { @@ -812,12 +835,12 @@ public: template <typename F> void validate(F validator) { - validator(mongo::fromjson(lines.back())); - validator(BSONObj(linesBson.back().data())); + validator(mongo::fromjson(lines->back())); + validator(BSONObj(linesBson->back().data())); } - LineCapture lines = makeLineCapture(JSONFormatter()); - LineCapture linesBson = makeLineCapture(BSONFormatter()); + std::unique_ptr<LineCapture> lines = makeLineCapture(JSONFormatter()); + std::unique_ptr<LineCapture> linesBson = makeLineCapture(BSONFormatter()); }; TEST_F(LogV2JsonBsonTest, Root) { @@ -1162,8 +1185,8 @@ public: /** Ensure json and bson modes both pass. */ template <typename F> void validate(F validator) { - validator(mongo::fromjson(json.back())); - validator(BSONObj(bson.back().data())); + validator(mongo::fromjson(json->back())); + validator(BSONObj(bson->back().data())); } }; @@ -1171,7 +1194,7 @@ public: TEST_F(LogV2ContainerTest, StandardSequential) { std::vector<std::string> vectorStrings = {"str1", "str2", "str3"}; LOGV2(20047, "{name}", "name"_attr = vectorStrings); - ASSERT_EQUALS(text.back(), textJoin(vectorStrings, [](auto&& s) { return s; })); + ASSERT_EQUALS(text->back(), textJoin(vectorStrings, [](auto&& s) { return s; })); validate([&vectorStrings](const BSONObj& obj) { std::vector<BSONElement> jsonVector = obj.getField(kAttributesFieldName).Obj().getField("name").Array(); @@ -1207,7 +1230,7 @@ TEST_F(LogV2ContainerTest, CustomFormatting) { std::list<TypeWithBSON> listCustom = { TypeWithBSON(0.0, 1.0), TypeWithBSON(2.0, 3.0), TypeWithBSON(4.0, 5.0)}; LOGV2(20048, "{name}", "name"_attr = listCustom); - ASSERT_EQUALS(text.back(), textJoin(listCustom, [](auto&& x) { return x.toString(); })); + ASSERT_EQUALS(text->back(), textJoin(listCustom, [](auto&& x) { return x.toString(); })); validate([&listCustom](const BSONObj& obj) { std::vector<BSONElement> jsonVector = obj.getField(kAttributesFieldName).Obj().getField("name").Array(); @@ -1223,7 +1246,7 @@ TEST_F(LogV2ContainerTest, CustomFormatting) { TEST_F(LogV2ContainerTest, OptionalsAsElements) { std::forward_list<boost::optional<bool>> listOptionalBool = {true, boost::none, false}; LOGV2(20049, "{name}", "name"_attr = listOptionalBool); - ASSERT_EQUALS(text.back(), textJoin(listOptionalBool, [](const auto& item) -> std::string { + ASSERT_EQUALS(text->back(), textJoin(listOptionalBool, [](const auto& item) -> std::string { if (!item) return constants::kNullOptionalString.toString(); if (*item) @@ -1250,7 +1273,7 @@ TEST_F(LogV2ContainerTest, OptionalsAsElements) { TEST_F(LogV2ContainerTest, Nested) { std::array<std::deque<int>, 4> arrayOfDeques = {{{0, 1}, {2, 3}, {4, 5}, {6, 7}}}; LOGV2(20050, "{name}", "name"_attr = arrayOfDeques); - ASSERT_EQUALS(text.back(), textJoin(arrayOfDeques, [](auto&& outer) { + ASSERT_EQUALS(text->back(), textJoin(arrayOfDeques, [](auto&& outer) { return textJoin(outer, [](auto&& v) { return fmt::format("{}", v); }); })); validate([&arrayOfDeques](const BSONObj& obj) { @@ -1276,7 +1299,7 @@ TEST_F(LogV2ContainerTest, Associative) { // Associative containers are also supported std::map<std::string, std::string> mapStrStr = {{"key1", "val1"}, {"key2", "val2"}}; LOGV2(20051, "{name}", "name"_attr = mapStrStr); - ASSERT_EQUALS(text.back(), textJoin(mapStrStr, [](const auto& item) { + ASSERT_EQUALS(text->back(), textJoin(mapStrStr, [](const auto& item) { return fmt::format("{}: {}", item.first, item.second); })); validate([&mapStrStr](const BSONObj& obj) { @@ -1295,7 +1318,7 @@ TEST_F(LogV2ContainerTest, AssociativeWithOptionalSequential) { {"key2", boost::optional<std::vector<int>>{boost::none}}}; LOGV2(20052, "{name}", "name"_attr = mapOptionalVector); - ASSERT_EQUALS(text.back(), textJoin(mapOptionalVector, [](auto&& item) { + ASSERT_EQUALS(text->back(), textJoin(mapOptionalVector, [](auto&& item) { std::string r = item.first + ": "; if (item.second) { r += textJoin(*item.second, [](int v) { return fmt::format("{}", v); }); @@ -1416,7 +1439,7 @@ TEST_F(LogV2Test, Unicode) { }; auto getLastMongo = [&]() { - return mongo::fromjson(lines.back()) + return mongo::fromjson(lines->back()) .getField(constants::kAttributesFieldName) .Obj() .getField("name") @@ -1426,7 +1449,7 @@ TEST_F(LogV2Test, Unicode) { auto getLastPtree = [&]() { namespace pt = boost::property_tree; - std::istringstream json_stream(lines.back()); + std::istringstream json_stream(lines->back()); pt::ptree ptree; pt::json_parser::read_json(json_stream, ptree); return ptree.get<std::string>(std::string(constants::kAttributesFieldName) + ".name"); @@ -1482,7 +1505,7 @@ TEST_F(LogV2Test, JsonTruncation) { // Attributes coming after the truncated one should be written ASSERT(obj.getField(constants::kAttributesFieldName).Obj().getField("attr2").Bool()); }; - validateTruncation(mongo::fromjson(lines.back())); + validateTruncation(mongo::fromjson(lines->back())); LOGV2_OPTIONS(20086, {LogTruncation::Disabled}, "{name}", "name"_attr = builder.done()); auto validateTruncationDisabled = [&](const BSONObj& obj) { @@ -1501,7 +1524,7 @@ TEST_F(LogV2Test, JsonTruncation) { ASSERT(!obj.hasField(constants::kTruncatedFieldName)); ASSERT(!obj.hasField(constants::kTruncatedSizeFieldName)); }; - validateTruncationDisabled(mongo::fromjson(lines.back())); + validateTruncationDisabled(mongo::fromjson(lines->back())); BSONArrayBuilder arrBuilder; // Fields will use more than one byte each so this will truncate at some point @@ -1526,7 +1549,7 @@ TEST_F(LogV2Test, JsonTruncation) { obj.getField(constants::kTruncatedSizeFieldName).Obj().getField("name"_sd).Int(), arrToLog.objsize()); }; - validateArrayTruncation(mongo::fromjson(lines.back())); + validateArrayTruncation(mongo::fromjson(lines->back())); } TEST_F(LogV2Test, StringTruncation) { @@ -1562,7 +1585,7 @@ TEST_F(LogV2Test, StringTruncation) { for (const auto& [input, suffix, note] : tests) { LOGV2(6694001, "name", "name"_attr = input); - BSONObj obj = fromjson(lines.back()); + BSONObj obj = fromjson(lines->back()); auto str = obj[constants::kAttributesFieldName]["name"].checkAndGetStringData(); std::string context = "Failed test: " + note; @@ -1609,9 +1632,9 @@ TEST_F(LogV2Test, Threads) { thread.join(); } - ASSERT(linesPlain.size() == threads.size() * kNumPerThread); - ASSERT(linesText.size() == threads.size() * kNumPerThread); - ASSERT(linesJson.size() == threads.size() * kNumPerThread); + ASSERT(linesPlain->size() == threads.size() * kNumPerThread); + ASSERT(linesText->size() == threads.size() * kNumPerThread); + ASSERT(linesJson->size() == threads.size() * kNumPerThread); } TEST_F(LogV2Test, Ramlog) { @@ -1625,7 +1648,7 @@ TEST_F(LogV2Test, Ramlog) { auto verifyRamLog = [&] { RamLog::LineIterator iter(ramlog); - for (const auto& s : lines.lines()) { + for (const auto& s : lines->lines()) { const auto next = iter.next(); if (s != next) { std::cout << "\n\n\n********************** s='" << s << "', next='" << next @@ -1748,8 +1771,8 @@ TEST_F(LogV2Test, MultipleDomains) { } }; LogDomain other_domain(std::make_unique<OtherDomain>()); - std::vector<std::string> other_lines; - auto other_sink = LogCaptureBackend::create(other_lines, true); + synchronized_value<std::vector<std::string>> other_lines; + auto other_sink = LogCaptureBackend::create(std::make_unique<Listener>(&other_lines), true); other_sink->set_filter(ComponentSettingsFilter(other_domain, mgr().getGlobalSettings())); other_sink->set_formatter(PlainFormatter()); attachSink(other_sink); @@ -1757,12 +1780,13 @@ TEST_F(LogV2Test, MultipleDomains) { auto global_lines = makeLineCapture(PlainFormatter()); LOGV2_OPTIONS(20070, {&other_domain}, "test"); - ASSERT(global_lines.lines().empty()); - ASSERT(other_lines.back() == "test"); + auto logLinesLockGuard = *other_lines; + ASSERT(global_lines->lines().empty()); + ASSERT(logLinesLockGuard->back() == "test"); LOGV2(20060, "global domain log"); - ASSERT(global_lines.back() == "global domain log"); - ASSERT(other_lines.back() == "test"); + ASSERT(global_lines->back() == "global domain log"); + ASSERT(logLinesLockGuard->back() == "test"); } TEST_F(LogV2Test, FileLogging) { @@ -1814,9 +1838,10 @@ TEST_F(LogV2Test, FileLogging) { } TEST_F(LogV2Test, UserAssert) { - std::vector<std::string> lines; + synchronized_value<std::vector<std::string>> syncedLines; auto sink = wrapInSynchronousSink(wrapInCompositeBackend( - boost::make_shared<LogCaptureBackend>(lines, true), boost::make_shared<UserAssertSink>())); + boost::make_shared<LogCaptureBackend>(std::make_unique<Listener>(&syncedLines), true), + boost::make_shared<UserAssertSink>())); applyDefaultFilterToSink(sink); sink->set_formatter(PlainFormatter()); attachSink(sink); @@ -1826,31 +1851,31 @@ TEST_F(LogV2Test, UserAssert) { ASSERT_THROWS_WITH_CHECK( LOGV2_OPTIONS(4652000, {UserAssertAfterLog(ErrorCodes::BadValue)}, "uasserting log"), DBException, - [&lines](const DBException& ex) { + [&syncedLines](const DBException& ex) { ASSERT_EQUALS(ex.code(), ErrorCodes::BadValue); ASSERT_EQUALS(ex.reason(), "uasserting log"); - ASSERT_EQUALS(lines.front(), ex.reason()); + ASSERT_EQUALS((**syncedLines).front(), ex.reason()); }); - lines.clear(); + (**syncedLines).clear(); ASSERT_THROWS_WITH_CHECK(LOGV2_OPTIONS(4652001, {UserAssertAfterLog(ErrorCodes::BadValue)}, "uasserting log {name}", "name"_attr = 1), DBException, - [&lines](const DBException& ex) { + [&syncedLines](const DBException& ex) { ASSERT_EQUALS(ex.code(), ErrorCodes::BadValue); ASSERT_EQUALS(ex.reason(), "uasserting log 1"); - ASSERT_EQUALS(lines.front(), ex.reason()); + ASSERT_EQUALS((**syncedLines).front(), ex.reason()); }); - lines.clear(); + (**syncedLines).clear(); ASSERT_THROWS_WITH_CHECK(LOGV2_OPTIONS(4716000, {UserAssertAfterLog()}, "uasserting log"), DBException, - [&lines](const DBException& ex) { + [&syncedLines](const DBException& ex) { ASSERT_EQUALS(ex.code(), 4716000); ASSERT_EQUALS(ex.reason(), "uasserting log"); - ASSERT_EQUALS(lines.front(), ex.reason()); + ASSERT_EQUALS((**syncedLines).front(), ex.reason()); }); } diff --git a/src/mongo/logv2/redaction.cpp b/src/mongo/logv2/redaction.cpp index 0b143ba39ae..29536660722 100644 --- a/src/mongo/logv2/redaction.cpp +++ b/src/mongo/logv2/redaction.cpp @@ -50,12 +50,12 @@ constexpr auto kRedactionDefaultMask = "###"_sd; BSONObj redact(const BSONObj& objectToRedact) { if (!logv2::shouldRedactLogs()) { if (!logv2::shouldRedactBinDataEncrypt()) { - return objectToRedact; + return objectToRedact.redact(BSONObj::RedactLevel::sensitiveOnly); } - return objectToRedact.redact(true /* onlyEncryptedFields */); + return objectToRedact.redact(BSONObj::RedactLevel::encryptedAndSensitive); } - return objectToRedact.redact(false /* onlyEncryptedFields */); + return objectToRedact.redact(BSONObj::RedactLevel::all); } StringData redact(StringData stringToRedact) { diff --git a/src/mongo/logv2/redaction_test.cpp b/src/mongo/logv2/redaction_test.cpp index dda840a4681..ca61f7ec295 100644 --- a/src/mongo/logv2/redaction_test.cpp +++ b/src/mongo/logv2/redaction_test.cpp @@ -34,6 +34,7 @@ #include "mongo/base/error_extra_info.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/bsontypes.h" +#include "mongo/bson/json.h" #include "mongo/db/jsobj.h" #include "mongo/logv2/log_util.h" #include "mongo/unittest/unittest.h" @@ -140,8 +141,6 @@ TEST(RedactEncryptedStringTest, BasicStrings) { } BSONObj obj = builder.done(); - std::cout << "This is obj: " << obj.toString() << std::endl; - auto redactedStr = R"({ type6: "###", string: "string", nestedobj: { subobj: "###" } })"; ASSERT_EQ(redact(obj).toString(), redactedStr); @@ -149,6 +148,98 @@ TEST(RedactEncryptedStringTest, BasicStrings) { ASSERT_EQ(redact(obj).toString(), obj.toString()); } +TEST(RedactSensitiveStringTest, BasicStrings) { + BSONObjBuilder builder{}; + builder.appendBinData("type8", sizeof(zero), BinDataType::Sensitive, zero); + builder.append("string", "string"); + { + BSONObjBuilder sub(builder.subobjStart("nestedobj")); + sub.appendBinData("subobj", sizeof(zero), BinDataType::Sensitive, zero); + } + const BSONObj obj = builder.done(); + + { + logv2::setShouldRedactBinDataEncrypt(true); + logv2::setShouldRedactLogs(true); + + // Fully-redacted logs should just redact everything + const auto redactedStr = R"({ type8: "###", string: "###", nestedobj: { subobj: "###" } })"; + ASSERT_EQ(redact(obj).toString(), redactedStr); + } + + { + const auto redactedStr = + R"({ type8: "###", string: "string", nestedobj: { subobj: "###" } })"; + // The setting for redacting logs shouldn't affect sensitive BinData. + logv2::setShouldRedactLogs(false); + ASSERT_EQ(redact(obj).toString(), redactedStr); + + // The setting for redacting encrypted BinData shouldn't affect sensitive BinData, either. + logv2::setShouldRedactBinDataEncrypt(false); + ASSERT_EQ(redact(obj).toString(), redactedStr); + } +} + +TEST(RedactSensitiveStringTest, NestedStrings) { + // The setting for redacting logs shouldn't affect sensitive BinData. + logv2::setShouldRedactBinDataEncrypt(false); + // The setting for redacting encrypted BinData shouldn't affect sensitive BinData, either. + logv2::setShouldRedactLogs(false); + + BSONObjBuilder builder{}; + + // Test for [ "###", { ...: "###" }, ... ] shape cases. + { + auto subarray = BSONObjBuilder(builder.subarrayStart("subarray")); + subarray.appendBinData("0", sizeof(zero), BinDataType::Sensitive, zero); + + for (auto nSubobjs = 0; nSubobjs < 3; ++nSubobjs) { + BSONObjBuilder(subarray.subobjStart("subobj")) + .appendBinData("type8", sizeof(zero), BinDataType::Sensitive, zero); + } + } + + // Test for { ...: "###", ...: [ "###", ... ] } shape cases. + { + auto subobj = BSONObjBuilder(builder.subobjStart("subobj")); + subobj.appendBinData("type8", sizeof(zero), BinDataType::Sensitive, zero); + + auto subarray = BSONObjBuilder(subobj.subarrayStart("subarray")); + for (auto nSubobjs = 0; nSubobjs < 3; ++nSubobjs) { + subarray.appendBinData("0", sizeof(zero), BinDataType::Sensitive, zero); + } + } + + // Test for [ [ [ "###", ... ] ] ] shape cases. + { + auto subarray1 = BSONObjBuilder(builder.subarrayStart("subarrays")); + auto subarray2 = BSONObjBuilder(subarray1.subarrayStart("subarray")); + auto subarray3 = BSONObjBuilder(subarray2.subarrayStart("subarray")); + for (auto nSubobjs = 0; nSubobjs < 3; ++nSubobjs) { + subarray3.appendBinData("0", sizeof(zero), BinDataType::Sensitive, zero); + } + } + + // Test for { ...: { ...: { ...: "###" } } } shape cases. + { + auto subobj1 = BSONObjBuilder(builder.subobjStart("subobjs")); + auto subobj2 = BSONObjBuilder(subobj1.subobjStart("subobj")); + auto subobj3 = BSONObjBuilder(subobj2.subobjStart("subobj")); + subobj3.appendBinData("type8", sizeof(zero), BinDataType::Sensitive, zero); + } + + const BSONObj obj = builder.done(); + + // Type 8 values should all be redacted. + const BSONObj expected = fromjson(R"({ + subarray: [ "###", { type8: "###" }, { type8: "###" }, { type8: "###" } ], + subobj: { type8: "###", subarray: [ "###", "###", "###" ] }, + subarrays: [ [ [ "###", "###", "###" ] ] ], + subobjs: { subobj: { subobj: { type8: "###" } } } + })"); + ASSERT_EQ(redact(obj).toString(), expected.toString()); +} + void testBSONCases(std::vector<BSONStringPair>& testCases) { for (auto m : testCases) { ASSERT_EQ(redact(m.first).toString(), m.second); diff --git a/src/mongo/platform/atomic_word.h b/src/mongo/platform/atomic_word.h index c277497b15c..5a7d38ccca7 100644 --- a/src/mongo/platform/atomic_word.h +++ b/src/mongo/platform/atomic_word.h @@ -87,9 +87,7 @@ public: } /** - * Gets the current value of this AtomicWord. - * - * Has relaxed semantics. + * Gets the current value of this AtomicWord using relaxed memory order. */ WordType loadRelaxed() const { return _value.load(std::memory_order_relaxed); @@ -103,6 +101,13 @@ public: } /** + * Sets the value of this AtomicWord to "newValue" using relaxed memory order. + */ + void storeRelaxed(WordType newValue) { + _value.store(newValue, std::memory_order_relaxed); + } + + /** * Atomically swaps the current value of this with "newValue". * * Returns the old value. diff --git a/src/mongo/platform/decimal128.cpp b/src/mongo/platform/decimal128.cpp index 0d12479f70a..ef785025b17 100644 --- a/src/mongo/platform/decimal128.cpp +++ b/src/mongo/platform/decimal128.cpp @@ -827,36 +827,33 @@ Decimal128 Decimal128::exponential(std::uint32_t* signalingFlags, RoundingMode r return Decimal128{libraryTypeToValue(current)}; } -Decimal128 Decimal128::logarithm(RoundingMode roundMode) const { +Decimal128 Decimal128::naturalLogarithm(RoundingMode roundMode) const { std::uint32_t throwAwayFlag = 0; - return logarithm(&throwAwayFlag, roundMode); + return naturalLogarithm(&throwAwayFlag, roundMode); } -Decimal128 Decimal128::logarithm(std::uint32_t* signalingFlags, RoundingMode roundMode) const { +Decimal128 Decimal128::naturalLogarithm(std::uint32_t* signalingFlags, + RoundingMode roundMode) const { BID_UINT128 current = decimal128ToLibraryType(_value); current = bid128_log(current, roundMode, signalingFlags); return Decimal128{libraryTypeToValue(current)}; } -Decimal128 Decimal128::logarithm(const Decimal128& other, RoundingMode roundMode) const { +Decimal128 Decimal128::logarithm(const Decimal128& base, RoundingMode roundMode) const { std::uint32_t throwAwayFlag = 0; - if (other.isEqual(Decimal128(2))) { + if (base.isEqual(Decimal128(2))) { BID_UINT128 current = decimal128ToLibraryType(_value); current = bid128_log2(current, roundMode, &throwAwayFlag); return Decimal128{libraryTypeToValue(current)}; } - if (other.isEqual(Decimal128(10))) { + if (base.isEqual(Decimal128(10))) { BID_UINT128 current = decimal128ToLibraryType(_value); current = bid128_log10(current, roundMode, &throwAwayFlag); return Decimal128{libraryTypeToValue(current)}; } - return logarithm(other, &throwAwayFlag); -} - -Decimal128 Decimal128::logarithm(const Decimal128& other, - std::uint32_t* signalingFlags, - RoundingMode roundMode) const { - return logarithm(signalingFlags, roundMode).divide(other); + // Logarithm with a generic base is equivalent to `ln(input) / ln(base)`. + return naturalLogarithm(&throwAwayFlag, roundMode) + .divide(base.naturalLogarithm(&throwAwayFlag, roundMode)); } Decimal128 Decimal128::modulo(const Decimal128& other) const { diff --git a/src/mongo/platform/decimal128.h b/src/mongo/platform/decimal128.h index 64306df0e0c..06a78352546 100644 --- a/src/mongo/platform/decimal128.h +++ b/src/mongo/platform/decimal128.h @@ -467,13 +467,14 @@ public: Decimal128 exponential(RoundingMode roundMode = kRoundTiesToEven) const; Decimal128 exponential(std::uint32_t* signalingFlags, RoundingMode roundMode = kRoundTiesToEven) const; - Decimal128 logarithm(RoundingMode roundMode = kRoundTiesToEven) const; - Decimal128 logarithm(std::uint32_t* signalingFlags, - RoundingMode roundMode = kRoundTiesToEven) const; - Decimal128 logarithm(const Decimal128& other, RoundingMode roundMode = kRoundTiesToEven) const; - Decimal128 logarithm(const Decimal128& other, - std::uint32_t* signalingFlags, - RoundingMode roundMode = kRoundTiesToEven) const; + Decimal128 naturalLogarithm(RoundingMode roundMode = kRoundTiesToEven) const; + Decimal128 naturalLogarithm(std::uint32_t* signalingFlags, + RoundingMode roundMode = kRoundTiesToEven) const; + // Calculate the logarithm with the given `base`, using this number as input. Uses fast paths + // for base 2 and 10. + // TODO SERVER-91935 convert `logarithm` and others to static methods for a more clear + // interface. + Decimal128 logarithm(const Decimal128& base, RoundingMode roundMode = kRoundTiesToEven) const; Decimal128 modulo(const Decimal128& other) const; Decimal128 modulo(const Decimal128& other, std::uint32_t* signalingFlags) const; diff --git a/src/mongo/rpc/SConscript b/src/mongo/rpc/SConscript index 5fa4e0bf85b..10d73388ce1 100644 --- a/src/mongo/rpc/SConscript +++ b/src/mongo/rpc/SConscript @@ -40,6 +40,7 @@ protoEnv.Library( '$BUILD_DIR/mongo/db/wire_version', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/bson/util/bson_extract', '$BUILD_DIR/mongo/db/auth/security_token', '$BUILD_DIR/mongo/db/bson/dotted_path_support', @@ -64,6 +65,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/dbmessage', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/stats/counters', diff --git a/src/mongo/rpc/metadata/client_metadata.cpp b/src/mongo/rpc/metadata/client_metadata.cpp index 33d3e5836c1..8ac4f4a9d97 100644 --- a/src/mongo/rpc/metadata/client_metadata.cpp +++ b/src/mongo/rpc/metadata/client_metadata.cpp @@ -269,6 +269,7 @@ Status ClientMetadata::validateOperatingSystemDocument(const BSONObj& doc) { void ClientMetadata::setMongoSMetadata(StringData hostAndPort, StringData mongosClient, StringData version) { + _documentWithoutMongosInfo = _document; BSONObjBuilder builder; builder.appendElements(_document); @@ -381,6 +382,14 @@ const BSONObj& ClientMetadata::getDocument() const { return _document; } +unsigned long ClientMetadata::hashWithoutMongosInfo() const { + return _hashWithoutMongos.get(documentWithoutMongosInfo()); +} + +const BSONObj& ClientMetadata::documentWithoutMongosInfo() const { + return _documentWithoutMongosInfo.get(_document); +} + void ClientMetadata::logClientMetadata(Client* client) const { if (getDocument().isEmpty()) { return; diff --git a/src/mongo/rpc/metadata/client_metadata.h b/src/mongo/rpc/metadata/client_metadata.h index 30f7643bcae..0c5cabdae7b 100644 --- a/src/mongo/rpc/metadata/client_metadata.h +++ b/src/mongo/rpc/metadata/client_metadata.h @@ -36,6 +36,8 @@ #include "mongo/base/string_data.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" +#include "mongo/bson/simple_bsonobj_comparator.h" +#include "mongo/db/query/util/deferred.h" namespace mongo { @@ -287,6 +289,21 @@ public: const BSONObj& getDocument() const; /** + * A lazily computed (and subsequently cached) copy of the metadata with the mongos info + * removed. This is useful for collecting query stats where we want to scrub out this + * high-cardinality field, and we don't want to re-do this computation over and over again. + */ + const BSONObj& documentWithoutMongosInfo() const; + + /** + * Get the simple hash of the client metadata document (simple meaning no collation). + * + * The hash is generated on the first call to this method. Future calls will return the cached + * hash rather than recomputing. + */ + unsigned long hashWithoutMongosInfo() const; + + /** * Log client and client metadata information to disk. */ void logClientMetadata(Client* client) const; @@ -337,6 +354,16 @@ private: // Application Name extracted from the client metadata document. // May be empty std::string _appName; + + // See documentWithoutMongosInfo(). + Deferred<BSONObj, const BSONObj&> _documentWithoutMongosInfo{ + [](const BSONObj& fullDocument) { return fullDocument.removeField("mongos"); }}; + + // See hashWithoutMongosInfo(). + Deferred<unsigned long, const BSONObj&> _hashWithoutMongos{ + [](const BSONObj& documentWithoutMongosInfo) { + return simpleHash(documentWithoutMongosInfo); + }}; }; } // namespace mongo diff --git a/src/mongo/rpc/metadata/client_metadata_test.cpp b/src/mongo/rpc/metadata/client_metadata_test.cpp index becb3af5698..4b698aa3a14 100644 --- a/src/mongo/rpc/metadata/client_metadata_test.cpp +++ b/src/mongo/rpc/metadata/client_metadata_test.cpp @@ -308,16 +308,17 @@ TEST(ClientMetadataTest, TestMongoSAppend) { auto obj = builder.obj(); auto swParseStatus = ClientMetadata::parse(obj[kMetadataDoc]); ASSERT_OK(swParseStatus.getStatus()); - ASSERT_EQUALS("g", swParseStatus.getValue().get().getApplicationName()); + auto metaObj = swParseStatus.getValue().value(); + ASSERT_EQUALS("g", metaObj.getApplicationName()); + auto docBeforeMongos = obj[kMetadataDoc].Obj(); + ASSERT_BSONOBJ_EQ(metaObj.getDocument(), docBeforeMongos); - swParseStatus.getValue().get().setMongoSMetadata("h", "i", "j"); - ASSERT_EQUALS("g", swParseStatus.getValue().get().getApplicationName()); + metaObj.setMongoSMetadata("h", "i", "j"); + ASSERT_BSONOBJ_NE(metaObj.getDocument(), docBeforeMongos); + ASSERT_EQUALS("g", metaObj.getApplicationName()); - auto doc = swParseStatus.getValue().get().getDocument(); - - constexpr auto kMongos = "mongos"_sd; - constexpr auto kClient = "client"_sd; - constexpr auto kHost = "host"_sd; + auto docWithMongosInfo = metaObj.getDocument(); + ASSERT_BSONOBJ_EQ(metaObj.documentWithoutMongosInfo(), docBeforeMongos); auto pid = ProcessId::getCurrent().toString(); @@ -340,7 +341,60 @@ TEST(ClientMetadataTest, TestMongoSAppend) { .append(kMongos, BOB{}.append(kHost, "h").append(kClient, "i").append(kVersion, "j").obj()) .obj(); - ASSERT_BSONOBJ_EQ(doc, outDoc); + ASSERT_BSONOBJ_EQ(docWithMongosInfo, outDoc); +} + +// Test that if mongos information is present from the beginning, we can still request the document +// without the mongos info. +TEST(ClientMetadataTest, MongosMetaCanBeRemoved) { + BSONObjBuilder realBuilder; + BSONObjBuilder tmpBuilder; + ASSERT_OK(ClientMetadata::serializePrivate("a", "b", "c", "d", "e", "f", "g", &tmpBuilder)); + auto objWithoutMongosMeta = tmpBuilder.obj(); + const auto metaBsonNoMongosInfo = objWithoutMongosMeta[kMetadataDoc].Obj(); + { + BSONObjBuilder metaBuilder = realBuilder.subobjStart(kMetadataDoc); + metaBuilder.appendElements(metaBsonNoMongosInfo); + metaBuilder.append("mongos", BSON(kHost << "h" << kClient << "i" << kVersion << "j")); + metaBuilder.doneFast(); + } + + const auto wrappingMetaBson = realBuilder.obj(); + const auto metaElt = wrappingMetaBson[kMetadataDoc]; + // Add this mongos info without calling 'setMongoSMetadata().' + ASSERT_BSONOBJ_NE(metaElt.Obj(), metaBsonNoMongosInfo); + + auto swParseStatus = ClientMetadata::parse(metaElt); + ASSERT_OK(swParseStatus.getStatus()); + const auto& metaObj = swParseStatus.getValue().value(); + // Test the various copy/move constructors. + ClientMetadata copyConstructed(metaObj); + auto tmpThirdCopy = metaObj; + ClientMetadata moveConstructed(std::move(tmpThirdCopy)); + + auto tmpFourthCopy = metaObj; + auto moveAssigned = metaObj; // copy for now, until next line. + moveAssigned = std::move(tmpFourthCopy); + + const auto tmpFifthCopy = metaObj; + auto copyAssigned = metaObj; // copy construct. + copyAssigned = tmpFifthCopy; // copy assign. + + ASSERT_BSONOBJ_EQ(metaObj.getDocument(), metaElt.Obj()); + ASSERT_BSONOBJ_EQ(metaObj.documentWithoutMongosInfo(), metaBsonNoMongosInfo); + ASSERT_BSONOBJ_EQ(metaObj.documentWithoutMongosInfo(), + copyConstructed.documentWithoutMongosInfo()); + ASSERT_BSONOBJ_EQ(metaObj.documentWithoutMongosInfo(), + moveConstructed.documentWithoutMongosInfo()); + ASSERT_BSONOBJ_EQ(metaObj.documentWithoutMongosInfo(), + copyAssigned.documentWithoutMongosInfo()); + ASSERT_BSONOBJ_EQ(metaObj.documentWithoutMongosInfo(), + moveAssigned.documentWithoutMongosInfo()); + + ASSERT_EQ(metaObj.hashWithoutMongosInfo(), copyConstructed.hashWithoutMongosInfo()); + ASSERT_EQ(metaObj.hashWithoutMongosInfo(), moveConstructed.hashWithoutMongosInfo()); + ASSERT_EQ(metaObj.hashWithoutMongosInfo(), copyAssigned.hashWithoutMongosInfo()); + ASSERT_EQ(metaObj.hashWithoutMongosInfo(), moveAssigned.hashWithoutMongosInfo()); } TEST(ClientMetadataTest, TestInvalidDocWhileSettingOpCtxMetadata) { diff --git a/src/mongo/rpc/op_legacy_integration_test.cpp b/src/mongo/rpc/op_legacy_integration_test.cpp index 75e56d7abfa..3722c4f1bc4 100644 --- a/src/mongo/rpc/op_legacy_integration_test.cpp +++ b/src/mongo/rpc/op_legacy_integration_test.cpp @@ -58,7 +58,7 @@ Message makeUnsupportedOpUpdateMessage(StringData ns, BSONObj query, BSONObj upd return makeMessage(dbUpdate, [&](BufBuilder& b) { const int reservedFlags = 0; b.appendNum(reservedFlags); - b.appendStr(ns); + b.appendCStr(ns); b.appendNum(flags); query.appendSelfToBufBuilder(b); @@ -70,7 +70,7 @@ Message makeUnsupportedOpRemoveMessage(StringData ns, BSONObj query, int flags) return makeMessage(dbDelete, [&](BufBuilder& b) { const int reservedFlags = 0; b.appendNum(reservedFlags); - b.appendStr(ns); + b.appendCStr(ns); b.appendNum(flags); query.appendSelfToBufBuilder(b); @@ -93,7 +93,7 @@ Message makeUnsupportedOpQueryMessage(StringData ns, int queryOptions) { return makeMessage(dbQuery, [&](BufBuilder& b) { b.appendNum(queryOptions); - b.appendStr(ns); + b.appendCStr(ns); b.appendNum(nToSkip); b.appendNum(nToReturn); query.appendSelfToBufBuilder(b); @@ -108,7 +108,7 @@ Message makeUnsupportedOpGetMoreMessage(StringData ns, int flags) { return makeMessage(dbGetMore, [&](BufBuilder& b) { b.appendNum(flags); - b.appendStr(ns); + b.appendCStr(ns); b.appendNum(nToReturn); b.appendNum(cursorId); }); @@ -383,6 +383,18 @@ TEST(OpLegacy, UnsupportedOpsLogging) { exerciseUnsupportedOps(conn.get(), "D2" /*expectedSeverity*/); } +TEST(OpLegacy, InvalidNs) { + auto conn = getIntegrationTestConnection(); + + auto msg = makeMessage(dbQuery, [&](BufBuilder& b) { + b.appendNum(0); + b.appendStrBytes("nonullbyte"); + }); + // Since our request is not able to be parsed, we don't receive a response from the server. + Message ignore; + ASSERT_THROWS(conn->call(msg, ignore), DBException); +} + TEST(OpLegacy, GenericCommandViaOpQuery) { auto conn = getIntegrationTestConnection(); diff --git a/src/mongo/rpc/op_msg.cpp b/src/mongo/rpc/op_msg.cpp index d6a91c7a69d..8565c34c6a5 100644 --- a/src/mongo/rpc/op_msg.cpp +++ b/src/mongo/rpc/op_msg.cpp @@ -301,7 +301,7 @@ auto OpMsgBuilder::beginDocSequence(StringData name) -> DocSequenceBuilder { _buf.appendStruct(Section::kDocSequence); int sizeOffset = _buf.len(); _buf.skip(sizeof(int32_t)); // section size. - _buf.appendStr(name, true); + _buf.appendCStr(name); return DocSequenceBuilder(this, &_buf, sizeOffset); } diff --git a/src/mongo/rpc/op_msg_test.h b/src/mongo/rpc/op_msg_test.h index 58f25f7417f..0e8ad918997 100644 --- a/src/mongo/rpc/op_msg_test.h +++ b/src/mongo/rpc/op_msg_test.h @@ -80,7 +80,7 @@ protected: template <typename... Rest> void append(StringData arg, Rest&&... rest) { - buffer.appendStr(arg, /* null terminate*/ true); + buffer.appendCStr(arg); append(rest...); } diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index e443f02fa0a..e24b7998cb4 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -60,9 +60,9 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/commands/txn_cmd_request', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/logical_session_id_helpers', '$BUILD_DIR/mongo/db/pipeline/process_interface/mongo_process_interface', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/repl/read_concern_args', '$BUILD_DIR/mongo/db/session_catalog', '$BUILD_DIR/mongo/db/shared_request_handling', @@ -239,6 +239,7 @@ env.Library( '$BUILD_DIR/mongo/db/index_commands_idl', '$BUILD_DIR/mongo/db/namespace_string', '$BUILD_DIR/mongo/db/query/query_request', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', '$BUILD_DIR/mongo/db/repl/optime', '$BUILD_DIR/mongo/db/server_options', '$BUILD_DIR/mongo/idl/feature_flag', @@ -462,7 +463,6 @@ env.Library( '$BUILD_DIR/mongo/db/commands/server_status', '$BUILD_DIR/mongo/db/commands/server_status_core', '$BUILD_DIR/mongo/db/commands/server_status_servers', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/dbdirectclient', '$BUILD_DIR/mongo/db/ftdc/ftdc_mongos', '$BUILD_DIR/mongo/db/logical_session_cache', @@ -470,6 +470,7 @@ env.Library( '$BUILD_DIR/mongo/db/logical_time_metadata_hook', '$BUILD_DIR/mongo/db/pipeline/process_interface/mongos_process_interface_factory', '$BUILD_DIR/mongo/db/process_health/fault_manager', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/read_write_concern_defaults', '$BUILD_DIR/mongo/db/server_options', '$BUILD_DIR/mongo/db/server_options_base', diff --git a/src/mongo/s/catalog/sharding_catalog_client.h b/src/mongo/s/catalog/sharding_catalog_client.h index 6b4ea0f4d7d..41057194e3a 100644 --- a/src/mongo/s/catalog/sharding_catalog_client.h +++ b/src/mongo/s/catalog/sharding_catalog_client.h @@ -315,19 +315,6 @@ public: const WriteConcernOptions& writeConcern) = 0; /** - * Directly inserts documents in the specified namespace on the config server. Inserts said - * documents using a retryable write. Underneath, a session is created and destroyed -- this - * ad-hoc session creation strategy should never be used outside of specific, non-performant - * code paths. - * - * Must only be used for insertions in the 'config' database. - */ - virtual void insertConfigDocumentsAsRetryableWrite(OperationContext* opCtx, - const NamespaceString& nss, - std::vector<BSONObj> docs, - const WriteConcernOptions& writeConcern) = 0; - - /** * Updates a single document in the specified namespace on the config server. Must only be used * for updates to the 'config' database. * diff --git a/src/mongo/s/catalog/sharding_catalog_client_impl.cpp b/src/mongo/s/catalog/sharding_catalog_client_impl.cpp index 9720eeef27c..38949060bf1 100644 --- a/src/mongo/s/catalog/sharding_catalog_client_impl.cpp +++ b/src/mongo/s/catalog/sharding_catalog_client_impl.cpp @@ -102,37 +102,6 @@ void toBatchError(const Status& status, BatchedCommandResponse* response) { response->setStatus(status); } -void sendRetryableWriteBatchRequestToConfig(OperationContext* opCtx, - const NamespaceString& nss, - std::vector<BSONObj>& docs, - TxnNumber txnNumber, - const WriteConcernOptions& writeConcern) { - auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - - BatchedCommandRequest request([&] { - write_ops::InsertCommandRequest insertOp(nss); - insertOp.setDocuments(docs); - return insertOp; - }()); - request.setWriteConcern(writeConcern.toBSON()); - - BSONObj cmdObj = request.toBSON(); - BSONObjBuilder bob(cmdObj); - bob.append(OperationSessionInfo::kTxnNumberFieldName, txnNumber); - - BatchedCommandResponse batchResponse; - auto response = configShard->runCommand(opCtx, - ReadPreferenceSetting{ReadPreference::PrimaryOnly}, - nss.db().toString(), - bob.obj(), - Shard::kDefaultConfigCommandTimeout, - Shard::RetryPolicy::kIdempotent); - - auto writeStatus = Shard::CommandResponse::processBatchWriteResponse(response, &batchResponse); - - uassertStatusOK(batchResponse.toStatus()); - uassertStatusOK(writeStatus); -} AggregateCommandRequest makeCollectionAndChunksAggregation(OperationContext* opCtx, const NamespaceString& nss, @@ -1042,12 +1011,14 @@ Status ShardingCatalogClientImpl::insertConfigDocument(OperationContext* opCtx, insertOp.setDocuments({doc}); return insertOp; }()); - request.setWriteConcern(writeConcern.toBSON()); auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); for (int retry = 1; retry <= kMaxWriteRetry; retry++) { - auto response = configShard->runBatchWriteCommand( - opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kNoRetry); + auto response = configShard->runBatchWriteCommand(opCtx, + Shard::kDefaultConfigCommandTimeout, + request, + writeConcern, + Shard::RetryPolicy::kNoRetry); Status status = response.toStatus(); @@ -1102,49 +1073,6 @@ Status ShardingCatalogClientImpl::insertConfigDocument(OperationContext* opCtx, MONGO_UNREACHABLE; } -void ShardingCatalogClientImpl::insertConfigDocumentsAsRetryableWrite( - OperationContext* opCtx, - const NamespaceString& nss, - std::vector<BSONObj> docs, - const WriteConcernOptions& writeConcern) { - invariant(nss.db() == NamespaceString::kAdminDb || nss.db() == NamespaceString::kConfigDb); - - AlternativeSessionRegion asr(opCtx); - TxnNumber currentTxnNumber = 0; - - std::vector<BSONObj> workingBatch; - size_t workingBatchItemSize = 0; - int workingBatchDocSize = 0; - - while (!docs.empty()) { - BSONObj toAdd = docs.back(); - docs.pop_back(); - - const int docSizePlusOverhead = - toAdd.objsize() + write_ops::kRetryableAndTxnBatchWriteBSONSizeOverhead; - // Check if pushing this object will exceed the batch size limit or the max object size - if ((workingBatchItemSize + 1 > write_ops::kMaxWriteBatchSize) || - (workingBatchDocSize + docSizePlusOverhead > BSONObjMaxUserSize)) { - sendRetryableWriteBatchRequestToConfig( - asr.opCtx(), nss, workingBatch, currentTxnNumber, writeConcern); - ++currentTxnNumber; - - workingBatch.clear(); - workingBatchItemSize = 0; - workingBatchDocSize = 0; - } - - workingBatch.push_back(toAdd); - ++workingBatchItemSize; - workingBatchDocSize += docSizePlusOverhead; - } - - if (!workingBatch.empty()) { - sendRetryableWriteBatchRequestToConfig( - asr.opCtx(), nss, workingBatch, currentTxnNumber, writeConcern); - } -} - StatusWith<bool> ShardingCatalogClientImpl::updateConfigDocument( OperationContext* opCtx, const NamespaceString& nss, @@ -1189,11 +1117,10 @@ StatusWith<bool> ShardingCatalogClientImpl::_updateConfigDocument( }()}); return updateOp; }()); - request.setWriteConcern(writeConcern.toBSON()); auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); auto response = configShard->runBatchWriteCommand( - opCtx, maxTimeMs, request, Shard::RetryPolicy::kIdempotent); + opCtx, maxTimeMs, request, writeConcern, Shard::RetryPolicy::kIdempotent); Status status = response.toStatus(); if (!status.isOK()) { @@ -1225,11 +1152,13 @@ Status ShardingCatalogClientImpl::removeConfigDocuments(OperationContext* opCtx, }()}); return deleteOp; }()); - request.setWriteConcern(writeConcern.toBSON()); auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto response = configShard->runBatchWriteCommand( - opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kIdempotent); + auto response = configShard->runBatchWriteCommand(opCtx, + Shard::kDefaultConfigCommandTimeout, + request, + writeConcern, + Shard::RetryPolicy::kIdempotent); return response.toStatus(); } diff --git a/src/mongo/s/catalog/sharding_catalog_client_impl.h b/src/mongo/s/catalog/sharding_catalog_client_impl.h index 874c58ec526..e0aa93ad3b6 100644 --- a/src/mongo/s/catalog/sharding_catalog_client_impl.h +++ b/src/mongo/s/catalog/sharding_catalog_client_impl.h @@ -144,11 +144,6 @@ public: const BSONObj& doc, const WriteConcernOptions& writeConcern) override; - void insertConfigDocumentsAsRetryableWrite(OperationContext* opCtx, - const NamespaceString& nss, - std::vector<BSONObj> docs, - const WriteConcernOptions& writeConcern) override; - StatusWith<bool> updateConfigDocument(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& query, diff --git a/src/mongo/s/catalog/sharding_catalog_client_mock.cpp b/src/mongo/s/catalog/sharding_catalog_client_mock.cpp index e3011cb17d7..7f19157811b 100644 --- a/src/mongo/s/catalog/sharding_catalog_client_mock.cpp +++ b/src/mongo/s/catalog/sharding_catalog_client_mock.cpp @@ -152,12 +152,6 @@ Status ShardingCatalogClientMock::insertConfigDocument(OperationContext* opCtx, return {ErrorCodes::InternalError, "Method not implemented"}; } -void ShardingCatalogClientMock::insertConfigDocumentsAsRetryableWrite( - OperationContext* opCtx, - const NamespaceString& nss, - std::vector<BSONObj> docs, - const WriteConcernOptions& writeConcern) {} - StatusWith<bool> ShardingCatalogClientMock::updateConfigDocument( OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/s/catalog/sharding_catalog_client_mock.h b/src/mongo/s/catalog/sharding_catalog_client_mock.h index 9cd6096a358..4cbfd759604 100644 --- a/src/mongo/s/catalog/sharding_catalog_client_mock.h +++ b/src/mongo/s/catalog/sharding_catalog_client_mock.h @@ -121,11 +121,6 @@ public: const BSONObj& doc, const WriteConcernOptions& writeConcern) override; - void insertConfigDocumentsAsRetryableWrite(OperationContext* opCtx, - const NamespaceString& nss, - std::vector<BSONObj> docs, - const WriteConcernOptions& writeConcern) override; - StatusWith<bool> updateConfigDocument(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& query, diff --git a/src/mongo/s/catalog_cache_refresh_test.cpp b/src/mongo/s/catalog_cache_refresh_test.cpp index c958dec98fb..39c8e4b13cd 100644 --- a/src/mongo/s/catalog_cache_refresh_test.cpp +++ b/src/mongo/s/catalog_cache_refresh_test.cpp @@ -33,6 +33,7 @@ #include "mongo/db/concurrency/locker_noop.h" #include "mongo/db/pipeline/aggregation_request_helper.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog/type_database_gen.h" diff --git a/src/mongo/s/catalog_cache_test.cpp b/src/mongo/s/catalog_cache_test.cpp index b894d6cdee6..61a6ca2b067 100644 --- a/src/mongo/s/catalog_cache_test.cpp +++ b/src/mongo/s/catalog_cache_test.cpp @@ -33,6 +33,8 @@ #include <boost/optional/optional_io.hpp> +#include "mongo/db/cursor_id.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/s/catalog/type_database_gen.h" #include "mongo/s/catalog_cache.h" #include "mongo/s/catalog_cache_loader_mock.h" diff --git a/src/mongo/s/catalog_cache_test_fixture.cpp b/src/mongo/s/catalog_cache_test_fixture.cpp index 6e66a30d6b2..9edd553f704 100644 --- a/src/mongo/s/catalog_cache_test_fixture.cpp +++ b/src/mongo/s/catalog_cache_test_fixture.cpp @@ -38,7 +38,9 @@ #include "mongo/client/remote_command_targeter_factory_mock.h" #include "mongo/client/remote_command_targeter_mock.h" #include "mongo/db/client.h" +#include "mongo/db/cursor_id.h" #include "mongo/db/query/collation/collator_factory_mock.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog/type_database_gen.h" diff --git a/src/mongo/s/chunk.cpp b/src/mongo/s/chunk.cpp index 35132c6d82c..e141cfea226 100644 --- a/src/mongo/s/chunk.cpp +++ b/src/mongo/s/chunk.cpp @@ -120,7 +120,7 @@ BSONObj ChunkInfo::toBSON() const { bob.append("maxKeyString", _maxKeyString); bob.append("shardId", _shardId); _lastmod.serializeToBSON("lastmod", &bob); - bob.append("jumbo", _jumbo); + bob.append("jumbo", _jumbo.load()); bob.append("bytesWritten", (long long)_writesTracker->getBytesWritten()); BSONArrayBuilder historyArr{bob.subarrayStart("history")}; @@ -132,7 +132,7 @@ BSONObj ChunkInfo::toBSON() const { } void ChunkInfo::markAsJumbo() { - _jumbo = true; + _jumbo.store(true); } void Chunk::throwIfMoved() const { diff --git a/src/mongo/s/chunk.h b/src/mongo/s/chunk.h index 2c0dabe7a3f..16f5909a72a 100644 --- a/src/mongo/s/chunk.h +++ b/src/mongo/s/chunk.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/platform/atomic_word.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/chunk_version.h" #include "mongo/s/shard_id.h" @@ -92,7 +93,7 @@ public: } bool isJumbo() const { - return _jumbo; + return _jumbo.load(); } /** @@ -132,7 +133,7 @@ private: // Indicates whether this chunk should be treated as jumbo and not attempted to be moved or // split - mutable bool _jumbo; + AtomicWord<bool> _jumbo; // Used for tracking writes to this chunk, to estimate its size for the autosplitter. Since // ChunkInfo objects are always treated as const, and this contains metadata about the chunk diff --git a/src/mongo/s/chunk_manager_targeter.cpp b/src/mongo/s/chunk_manager_targeter.cpp index f750554ea56..f7380a00920 100644 --- a/src/mongo/s/chunk_manager_targeter.cpp +++ b/src/mongo/s/chunk_manager_targeter.cpp @@ -674,7 +674,7 @@ void ChunkManagerTargeter::noteStaleShardResponse(OperationContext* opCtx, Grid::get(opCtx) ->catalogCache() ->invalidateShardOrEntireCollectionEntryForShardedCollection( - _nss, staleInfo.getVersionWanted(), endpoint.shardName); + _nss, boost::none, endpoint.shardName); } _lastError = LastErrorType::kStaleShardVersion; diff --git a/src/mongo/s/client/shard.cpp b/src/mongo/s/client/shard.cpp index ac360694af4..1408b227b1e 100644 --- a/src/mongo/s/client/shard.cpp +++ b/src/mongo/s/client/shard.cpp @@ -41,7 +41,6 @@ namespace mongo { namespace { const int kOnErrorNumRetries = 3; - } // namespace Status Shard::CommandResponse::getEffectiveStatus( @@ -194,36 +193,6 @@ StatusWith<Shard::QueryResponse> Shard::runExhaustiveCursorCommand( MONGO_UNREACHABLE; } -BatchedCommandResponse Shard::runBatchWriteCommand(OperationContext* opCtx, - const Milliseconds maxTimeMS, - const BatchedCommandRequest& batchRequest, - RetryPolicy retryPolicy) { - const StringData dbname = batchRequest.getNS().db(); - const BSONObj cmdObj = batchRequest.toBSON(); - - for (int retry = 1; retry <= kOnErrorNumRetries; ++retry) { - // Note: write commands can only be issued against a primary. - auto swResponse = _runCommand( - opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}, dbname, maxTimeMS, cmdObj); - - BatchedCommandResponse batchResponse; - auto writeStatus = CommandResponse::processBatchWriteResponse(swResponse, &batchResponse); - if (retry < kOnErrorNumRetries && isRetriableError(writeStatus.code(), retryPolicy)) { - LOGV2_DEBUG(22721, - 2, - "Batch write command to shard {shardId} failed with retryable error " - "and will be retried. Caused by {error}", - "Batch write command failed with retryable error and will be retried", - "shardId"_attr = getId(), - "error"_attr = redact(writeStatus)); - continue; - } - - return batchResponse; - } - MONGO_UNREACHABLE; -} - StatusWith<Shard::QueryResponse> Shard::exhaustiveFindOnConfig( OperationContext* opCtx, const ReadPreferenceSetting& readPref, @@ -250,4 +219,34 @@ StatusWith<Shard::QueryResponse> Shard::exhaustiveFindOnConfig( MONGO_UNREACHABLE; } +BatchedCommandResponse Shard::_submitBatchWriteCommand(OperationContext* opCtx, + const BSONObj& serialisedBatchRequest, + StringData dbName, + Milliseconds maxTimeMS, + RetryPolicy retryPolicy) { + for (int retry = 1; retry <= kOnErrorNumRetries; ++retry) { + // Note: write commands can only be issued against a primary. + auto swResponse = _runCommand(opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + dbName, + maxTimeMS, + serialisedBatchRequest); + + BatchedCommandResponse batchResponse; + auto writeStatus = CommandResponse::processBatchWriteResponse(swResponse, &batchResponse); + if (retry < kOnErrorNumRetries && isRetriableError(writeStatus.code(), retryPolicy)) { + LOGV2_DEBUG(22721, + 2, + "Batch write command failed with retryable error and will be retried", + "shardId"_attr = getId(), + "error"_attr = redact(writeStatus)); + continue; + } + + return batchResponse; + } + MONGO_UNREACHABLE; +} + + } // namespace mongo diff --git a/src/mongo/s/client/shard.h b/src/mongo/s/client/shard.h index f690341fcb2..e545f0b2f6b 100644 --- a/src/mongo/s/client/shard.h +++ b/src/mongo/s/client/shard.h @@ -226,10 +226,11 @@ public: * commands return errors in a different format than regular commands do, so checking for * retriable errors must be done differently. */ - BatchedCommandResponse runBatchWriteCommand(OperationContext* opCtx, - Milliseconds maxTimeMS, - const BatchedCommandRequest& batchRequest, - RetryPolicy retryPolicy); + virtual BatchedCommandResponse runBatchWriteCommand(OperationContext* opCtx, + Milliseconds maxTimeMS, + const BatchedCommandRequest& batchRequest, + const WriteConcernOptions& writeConcern, + RetryPolicy retryPolicy) = 0; /** * Warning: This method exhausts the cursor and pulls all data into memory. @@ -292,10 +293,22 @@ public: protected: Shard(const ShardId& id); + /** + * Submits the batch request applying the specified retry policy and timeout and using the + * machinery provided by each implementation. + * Callers of this function must ensure to have configured the write concern settings + * accordingly to their specific semantics. + */ + BatchedCommandResponse _submitBatchWriteCommand(OperationContext* opCtx, + const BSONObj& serialisedBatchRequest, + StringData dbName, + Milliseconds maxTimeMS, + RetryPolicy retryPolicy); + private: /** - * Runs the specified command against the shard backed by this object with a timeout set to the - * minimum of maxTimeMSOverride or the timeout of the OperationContext. + * Runs the specified command against the shard backed by this object with a timeout set to + * the minimum of maxTimeMSOverride or the timeout of the OperationContext. * * The return value exposes RemoteShard's host for calls to updateReplSetMonitor. * diff --git a/src/mongo/s/client/shard_remote.cpp b/src/mongo/s/client/shard_remote.cpp index 9c9241ec7e9..156dad7171a 100644 --- a/src/mongo/s/client/shard_remote.cpp +++ b/src/mongo/s/client/shard_remote.cpp @@ -540,6 +540,23 @@ Status ShardRemote::runAggregation( } +BatchedCommandResponse ShardRemote::runBatchWriteCommand(OperationContext* opCtx, + const Milliseconds maxTimeMS, + const BatchedCommandRequest& batchRequest, + const WriteConcernOptions& writeConcern, + RetryPolicy retryPolicy) { + const auto dbName = batchRequest.getNS().db(); + const BSONObj cmdObj = [&] { + BSONObjBuilder cmdObjBuilder; + batchRequest.serialize(&cmdObjBuilder); + cmdObjBuilder.append(WriteConcernOptions::kWriteConcernField, writeConcern.toBSON()); + return cmdObjBuilder.obj(); + }(); + + return _submitBatchWriteCommand(opCtx, cmdObj, dbName, maxTimeMS, retryPolicy); +} + + StatusWith<ShardRemote::AsyncCmdHandle> ShardRemote::_scheduleCommand( OperationContext* opCtx, const ReadPreferenceSetting& readPref, diff --git a/src/mongo/s/client/shard_remote.h b/src/mongo/s/client/shard_remote.h index 6c99a8a5247..6aec3f87da5 100644 --- a/src/mongo/s/client/shard_remote.h +++ b/src/mongo/s/client/shard_remote.h @@ -91,6 +91,12 @@ public: std::function<bool(const std::vector<BSONObj>& batch, const boost::optional<BSONObj>& postBatchResumeToken)> callback); + BatchedCommandResponse runBatchWriteCommand(OperationContext* opCtx, + Milliseconds maxTimeMS, + const BatchedCommandRequest& batchRequest, + const WriteConcernOptions& writeConcern, + RetryPolicy retryPolicy) final; + private: struct AsyncCmdHandle { HostAndPort hostTargetted; diff --git a/src/mongo/s/cluster_commands_helpers.cpp b/src/mongo/s/cluster_commands_helpers.cpp index 416dbec7324..1a84121bf9e 100644 --- a/src/mongo/s/cluster_commands_helpers.cpp +++ b/src/mongo/s/cluster_commands_helpers.cpp @@ -662,7 +662,7 @@ bool appendEmptyResultSet(OperationContext* opCtx, const std::string& ns) { invariant(!status.isOK()); - CurOp::get(opCtx)->debug().nreturned = 0; + CurOp::get(opCtx)->debug().additiveMetrics.nreturned = 0; CurOp::get(opCtx)->debug().nShards = 0; if (status == ErrorCodes::NamespaceNotFound) { diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index 33bd413b4f7..9f9c43c079a 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -132,6 +132,7 @@ env.Library( '$BUILD_DIR/mongo/db/query/command_request_response', '$BUILD_DIR/mongo/db/query/cursor_response_idl', '$BUILD_DIR/mongo/db/query/map_reduce_output_format', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', '$BUILD_DIR/mongo/db/read_write_concern_defaults', '$BUILD_DIR/mongo/db/repl/hello_auth', '$BUILD_DIR/mongo/db/repl/hello_command', @@ -240,6 +241,7 @@ env.CppUnitTest( "cluster_command_test_fixture.cpp", "cluster_delete_test.cpp", "cluster_distinct_test.cpp", + "cluster_explain_test.cpp", "cluster_find_and_modify_test.cpp", "cluster_find_test.cpp", "cluster_insert_test.cpp", diff --git a/src/mongo/s/commands/cluster_db_stats_cmd.cpp b/src/mongo/s/commands/cluster_db_stats_cmd.cpp index 9154d5f40d8..9439e967521 100644 --- a/src/mongo/s/commands/cluster_db_stats_cmd.cpp +++ b/src/mongo/s/commands/cluster_db_stats_cmd.cpp @@ -131,11 +131,12 @@ public: opCtx, this, CommandHelpers::filterCommandRequestForPassthrough(cmdObj)), ReadPreferenceSetting::get(opCtx), Shard::RetryPolicy::kIdempotent); - if (!appendRawResponses(opCtx, &errmsg, &output, shardResponses).responseOK) { + auto appendResult = appendRawResponses(opCtx, &errmsg, &output, shardResponses); + if (!appendResult.responseOK) { return false; } - aggregateResults(scale, shardResponses, output); + aggregateResults(scale, appendResult.successResponses, output); return true; } diff --git a/src/mongo/s/commands/cluster_explain.cpp b/src/mongo/s/commands/cluster_explain.cpp index 759ce8ed3d4..dd85d94d57f 100644 --- a/src/mongo/s/commands/cluster_explain.cpp +++ b/src/mongo/s/commands/cluster_explain.cpp @@ -30,7 +30,6 @@ #include "mongo/platform/basic.h" #include "mongo/bson/bsonmisc.h" -#include "mongo/db/commands.h" #include "mongo/db/query/explain_common.h" #include "mongo/idl/command_generic_argument.h" #include "mongo/rpc/get_status_from_command_result.h" @@ -115,18 +114,36 @@ void throwOnBadARSResponse(const AsyncRequestsSender::Response& arsResponse) { // static BSONObj ClusterExplain::wrapAsExplain(const BSONObj& cmdObj, ExplainOptions::Verbosity verbosity) { - auto filtered = CommandHelpers::filterCommandRequestForPassthrough(cmdObj); BSONObjBuilder out; - out.append("explain", filtered); - out.append("verbosity", ExplainOptions::verbosityString(verbosity)); - - // Propagate all generic arguments out of the inner command since the shards will only process - // them at the top level. - for (auto elem : filtered) { - if (isGenericArgument(elem.fieldNameStringData())) { - out.append(elem); + // Prune generic arguments out of the inner command since any relevant ones should already + // be provided to the outer explain command. The shards will only process them at the top level. + // As an exception, the "comment" parameter will be propagated out of the inner command to + // maintain the behavior in our documentation: + // https://www.mongodb.com/docs/manual/reference/command/explain/. + // The "readConcern" parameter will also be propagated out of the inner command as the final + // explain command inherits readConcern from the inner command invocation. + BSONObjBuilder explainBuilder = out.subobjStart("explain"); + BSONElement commentField; + BSONElement readConcernField; + for (auto&& elem : cmdObj) { + const auto& fieldName = elem.fieldNameStringData(); + if (!isGenericArgument(fieldName)) { + explainBuilder.append(elem); + } else if (fieldName == "comment"_sd) { + commentField = elem; + } else if (fieldName == "readConcern"_sd) { + readConcernField = elem; } } + explainBuilder.done(); + + out.append("verbosity", ExplainOptions::verbosityString(verbosity)); + if (commentField) { + out.append(commentField); + } + if (readConcernField) { + out.append(readConcernField); + } return out.obj(); } diff --git a/src/mongo/s/commands/cluster_explain.h b/src/mongo/s/commands/cluster_explain.h index f1ff6716de8..9be0ccb960a 100644 --- a/src/mongo/s/commands/cluster_explain.h +++ b/src/mongo/s/commands/cluster_explain.h @@ -46,7 +46,8 @@ class ClusterExplain { public: /** * Returns an explain command request wrapping the passed in command at the given verbosity - * level, propagating generic top-level command arguments. + * level, pruning any generic arguments in the inner command as they should already be provided + * on the top-level outer commmand. */ static BSONObj wrapAsExplain(const BSONObj& cmdObj, ExplainOptions::Verbosity verbosity); diff --git a/src/mongo/s/commands/cluster_explain_test.cpp b/src/mongo/s/commands/cluster_explain_test.cpp new file mode 100644 index 00000000000..298ed85ca61 --- /dev/null +++ b/src/mongo/s/commands/cluster_explain_test.cpp @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/explain_verbosity_gen.h" +#include "mongo/idl/command_generic_argument.h" +#include "mongo/s/commands/cluster_explain.h" +#include "mongo/unittest/bson_test_util.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/assert_util.h" + +namespace mongo { +namespace { +void testPruneGenericArgs(const std::string& genericArg) { + auto internalCmd = fromjson("{find: 'test', filter: {a: 1}, " + genericArg + "}"); + auto verbosity = explain::VerbosityEnum::kQueryPlanner; + auto expected = + fromjson("{explain: {find: 'test', filter: {a: 1}}, verbosity: 'queryPlanner'}"); + ASSERT_BSONOBJ_EQ(ClusterExplain::wrapAsExplain(internalCmd, verbosity), expected); +} + +void testPropagateGenericArgs(const std::string& genericArg) { + auto internalCmd = fromjson("{find: 'test', filter: {a: 1}, " + genericArg + "}"); + auto verbosity = explain::VerbosityEnum::kQueryPlanner; + auto expected = fromjson( + "{explain: {find: 'test', filter: {a: 1}}, verbosity: 'queryPlanner', " + genericArg + "}"); + ASSERT_BSONOBJ_EQ(ClusterExplain::wrapAsExplain(internalCmd, verbosity), expected); +} + +TEST(ClusterExplainTest, PruneMaxTimeMS) { + std::string maxTimeMS = "maxTimeMS: 1"; + testPruneGenericArgs(maxTimeMS); +} + +TEST(ClusterExplainTest, PruneWriteConcern) { + std::string writeConcern = "writeConcern: {w: 1}"; + testPruneGenericArgs(writeConcern); +} + +TEST(ClusterExplainTest, PruneLsid) { + auto internalCmd = BSON("find" + << "test" + << "filter" << BSON("a" << 1) << "lsid" + << BSON("id" << mongo::UUID::gen())); + auto verbosity = explain::VerbosityEnum::kQueryPlanner; + auto expected = + fromjson("{explain: {find: 'test', filter: {a: 1}}, verbosity: 'queryPlanner'}"); + ASSERT_BSONOBJ_EQ(ClusterExplain::wrapAsExplain(internalCmd, verbosity), expected); +} + +TEST(ClusterExplainTest, PruneReadPreference) { + std::string readPreference = "$queryOptions: {$readPreference: 'secondary'}"; + testPruneGenericArgs(readPreference); +} + +TEST(ClusterExplainTest, PruneClusterTime) { + auto internalCmd = BSON("find" + << "test" + << "filter" << BSON("a" << 1) << "$clusterTime" + << BSON("clusterTime" << Timestamp(2, 2)) << "$configTime" + << Timestamp(2, 2) << "$topologyTime" << Timestamp(2, 2)); + auto verbosity = explain::VerbosityEnum::kQueryPlanner; + auto expected = + fromjson("{explain: {find: 'test', filter: {a: 1}}, verbosity: 'queryPlanner'}"); + ASSERT_BSONOBJ_EQ(ClusterExplain::wrapAsExplain(internalCmd, verbosity), expected); +} + +TEST(ClusterExplainTest, PropagateComment) { + std::string comment = "comment: 'Quetzlcoatl'"; + testPropagateGenericArgs(comment); +} + +TEST(ClusterExplainTest, PropagateReadConcern) { + std::string readConcern = "readConcern: {level: 'linearizable'}"; + testPropagateGenericArgs(readConcern); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/s/commands/cluster_find_cmd.h b/src/mongo/s/commands/cluster_find_cmd.h index 7d58f7ab0c4..6476d883200 100644 --- a/src/mongo/s/commands/cluster_find_cmd.h +++ b/src/mongo/s/commands/cluster_find_cmd.h @@ -38,6 +38,9 @@ #include "mongo/db/fle_crud.h" #include "mongo/db/matcher/extensions_callback_noop.h" #include "mongo/db/query/cursor_response.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_stats/find_key.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/stats/counters.h" #include "mongo/db/views/resolved_view.h" #include "mongo/rpc/get_status_from_command_result.h" @@ -201,16 +204,23 @@ public: Impl::checkCanRunHere(opCtx); - auto findCommand = _parseCmdObjectToFindCommandRequest(opCtx, ns(), _request.body); - - const boost::intrusive_ptr<ExpressionContext> expCtx; - auto cq = uassertStatusOK( - CanonicalQuery::canonicalize(opCtx, - std::move(findCommand), - false, /* isExplain */ - expCtx, - ExtensionsCallbackNoop(), - MatchExpressionParser::kAllowAllSpecialFeatures)); + auto&& parsedFindResult = uassertStatusOK(parsed_find_command::parse( + opCtx, + _parseCmdObjectToFindCommandRequest(opCtx, ns(), _request.body), + ExtensionsCallbackNoop(), + MatchExpressionParser::kAllowAllSpecialFeatures)); + auto& expCtx = parsedFindResult.first; + auto& parsedFind = parsedFindResult.second; + + if (!_didDoFLERewrite) { + query_stats::registerRequest(opCtx, expCtx->ns, [&]() { + // This callback is either never invoked or invoked + // immediately within registerRequest, so + // use-after-move of parsedFind isn't an issue. + return std::make_unique<query_stats::FindKey>(expCtx, *parsedFind); + }); + } + auto cq = uassertStatusOK(CanonicalQuery::canonicalize(expCtx, std::move(parsedFind))); try { // Do the work to generate the first batch of results. This blocks waiting to get @@ -264,7 +274,7 @@ public: * were supplied with the command, and sets the constant runtime values that will be * forwarded to each shard. */ - static std::unique_ptr<FindCommandRequest> _parseCmdObjectToFindCommandRequest( + std::unique_ptr<FindCommandRequest> _parseCmdObjectToFindCommandRequest( OperationContext* opCtx, NamespaceString nss, BSONObj cmdObj) { auto findCommand = query_request_helper::makeFromFindCommand( std::move(cmdObj), @@ -291,6 +301,7 @@ public: invariant(findCommand->getNamespaceOrUUID().nss()); processFLEFindS( opCtx, findCommand->getNamespaceOrUUID().nss().get(), findCommand.get()); + _didDoFLERewrite = true; } return findCommand; @@ -298,6 +309,7 @@ public: const OpMsgRequest& _request; const StringData _dbName; + bool _didDoFLERewrite{false}; }; }; diff --git a/src/mongo/s/commands/cluster_fle2_compact_cmd.cpp b/src/mongo/s/commands/cluster_fle2_compact_cmd.cpp index b1cecbf555e..7abae202f17 100644 --- a/src/mongo/s/commands/cluster_fle2_compact_cmd.cpp +++ b/src/mongo/s/commands/cluster_fle2_compact_cmd.cpp @@ -32,6 +32,7 @@ #include "mongo/db/auth/authorization_session.h" #include "mongo/db/commands.h" #include "mongo/db/commands/fle2_compact_gen.h" +#include "mongo/db/curop.h" #include "mongo/s/cluster_commands_helpers.h" #include "mongo/s/grid.h" diff --git a/src/mongo/s/commands/cluster_write_cmd.cpp b/src/mongo/s/commands/cluster_write_cmd.cpp index 637ad92085d..8590832acc6 100644 --- a/src/mongo/s/commands/cluster_write_cmd.cpp +++ b/src/mongo/s/commands/cluster_write_cmd.cpp @@ -145,11 +145,6 @@ boost::optional<WouldChangeOwningShardInfo> getWouldChangeOwningShardErrorInfo( void handleWouldChangeOwningShardErrorRetryableWrite(OperationContext* opCtx, BatchedCommandRequest* request, BatchedCommandResponse* response) { - // Strip write concern because this command will be sent as part of a - // transaction and the write concern has already been loaded onto the opCtx and - // will be picked up by the transaction API. - request->unsetWriteConcern(); - // Strip runtime constants because they will be added again when the API sends this command // through the service entry point. request->unsetLegacyRuntimeConstants(); @@ -318,11 +313,6 @@ bool handleWouldChangeOwningShardError(OperationContext* opCtx, auto& readConcernArgs = repl::ReadConcernArgs::get(opCtx); readConcernArgs = repl::ReadConcernArgs(repl::ReadConcernLevel::kLocalReadConcern); - // Ensure the retried operation does not include WC inside the transaction. The - // transaction commit will still use the WC, because it uses the WC from the opCtx - // (which has been set previously in Strategy). - request->unsetWriteConcern(); - documentShardKeyUpdateUtil::startTransactionForShardKeyUpdate(opCtx); // Clear the error details from the response object before sending the write again response->unsetErrDetails(); @@ -507,22 +497,6 @@ bool ClusterWriteCmd::InvocationBase::runImpl(OperationContext* opCtx, BatchWriteExecStats stats; BatchedCommandResponse response; - // Append the write concern from the opCtx extracted during command setup. - if (!batchedRequest.hasWriteConcern()) { - batchedRequest.setWriteConcern(opCtx->getWriteConcern().toBSON()); - } - - // Write ops are never allowed to have writeConcern inside transactions. Normally - // disallowing WC on non-terminal commands in a transaction is handled earlier, during - // command dispatch. However, if this is a regular write operation being automatically - // retried inside a transaction (such as changing a document's shard key across shards), - // then batchedRequest will have a writeConcern (added by the if() above) from when it was - // initially run outside a transaction. Thus it's necessary to unconditionally clear the - // writeConcern when in a transaction. - if (TransactionRouter::get(opCtx)) { - batchedRequest.unsetWriteConcern(); - } - cluster::write(opCtx, batchedRequest, &stats, &response); bool updatedShardKey = false; diff --git a/src/mongo/s/mongos_main.cpp b/src/mongo/s/mongos_main.cpp index dd78e7772d1..665f3fe08d4 100644 --- a/src/mongo/s/mongos_main.cpp +++ b/src/mongo/s/mongos_main.cpp @@ -316,7 +316,10 @@ void cleanupTask(const ShutdownTaskArgs& shutdownArgs) { ReplicaSetMonitor::shutdown(); - opCtx->setIsExecutingShutdown(); + { + stdx::lock_guard lg(client); + opCtx->setIsExecutingShutdown(); + } if (serviceContext) { serviceContext->setKillAllOperations(); diff --git a/src/mongo/s/query/SConscript b/src/mongo/s/query/SConscript index 94bfbde4878..f8179e37eeb 100644 --- a/src/mongo/s/query/SConscript +++ b/src/mongo/s/query/SConscript @@ -8,17 +8,17 @@ env.Library( target="cluster_query", source=[ "cluster_find.cpp", - 'cluster_query_knobs.idl', + "cluster_query_knobs.idl", + "store_possible_cursor.cpp", ], LIBDEPS=[ '$BUILD_DIR/mongo/db/commands', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/curop_failpoint_helpers', '$BUILD_DIR/mongo/db/query/query_common', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/s/sharding_router_api', "cluster_client_cursor", "cluster_cursor_cleanup_job", - "store_possible_cursor", ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog/collection_uuid_mismatch_info', @@ -37,6 +37,7 @@ env.Library( '$BUILD_DIR/mongo/db/pipeline/pipeline', '$BUILD_DIR/mongo/db/pipeline/process_interface/mongos_process_interface', '$BUILD_DIR/mongo/db/pipeline/sharded_agg_helpers', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', '$BUILD_DIR/mongo/db/views/view_catalog_helpers', '$BUILD_DIR/mongo/db/views/views', '$BUILD_DIR/mongo/s/query/cluster_client_cursor', @@ -93,24 +94,11 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog/collection_uuid_mismatch_info', + '$BUILD_DIR/mongo/executor/async_multicaster', ] ) env.Library( - target="store_possible_cursor", - source=[ - "store_possible_cursor.cpp" - ], - LIBDEPS=[ - "$BUILD_DIR/mongo/base", - "$BUILD_DIR/mongo/db/curop", - "$BUILD_DIR/mongo/db/query/command_request_response", - "cluster_client_cursor", - "cluster_cursor_manager", - ], -) - -env.Library( target="cluster_cursor_manager", source=[ "cluster_cursor_manager.cpp", @@ -119,12 +107,12 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/auth/auth', '$BUILD_DIR/mongo/db/auth/authprivilege', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/generic_cursor', '$BUILD_DIR/mongo/db/kill_sessions', '$BUILD_DIR/mongo/db/logical_session_cache', '$BUILD_DIR/mongo/db/logical_session_id', '$BUILD_DIR/mongo/db/query/query_knobs', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', ], ) @@ -171,7 +159,7 @@ env.CppUnitTest( "cluster_aggregate", "cluster_client_cursor", "cluster_cursor_manager", + "cluster_query", "router_exec_stage", - "store_possible_cursor", ], ) diff --git a/src/mongo/s/query/async_results_merger.cpp b/src/mongo/s/query/async_results_merger.cpp index 50fb6310888..3ea786af9ae 100644 --- a/src/mongo/s/query/async_results_merger.cpp +++ b/src/mongo/s/query/async_results_merger.cpp @@ -161,6 +161,10 @@ AsyncResultsMerger::~AsyncResultsMerger() { invariant(_remotesExhausted(lk) || _lifecycleState == kKillComplete); } +const AsyncResultsMergerParams& AsyncResultsMerger::params() const { + return _params; +} + bool AsyncResultsMerger::remotesExhausted() const { stdx::lock_guard<Latch> lk(_mutex); return _remotesExhausted(lk); diff --git a/src/mongo/s/query/async_results_merger.h b/src/mongo/s/query/async_results_merger.h index 3fde29e141e..518a129b0ec 100644 --- a/src/mongo/s/query/async_results_merger.h +++ b/src/mongo/s/query/async_results_merger.h @@ -109,6 +109,11 @@ public: ~AsyncResultsMerger(); /** + * Returns a const reference to the parameters. + */ + const AsyncResultsMergerParams& params() const; + + /** * Returns true if all of the remote cursors are exhausted. */ bool remotesExhausted() const; @@ -485,7 +490,7 @@ private: OperationContext* _opCtx; std::shared_ptr<executor::TaskExecutor> _executor; TailableModeEnum _tailableMode; - AsyncResultsMergerParams _params; + const AsyncResultsMergerParams _params; // Must be acquired before accessing any data members (other than _params, which is read-only). mutable Mutex _mutex = MONGO_MAKE_LATCH("AsyncResultsMerger::_mutex"); diff --git a/src/mongo/s/query/async_results_merger_params.idl b/src/mongo/s/query/async_results_merger_params.idl index e3c4d03bdd3..5382c9cc718 100644 --- a/src/mongo/s/query/async_results_merger_params.idl +++ b/src/mongo/s/query/async_results_merger_params.idl @@ -50,15 +50,19 @@ types: structs: RemoteCursor: description: A description of a cursor opened on a remote server. + query_shape_component: true fields: shardId: type: string description: The shardId of the shard on which the cursor resides. + query_shape: anonymize hostAndPort: type: HostAndPort description: The exact host (within the shard) on which the cursor resides. + query_shape: anonymize cursorResponse: type: CursorResponse + query_shape: literal description: The response after establishing a cursor on the remote shard, including the first batch. @@ -66,35 +70,46 @@ structs: description: The parameters needed to establish an AsyncResultsMerger. chained_structs: OperationSessionInfoFromClient : OperationSessionInfo + query_shape_component: true fields: sort: type: object description: The sort requested on the merging operation. Empty if there is no sort. optional: true + query_shape: literal compareWholeSortKey: type: bool default: false + query_shape: literal description: >- When 'compareWholeSortKey' is true, $sortKey is a scalar value, rather than an object. We extract the sort key {$sortKey: <value>}. The sort key pattern is verified to be {$sortKey: 1}. - remotes: array<RemoteCursor> + remotes: + type: array<RemoteCursor> + query_shape: literal tailableMode: type: TailableMode optional: true description: If set, the tailability mode of this cursor. + query_shape: parameter batchSize: type: safeInt64 optional: true description: The batch size for this cursor. - nss: namespacestring + query_shape: literal + nss: + type: namespacestring + query_shape: custom allowPartialResults: type: bool default: false description: If set, error responses are ignored. + query_shape: parameter recordRemoteOpWaitTime: type: bool default: false + query_shape: parameter description: >- This parameter is not used anymore but should stay for a while for backward compatibility. diff --git a/src/mongo/s/query/async_results_merger_test.cpp b/src/mongo/s/query/async_results_merger_test.cpp index 80600272e63..cc1e1e473f3 100644 --- a/src/mongo/s/query/async_results_merger_test.cpp +++ b/src/mongo/s/query/async_results_merger_test.cpp @@ -2022,5 +2022,27 @@ TEST_F(AsyncResultsMergerTest, ShouldNotScheduleGetMoresWithoutAnOperationContex killFuture.wait(); } +TEST_F(AsyncResultsMergerTest, CanAccessParams) { + std::vector<RemoteCursor> cursors; + cursors.push_back( + makeRemoteCursor(kTestShardIds[0], kTestShardHosts[0], CursorResponse(kTestNss, 5, {}))); + auto arm = makeARMFromExistingCursors(std::move(cursors)); + + // Check actual parameters. + ASSERT_EQ(kTestNss, arm->params().getNss()); + ASSERT_EQ(1, arm->params().getRemotes().size()); + + // Schedule requests. We need to do this because the dtor of AsyncResultsMerger fires an + // assertion if the remotes are not exhausted and the AsyncResultsMerger hasn't been killed. + auto readyEvent = unittest::assertGet(arm->nextEvent()); + std::vector<CursorResponse> responses; + std::vector<BSONObj> batch = {fromjson("{_id: 1}"), fromjson("{_id: 2}"), fromjson("{_id: 3}")}; + responses.emplace_back(kTestNss, CursorId(0), batch); + scheduleNetworkResponses(std::move(responses)); + + // Now the AsyncResultsMerger can go out of scope without triggering the assertion failure. + ASSERT_TRUE(arm->remotesExhausted()); +} + } // namespace } // namespace mongo diff --git a/src/mongo/s/query/blocking_results_merger.cpp b/src/mongo/s/query/blocking_results_merger.cpp index fc56a0d9e3b..7cb19e3eb91 100644 --- a/src/mongo/s/query/blocking_results_merger.cpp +++ b/src/mongo/s/query/blocking_results_merger.cpp @@ -46,6 +46,10 @@ BlockingResultsMerger::BlockingResultsMerger(OperationContext* opCtx, _arm(opCtx, std::move(executor), std::move(armParams)), _resourceYielder(std::move(resourceYielder)) {} +const AsyncResultsMergerParams& BlockingResultsMerger::asyncResultsMergerParams() const { + return _arm.params(); +} + StatusWith<stdx::cv_status> BlockingResultsMerger::doWaiting( OperationContext* opCtx, const std::function<StatusWith<stdx::cv_status>()>& waitFn) noexcept { diff --git a/src/mongo/s/query/blocking_results_merger.h b/src/mongo/s/query/blocking_results_merger.h index c05cecc5da8..9c0e78f9ba5 100644 --- a/src/mongo/s/query/blocking_results_merger.h +++ b/src/mongo/s/query/blocking_results_merger.h @@ -48,6 +48,11 @@ public: std::unique_ptr<ResourceYielder> resourceYielder); /** + * Returns a const reference to the AsyncResultsMergerParams owned by the AsyncResultsMerger. + */ + const AsyncResultsMergerParams& asyncResultsMergerParams() const; + + /** * Blocks until the next result is available or an error is detected. */ StatusWith<ClusterQueryResult> next(OperationContext*); diff --git a/src/mongo/s/query/blocking_results_merger_test.cpp b/src/mongo/s/query/blocking_results_merger_test.cpp index 15e37b0460d..41dd31b895e 100644 --- a/src/mongo/s/query/blocking_results_merger_test.cpp +++ b/src/mongo/s/query/blocking_results_merger_test.cpp @@ -292,5 +292,20 @@ TEST_F(ResultsMergerTestFixture, ShouldBeAbleToHandleExceptionWhenUnyielding) { future.default_timed_get(); } +TEST_F(ResultsMergerTestFixture, CanAccessAsyncResultsMergerParams) { + std::vector<RemoteCursor> cursors; + cursors.emplace_back( + makeRemoteCursor(kTestShardIds[0], kTestShardHosts[0], CursorResponse(kTestNss, 1, {}))); + auto params = makeARMParamsFromExistingCursors(std::move(cursors)); + BlockingResultsMerger blockingMerger( + operationContext(), std::move(params), executor(), nullptr); + + ASSERT_EQ(kTestNss, blockingMerger.asyncResultsMergerParams().getNss()); + ASSERT_EQ(1, blockingMerger.asyncResultsMergerParams().getRemotes().size()); + + // Kill merger because otherwise it will run into an assertion in its dtor. + blockingMerger.kill(operationContext()); +} + } // namespace } // namespace mongo diff --git a/src/mongo/s/query/cluster_aggregate.cpp b/src/mongo/s/query/cluster_aggregate.cpp index 3d6e9b5c2af..6374bcfd494 100644 --- a/src/mongo/s/query/cluster_aggregate.cpp +++ b/src/mongo/s/query/cluster_aggregate.cpp @@ -27,6 +27,7 @@ * it in the license file. */ +#include "mongo/s/chunk_manager.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand #include "mongo/platform/basic.h" @@ -56,6 +57,9 @@ #include "mongo/db/query/explain_common.h" #include "mongo/db/query/find_common.h" #include "mongo/db/query/fle/server_rewrite.h" +#include "mongo/db/query/query_stats/agg_key.h" +#include "mongo/db/query/query_stats/key.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/db/timeseries/timeseries_options.h" #include "mongo/db/views/resolved_view.h" #include "mongo/db/views/view.h" @@ -95,7 +99,7 @@ namespace { // definition. It's okay that this is incorrect, we will repopulate the real namespace map on the // mongod. Note that this function must be called before forwarding an aggregation command on an // unsharded collection, in order to verify that the involved namespaces are allowed to be sharded. -auto resolveInvolvedNamespaces(stdx::unordered_set<NamespaceString> involvedNamespaces) { +auto resolveInvolvedNamespaces(const stdx::unordered_set<NamespaceString>& involvedNamespaces) { StringMap<ExpressionContext::ResolvedNamespace> resolvedNamespaces; for (auto&& nss : involvedNamespaces) { resolvedNamespaces.try_emplace(nss.coll(), nss, std::vector<BSONObj>{}); @@ -258,6 +262,68 @@ std::vector<BSONObj> rebuildPipelineWithTimeSeriesGranularity(const std::vector< return newPipeline; } +/** + * Builds an expCtx with which to parse the request's pipeline, then parses the pipeline and + * registers the pre-optimized pipeline with query stats collection. + */ +std::unique_ptr<Pipeline, PipelineDeleter> parsePipelineAndRegisterQueryStats( + OperationContext* opCtx, + const stdx::unordered_set<NamespaceString>& involvedNamespaces, + const NamespaceString& executionNss, + AggregateCommandRequest& request, + const boost::optional<ChunkManager>& cm, + const LiteParsedPipeline& liteParsedPipeline, + bool hasChangeStream, + bool shouldDoFLERewrite) { + // Populate the collection UUID and the appropriate collation to use. + auto [collationObj, uuid] = [&]() -> std::pair<BSONObj, boost::optional<UUID>> { + // If this is a change stream, take the user-defined collation if one exists, or an + // empty BSONObj otherwise. Change streams never inherit the collection's default + // collation, and since collectionless aggregations generally run on the 'admin' + // database, the standard logic would attempt to resolve its non-existent UUID and + // collation by sending a specious 'listCollections' command to the config servers. + if (hasChangeStream) { + return {request.getCollation().value_or(BSONObj()), boost::none}; + } + + return cluster_aggregation_planner::getCollationAndUUID( + opCtx, cm, executionNss, request.getCollation().value_or(BSONObj())); + }(); + + // Build an ExpressionContext for the pipeline. This instantiates an appropriate collator, + // resolves all involved namespaces, and creates a shared MongoProcessInterface for use by the + // pipeline's stages. + boost::intrusive_ptr<ExpressionContext> expCtx = + makeExpressionContext(opCtx, + request, + collationObj, + uuid, + resolveInvolvedNamespaces(involvedNamespaces), + hasChangeStream); + + // A pipeline with $changeStreamSplitLargeEvent requires the use of resume token format v2, + // since the 'fragmentNum' field only exists in this version and later. + if (hasChangeStream && liteParsedPipeline.endsWithChangeStreamSplitLargeEvent()) { + expCtx->changeStreamTokenVersion = 2; + } + + // Parse and optimize the full pipeline. + auto pipeline = Pipeline::parse(request.getPipeline(), expCtx); + + // Skip query stats recording for queryable encryption queries. + if (!shouldDoFLERewrite) { + query_stats::registerRequest( + opCtx, + executionNss, + [&]() { + return std::make_unique<query_stats::AggKey>( + request, *pipeline, expCtx, involvedNamespaces, executionNss); + }, + hasChangeStream); + } + return pipeline; +} + } // namespace Status ClusterAggregate::runAggregate(OperationContext* opCtx, @@ -351,39 +417,15 @@ Status ClusterAggregate::runAggregate(OperationContext* opCtx, boost::intrusive_ptr<ExpressionContext> expCtx; const auto pipelineBuilder = [&]() { - // Populate the collection UUID and the appropriate collation to use. - auto [collationObj, uuid] = [&]() -> std::pair<BSONObj, boost::optional<UUID>> { - // If this is a change stream, take the user-defined collation if one exists, or an - // empty BSONObj otherwise. Change streams never inherit the collection's default - // collation, and since collectionless aggregations generally run on the 'admin' - // database, the standard logic would attempt to resolve its non-existent UUID and - // collation by sending a specious 'listCollections' command to the config servers. - if (hasChangeStream) { - return {request.getCollation().value_or(BSONObj()), boost::none}; - } - - return cluster_aggregation_planner::getCollationAndUUID( - opCtx, cm, namespaces.executionNss, request.getCollation().value_or(BSONObj())); - }(); - - // Build an ExpressionContext for the pipeline. This instantiates an appropriate collator, - // resolves all involved namespaces, and creates a shared MongoProcessInterface for use by - // the pipeline's stages. - expCtx = makeExpressionContext(opCtx, - request, - collationObj, - uuid, - resolveInvolvedNamespaces(involvedNamespaces), - hasChangeStream); - - // A pipeline with $changeStreamSplitLargeEvent requires the use of resume token format v2, - // since the 'fragmentNum' field only exists in this version and later. - if (hasChangeStream && liteParsedPipeline.endsWithChangeStreamSplitLargeEvent()) { - expCtx->changeStreamTokenVersion = 2; - } - - // Parse and optimize the full pipeline. - auto pipeline = Pipeline::parse(request.getPipeline(), expCtx); + auto pipeline = parsePipelineAndRegisterQueryStats(opCtx, + involvedNamespaces, + namespaces.executionNss, + request, + cm, + liteParsedPipeline, + hasChangeStream, + shouldDoFLERewrite); + expCtx = pipeline->getContext(); // If the aggregate command supports encrypted collections, do rewrites of the pipeline to // support querying against encrypted fields. @@ -429,15 +471,48 @@ Status ClusterAggregate::runAggregate(OperationContext* opCtx, cluster_aggregation_planner::AggregationTargeter::TargetingPolicy::kMongosRequired); if (!expCtx) { - // When the AggregationTargeter chooses a "passthrough" policy, it does not call the - // 'pipelineBuilder' function, so we never get an expression context. Because this is a - // passthrough, we only need a bare minimum expression context anyway. + // When the AggregationTargeter chooses a "passthrough" or "specific shard only" policy, it + // does not call the 'pipelineBuilder' function, so we've yet to construct an expression + // context or register query stats. Because this is a passthrough, we only need a bare + // minimum expression context on mongos. invariant(targeter.policy == cluster_aggregation_planner::AggregationTargeter::kPassthrough || targeter.policy == cluster_aggregation_planner::AggregationTargeter::kSpecificShardOnly); + expCtx = make_intrusive<ExpressionContext>( opCtx, nullptr, namespaces.executionNss, boost::none, request.getLet()); + expCtx->addResolvedNamespaces(involvedNamespaces); + + + // We might need 'inMongos' temporarily set to true for query stats parsing, but we don't + // want to modify the value of 'expCtx' for future code execution so we will set it back to + // its original value. + ON_BLOCK_EXIT([&expCtx, originalInMongosVal = expCtx->inMongos]() { + expCtx->inMongos = originalInMongosVal; + }); + + // In order to parse a change stream request for query stats, 'inMongos' needs + // to be set to true. + if (hasChangeStream) { + expCtx->inMongos = true; + } + + // Skip query stats recording for queryable encryption queries. + if (!shouldDoFLERewrite) { + // We want to hold off parsing the pipeline until it's clear we must. Because of that, + // we wait to parse the pipeline until this callback is invoked within + // query_stats::registerRequest. + query_stats::registerRequest( + opCtx, + namespaces.executionNss, + [&]() { + auto pipeline = Pipeline::parse(request.getPipeline(), expCtx); + return std::make_unique<query_stats::AggKey>( + request, *pipeline, expCtx, involvedNamespaces, namespaces.executionNss); + }, + hasChangeStream); + } } if (request.getExplain()) { @@ -465,10 +540,11 @@ Status ClusterAggregate::runAggregate(OperationContext* opCtx, // If this is an explain write the explain output and return. auto expCtx = targeter.pipeline->getContext(); if (expCtx->explain) { + auto opts = SerializationOptions{}; + opts.verbosity = boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner); *result << "splitPipeline" << BSONNULL << "mongos" << Document{{"host", getHostNameCachedAndPort()}, - {"stages", - targeter.pipeline->writeExplainOps(*expCtx->explain)}}; + {"stages", targeter.pipeline->writeExplainOps(opts)}}; return Status::OK(); } @@ -537,11 +613,12 @@ Status ClusterAggregate::runAggregate(OperationContext* opCtx, updateHostsTargetedMetrics(opCtx, namespaces.executionNss, cm, involvedNamespaces); // Report usage statistics for each stage in the pipeline. liteParsedPipeline.tickGlobalStageCounters(); - // Add 'command' object to explain output. if (expCtx->explain) { explain_common::appendIfRoom( aggregation_request_helper::serializeToCommandObj(request), "command", result); + collectQueryStatsMongos(opCtx, + std::move(CurOp::get(opCtx)->debug().queryStatsInfo.key)); } } return status; diff --git a/src/mongo/s/query/cluster_aggregation_planner.cpp b/src/mongo/s/query/cluster_aggregation_planner.cpp index e124b63c4f9..70adf70404a 100644 --- a/src/mongo/s/query/cluster_aggregation_planner.cpp +++ b/src/mongo/s/query/cluster_aggregation_planner.cpp @@ -345,12 +345,27 @@ BSONObj establishMergingMongosCursor(OperationContext* opCtx, responseBuilder.setPostBatchResumeToken(ccc->getPostBatchResumeToken()); } + bool exhausted = cursorState != ClusterCursorManager::CursorState::NotExhausted; + int nShards = ccc->getNumRemotes(); + + auto&& opDebug = CurOp::get(opCtx)->debug(); + // Fill out the aggregation metrics in CurOp, and record queryStats metrics, before detaching + // the cursor from its opCtx. + opDebug.nShards = std::max(opDebug.nShards, nShards); + opDebug.cursorExhausted = exhausted; + opDebug.additiveMetrics.nBatches = 1; + CurOp::get(opCtx)->setEndOfOpMetrics(responseBuilder.numDocs()); + if (exhausted) { + collectQueryStatsMongos(opCtx, ccc->takeKey()); + } else { + collectQueryStatsMongos(opCtx, ccc); + } + ccc->detachFromOperationContext(); - int nShards = ccc->getNumRemotes(); CursorId clusterCursorId = 0; - if (cursorState == ClusterCursorManager::CursorState::NotExhausted) { + if (!exhausted) { auto authUsers = AuthorizationSession::get(opCtx->getClient())->getAuthenticatedUserNames(); clusterCursorId = uassertStatusOK(Grid::get(opCtx)->getCursorManager()->registerCursor( opCtx, @@ -359,16 +374,9 @@ BSONObj establishMergingMongosCursor(OperationContext* opCtx, ClusterCursorManager::CursorType::MultiTarget, ClusterCursorManager::CursorLifetime::Mortal, authUsers)); + opDebug.cursorid = clusterCursorId; } - // Fill out the aggregation metrics in CurOp. - if (clusterCursorId > 0) { - CurOp::get(opCtx)->debug().cursorid = clusterCursorId; - } - CurOp::get(opCtx)->debug().nShards = std::max(CurOp::get(opCtx)->debug().nShards, nShards); - CurOp::get(opCtx)->debug().cursorExhausted = (clusterCursorId == 0); - CurOp::get(opCtx)->debug().nreturned = responseBuilder.numDocs(); - responseBuilder.done(clusterCursorId, requestedNss.ns()); auto bodyBuilder = replyBuilder.getBodyBuilder(); @@ -599,12 +607,13 @@ AggregationTargeter AggregationTargeter::make( }(); // Determine whether this aggregation must be dispatched to all shards in the cluster. - const bool mustRunOnAll = - sharded_agg_helpers::mustRunOnAllShards(executionNss, hasChangeStream, startsWithDocuments); + const bool mustRunOnAllShards = sharded_agg_helpers::checkIfMustRunOnAllShards( + executionNss, hasChangeStream, startsWithDocuments); // If we don't have a routing table, then this is either a $changeStream which must run on all // shards or a $documents stage which must not. - invariant(cm || (mustRunOnAll && hasChangeStream) || (startsWithDocuments && !mustRunOnAll)); + invariant(cm || (mustRunOnAllShards && hasChangeStream) || + (startsWithDocuments && !mustRunOnAllShards)); // A pipeline is allowed to passthrough to the primary shard iff the following conditions are // met: @@ -616,7 +625,7 @@ AggregationTargeter AggregationTargeter::make( // $currentOp. // 4. Doesn't need transformation via DocumentSource::serialize(). For example, list sessions // needs to include information about users that can only be deduced on mongos. - if (cm && !cm->isSharded() && !mustRunOnAll && allowedToPassthrough && + if (cm && !cm->isSharded() && !mustRunOnAllShards && allowedToPassthrough && !involvesShardedCollections) { return AggregationTargeter{TargetingPolicy::kPassthrough, nullptr, cm}; } else { @@ -858,6 +867,7 @@ Status runPipelineOnSpecificShardOnly(const boost::intrusive_ptr<ExpressionConte if (explain) { // If this was an explain, then we get back an explain result object rather than a cursor. result = response.swResponse.getValue().data; + collectQueryStatsMongos(opCtx, std::move(CurOp::get(opCtx)->debug().queryStatsInfo.key)); } else { result = uassertStatusOK(storePossibleCursor( opCtx, diff --git a/src/mongo/s/query/cluster_client_cursor.h b/src/mongo/s/query/cluster_client_cursor.h index 8ff611eb308..23a3367416d 100644 --- a/src/mongo/s/query/cluster_client_cursor.h +++ b/src/mongo/s/query/cluster_client_cursor.h @@ -211,15 +211,30 @@ public: */ virtual boost::optional<uint32_t> getQueryHash() const = 0; + virtual boost::optional<std::size_t> getQueryStatsKeyHash() const = 0; + + virtual bool getQueryStatsWillNeverExhaust() const = 0; + /** * Returns the number of batches returned by this cursor. */ - virtual std::uint64_t getNBatches() const = 0; + std::uint64_t getNBatches() const { + return _metrics.nBatches.value_or(0); + } /** * Increment the number of batches returned so far by one. */ - virtual void incNBatches() = 0; + void incNBatches() { + _metrics.incrementNBatches(); + } + + void incrementCursorMetrics(OpDebug::AdditiveMetrics newMetrics) { + _metrics.add(newMetrics); + if (!_firstResponseExecutionTime) { + _firstResponseExecutionTime = _metrics.executionTime; + } + } // // maxTimeMS support. @@ -245,6 +260,20 @@ public: _leftoverMaxTimeMicros = leftoverMaxTimeMicros; } + /** + * Returns and releases ownership of the Key associated with the request this + * cursor is handling. + */ + virtual std::unique_ptr<query_stats::Key> takeKey() = 0; + +protected: + // Metrics that are accumulated over the lifetime of the cursor, incremented with each getMore. + // Useful for diagnostics like queryStats. + OpDebug::AdditiveMetrics _metrics; + + // The execution time collected from the initial operation prior to any getMore requests. + boost::optional<Microseconds> _firstResponseExecutionTime; + private: // Unused maxTime budget for this cursor. Microseconds _leftoverMaxTimeMicros = Microseconds::max(); diff --git a/src/mongo/s/query/cluster_client_cursor_impl.cpp b/src/mongo/s/query/cluster_client_cursor_impl.cpp index 73be5a7512a..6b094a604f4 100644 --- a/src/mongo/s/query/cluster_client_cursor_impl.cpp +++ b/src/mongo/s/query/cluster_client_cursor_impl.cpp @@ -27,6 +27,8 @@ * it in the license file. */ +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + #include "mongo/platform/basic.h" #include "mongo/s/query/cluster_client_cursor_impl.h" @@ -34,6 +36,8 @@ #include <memory> #include "mongo/db/curop.h" +#include "mongo/db/query/query_stats/query_stats.h" +#include "mongo/logv2/log.h" #include "mongo/s/query/router_stage_limit.h" #include "mongo/s/query/router_stage_merge.h" #include "mongo/s/query/router_stage_remove_metadata_fields.h" @@ -75,7 +79,10 @@ ClusterClientCursorImpl::ClusterClientCursorImpl(OperationContext* opCtx, _opCtx(opCtx), _createdDate(opCtx->getServiceContext()->getPreciseClockSource()->now()), _lastUseDate(_createdDate), - _queryHash(CurOp::get(opCtx)->debug().queryHash) { + _queryHash(CurOp::get(opCtx)->debug().queryHash), + _queryStatsKeyHash(CurOp::get(opCtx)->debug().queryStatsInfo.keyHash), + _queryStatsKey(std::move(CurOp::get(opCtx)->debug().queryStatsInfo.key)), + _queryStatsWillNeverExhaust(CurOp::get(opCtx)->debug().queryStatsInfo.willNeverExhaust) { dassert(!_params.compareWholeSortKeyOnRouter || SimpleBSONObjComparator::kInstance.evaluate( _params.sortToApplyOnRouter == AsyncResultsMerger::kWholeSortKeySortPattern)); @@ -92,7 +99,11 @@ ClusterClientCursorImpl::ClusterClientCursorImpl(OperationContext* opCtx, _opCtx(opCtx), _createdDate(opCtx->getServiceContext()->getPreciseClockSource()->now()), _lastUseDate(_createdDate), - _queryHash(CurOp::get(opCtx)->debug().queryHash) { + _queryHash(CurOp::get(opCtx)->debug().queryHash), + _queryStatsKeyHash(CurOp::get(opCtx)->debug().queryStatsInfo.keyHash), + _queryStatsKey(std::move(CurOp::get(opCtx)->debug().queryStatsInfo.key)), + _queryStatsWillNeverExhaust( + std::move(CurOp::get(opCtx)->debug().queryStatsInfo.willNeverExhaust)) { dassert(!_params.compareWholeSortKeyOnRouter || SimpleBSONObjComparator::kInstance.evaluate( _params.sortToApplyOnRouter == AsyncResultsMerger::kWholeSortKeySortPattern)); @@ -100,7 +111,7 @@ ClusterClientCursorImpl::ClusterClientCursorImpl(OperationContext* opCtx, } ClusterClientCursorImpl::~ClusterClientCursorImpl() { - if (_nBatchesReturned > 1) + if (_metrics.nBatches && *_metrics.nBatches > 1) mongosCursorStatsMoreThanOneBatch.increment(); } @@ -128,7 +139,25 @@ StatusWith<ClusterQueryResult> ClusterClientCursorImpl::next() { } void ClusterClientCursorImpl::kill(OperationContext* opCtx) { + if (_hasBeenKilled) { + LOGV2_DEBUG(7372700, + 3, + "Kill called on cluster client cursor after cursor has already been killed, so " + "ignoring"); + return; + } + + query_stats::writeQueryStatsOnCursorDisposeOrKill( + opCtx, + _queryStatsKeyHash, + std::move(_queryStatsKey), + _queryStatsWillNeverExhaust, + _metrics.executionTime.value_or(Microseconds{0}).count(), + _firstResponseExecutionTime.value_or(Microseconds{0}).count(), + _metrics.nreturned.value_or(0)); + _root->kill(opCtx); + _hasBeenKilled = true; } void ClusterClientCursorImpl::reattachToOperationContext(OperationContext* opCtx) { @@ -217,12 +246,12 @@ boost::optional<uint32_t> ClusterClientCursorImpl::getQueryHash() const { return _queryHash; } -std::uint64_t ClusterClientCursorImpl::getNBatches() const { - return _nBatchesReturned; +boost::optional<std::size_t> ClusterClientCursorImpl::getQueryStatsKeyHash() const { + return _queryStatsKeyHash; } -void ClusterClientCursorImpl::incNBatches() { - ++_nBatchesReturned; +bool ClusterClientCursorImpl::getQueryStatsWillNeverExhaust() const { + return _queryStatsWillNeverExhaust; } APIParameters ClusterClientCursorImpl::getAPIParameters() const { @@ -265,4 +294,7 @@ std::unique_ptr<RouterExecStage> ClusterClientCursorImpl::buildMergerPlan( return root; } +std::unique_ptr<query_stats::Key> ClusterClientCursorImpl::takeKey() { + return std::move(_queryStatsKey); +} } // namespace mongo diff --git a/src/mongo/s/query/cluster_client_cursor_impl.h b/src/mongo/s/query/cluster_client_cursor_impl.h index 2529254cfce..8064a45595b 100644 --- a/src/mongo/s/query/cluster_client_cursor_impl.h +++ b/src/mongo/s/query/cluster_client_cursor_impl.h @@ -32,6 +32,7 @@ #include <memory> #include <queue> +#include "mongo/bson/bsonobj.h" #include "mongo/executor/task_executor.h" #include "mongo/s/query/cluster_client_cursor.h" #include "mongo/s/query/cluster_client_cursor_guard.h" @@ -116,9 +117,11 @@ public: boost::optional<uint32_t> getQueryHash() const final; - std::uint64_t getNBatches() const final; + boost::optional<std::size_t> getQueryStatsKeyHash() const final; - void incNBatches() final; + bool getQueryStatsWillNeverExhaust() const final; + + std::unique_ptr<query_stats::Key> takeKey() final; public: /** @@ -175,8 +178,19 @@ private: // The hash of the query shape to be used for slow query logging; boost::optional<uint32_t> _queryHash; - // The number of batches returned by this cursor. - std::uint64_t _nBatchesReturned = 0; + // If boost::none, queryStats should not be collected for this cursor. + boost::optional<std::size_t> _queryStatsKeyHash; + + // The Key used by query stats to generate the query stats store key. + std::unique_ptr<query_stats::Key> _queryStatsKey; + + bool _queryStatsWillNeverExhaust = false; + + // Tracks if kill() has been called on the cursor. Multiple calls to kill() are treated as a + // noop. + // TODO SERVER-74482 investigate where kill() is called multiple times and remove unnecessary + // calls + bool _hasBeenKilled = false; }; } // namespace mongo diff --git a/src/mongo/s/query/cluster_client_cursor_mock.cpp b/src/mongo/s/query/cluster_client_cursor_mock.cpp index 567f3450499..103951694a0 100644 --- a/src/mongo/s/query/cluster_client_cursor_mock.cpp +++ b/src/mongo/s/query/cluster_client_cursor_mock.cpp @@ -89,14 +89,6 @@ long long ClusterClientCursorMock::getNumReturnedSoFar() const { return _numReturnedSoFar; } -std::uint64_t ClusterClientCursorMock::getNBatches() const { - return _nBatchesReturned; -} - -void ClusterClientCursorMock::incNBatches() { - ++_nBatchesReturned; -} - Date_t ClusterClientCursorMock::getCreatedDate() const { return _createdDate; } @@ -113,6 +105,14 @@ boost::optional<uint32_t> ClusterClientCursorMock::getQueryHash() const { return boost::none; } +boost::optional<std::size_t> ClusterClientCursorMock::getQueryStatsKeyHash() const { + return boost::none; +} + +bool ClusterClientCursorMock::getQueryStatsWillNeverExhaust() const { + return false; +} + void ClusterClientCursorMock::kill(OperationContext* opCtx) { _killed = true; if (_killCallback) { @@ -168,4 +168,8 @@ boost::optional<repl::ReadConcernArgs> ClusterClientCursorMock::getReadConcern() return boost::none; } +std::unique_ptr<query_stats::Key> ClusterClientCursorMock::takeKey() { + return nullptr; +} + } // namespace mongo diff --git a/src/mongo/s/query/cluster_client_cursor_mock.h b/src/mongo/s/query/cluster_client_cursor_mock.h index bc2991ecf89..64ec06d750f 100644 --- a/src/mongo/s/query/cluster_client_cursor_mock.h +++ b/src/mongo/s/query/cluster_client_cursor_mock.h @@ -33,7 +33,7 @@ #include <functional> #include <queue> -#include "mongo/db/logical_session_id.h" +#include "mongo/db/query/query_stats/key.h" #include "mongo/s/query/cluster_client_cursor.h" namespace mongo { @@ -106,9 +106,9 @@ public: boost::optional<uint32_t> getQueryHash() const final; - std::uint64_t getNBatches() const final; + boost::optional<std::size_t> getQueryStatsKeyHash() const final; - void incNBatches() final; + bool getQueryStatsWillNeverExhaust() const final; /** * Returns false unless the mock cursor has been fully iterated. @@ -120,6 +120,8 @@ public: */ void queueError(Status status); + std::unique_ptr<query_stats::Key> takeKey() final; + private: bool _killed = false; std::queue<StatusWith<ClusterQueryResult>> _resultsQueue; diff --git a/src/mongo/s/query/cluster_cursor_manager.cpp b/src/mongo/s/query/cluster_cursor_manager.cpp index 1209d709e22..4452e6a8811 100644 --- a/src/mongo/s/query/cluster_cursor_manager.cpp +++ b/src/mongo/s/query/cluster_cursor_manager.cpp @@ -41,6 +41,7 @@ #include "mongo/db/kill_sessions_common.h" #include "mongo/db/logical_session_cache.h" #include "mongo/db/query/query_knobs_gen.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/logv2/log.h" #include "mongo/util/clock_source.h" #include "mongo/util/str.h" @@ -246,6 +247,7 @@ StatusWith<ClusterCursorManager::PinnedCursor> ClusterCursorManager::checkOutCur cursorGuard->reattachToOperationContext(opCtx); CurOp::get(opCtx)->debug().queryHash = cursorGuard->getQueryHash(); + CurOp::get(opCtx)->debug().queryStatsInfo.keyHash = cursorGuard->getQueryStatsKeyHash(); return PinnedCursor(this, std::move(cursorGuard), entry->getNamespace(), cursorId); } @@ -574,4 +576,60 @@ StatusWith<ClusterClientCursorGuard> ClusterCursorManager::_detachCursor(WithLoc return std::move(cursor); } + +void collectQueryStatsMongos(OperationContext* opCtx, std::unique_ptr<query_stats::Key> key) { + // If we haven't registered a cursor to prepare for getMore requests, we record + // queryStats directly. + auto&& opDebug = CurOp::get(opCtx)->debug(); + int64_t execTime = opDebug.additiveMetrics.executionTime.value_or(Microseconds{0}).count(); + query_stats::writeQueryStats(opCtx, + opDebug.queryStatsInfo.keyHash, + std::move(key), + execTime, + execTime, + opDebug.additiveMetrics.nreturned.value_or(0)); +} + +void collectQueryStatsMongos(OperationContext* opCtx, ClusterClientCursorGuard& cursor) { + cursor->incrementCursorMetrics(CurOp::get(opCtx)->debug().additiveMetrics); + + // For a change stream query that never ends, we want to collect query stats on the initial + // query and each getMore. Here we record the initial query. + // TODO SERVER-89058 Modify comment to include tailable cursors. + if (cursor->getQueryStatsWillNeverExhaust()) { + auto& opDebug = CurOp::get(opCtx)->debug(); + + int64_t execTime = opDebug.additiveMetrics.executionTime.value_or(Microseconds{0}).count(); + + query_stats::writeQueryStats(opCtx, + opDebug.queryStatsInfo.keyHash, + cursor->takeKey(), + execTime, + execTime, + opDebug.additiveMetrics.nreturned.value_or(0), + cursor->getQueryStatsWillNeverExhaust()); + } +} + +void collectQueryStatsMongos(OperationContext* opCtx, ClusterCursorManager::PinnedCursor& cursor) { + cursor->incrementCursorMetrics(CurOp::get(opCtx)->debug().additiveMetrics); + + // For a change stream query that never ends, we want to update query stats for every getMore on + // the cursor. + // TODO SERVER-89058 Modify comment to include tailable cursors. + if (cursor->getQueryStatsWillNeverExhaust()) { + auto& opDebug = CurOp::get(opCtx)->debug(); + + int64_t execTime = opDebug.additiveMetrics.executionTime.value_or(Microseconds{0}).count(); + + query_stats::writeQueryStats(opCtx, + opDebug.queryStatsInfo.keyHash, + nullptr, + execTime, + execTime, + opDebug.additiveMetrics.nreturned.value_or(0), + cursor->getQueryStatsWillNeverExhaust()); + } +} + } // namespace mongo diff --git a/src/mongo/s/query/cluster_cursor_manager.h b/src/mongo/s/query/cluster_cursor_manager.h index be10b0d60bd..73d07d91476 100644 --- a/src/mongo/s/query/cluster_cursor_manager.h +++ b/src/mongo/s/query/cluster_cursor_manager.h @@ -599,4 +599,19 @@ private: size_t _cursorsTimedOut = 0; }; +/** + * Record metrics for the current operation on opDebug and aggregates those metrics for queryStats + * use. If a cursor is provided (via ClusterClientCursorGuard or + * ClusterCursorManager::PinnedCursor), metrics are aggregated on the cursor; otherwise, metrics are + * written directly to the queryStats store. + * NOTE: Metrics are taken from opDebug.additiveMetrics, so CurOp::setEndOfOpMetrics must be called + * *prior* to calling these. + * + * Currently, queryStats is only collected for find and aggregate requests (and their subsequent + * getMore requests), so these should only be called from those request paths. + */ +void collectQueryStatsMongos(OperationContext* opCtx, std::unique_ptr<query_stats::Key> key); +void collectQueryStatsMongos(OperationContext* opCtx, ClusterClientCursorGuard& cursor); +void collectQueryStatsMongos(OperationContext* opCtx, ClusterCursorManager::PinnedCursor& cursor); + } // namespace mongo diff --git a/src/mongo/s/query/cluster_find.cpp b/src/mongo/s/query/cluster_find.cpp index e27d4174e99..01c78f9c3c5 100644 --- a/src/mongo/s/query/cluster_find.cpp +++ b/src/mongo/s/query/cluster_find.cpp @@ -33,6 +33,7 @@ #include "mongo/s/query/cluster_find.h" +#include "mongo/db/query/query_stats/query_stats.h" #include <fmt/format.h> #include <memory> @@ -54,6 +55,7 @@ #include "mongo/db/query/find_common.h" #include "mongo/db/query/getmore_command_gen.h" #include "mongo/db/query/query_planner_common.h" +#include "mongo/db/query/query_stats/query_stats.h" #include "mongo/executor/task_executor_pool.h" #include "mongo/logv2/log.h" #include "mongo/platform/overflow_arithmetic.h" @@ -373,23 +375,26 @@ CursorId runQueryWithoutRetrying(OperationContext* opCtx, cursorState = ClusterCursorManager::CursorState::Exhausted; } + auto&& opDebug = CurOp::get(opCtx)->debug(); // Fill out query exec properties. - CurOp::get(opCtx)->debug().nShards = ccc->getNumRemotes(); - CurOp::get(opCtx)->debug().nreturned = results->size(); + opDebug.nShards = ccc->getNumRemotes(); + opDebug.additiveMetrics.nBatches = 1; // If the caller wants to know whether the cursor returned partial results, set it here. if (partialResultsReturned) { *partialResultsReturned = ccc->partialResultsReturned(); } + CurOp::get(opCtx)->setEndOfOpMetrics(results->size()); // If the cursor is exhausted, then there are no more results to return and we don't need to // allocate a cursor id. if (cursorState == ClusterCursorManager::CursorState::Exhausted) { - CurOp::get(opCtx)->debug().cursorExhausted = true; + opDebug.cursorExhausted = true; if (shardIds.size() > 0) { updateNumHostsTargetedMetrics(opCtx, cm, shardIds.size()); } + collectQueryStatsMongos(opCtx, ccc->takeKey()); return CursorId(0); } @@ -400,13 +405,13 @@ CursorId runQueryWithoutRetrying(OperationContext* opCtx, ? ClusterCursorManager::CursorLifetime::Immortal : ClusterCursorManager::CursorLifetime::Mortal; auto authUsers = AuthorizationSession::get(opCtx->getClient())->getAuthenticatedUserNames(); - ccc->incNBatches(); + collectQueryStatsMongos(opCtx, ccc); auto cursorId = uassertStatusOK(cursorManager->registerCursor( opCtx, ccc.releaseCursor(), query.nss(), cursorType, cursorLifetime, authUsers)); // Record the cursorID in CurOp. - CurOp::get(opCtx)->debug().cursorid = cursorId; + opDebug.cursorid = cursorId; if (shardIds.size() > 0) { updateNumHostsTargetedMetrics(opCtx, cm, shardIds.size()); @@ -466,6 +471,19 @@ Status setUpOperationContextStateForGetMore(OperationContext* opCtx, return Status::OK(); } +CursorId earlyExitWithNoResults(OperationContext* opCtx, + const CanonicalQuery& query, + const FindCommandRequest& findCommand) { + uassert(CollectionUUIDMismatchInfo(query.nss().db().toString(), + *findCommand.getCollectionUUID(), + query.nss().coll().toString(), + boost::none), + "Database does not exist", + !findCommand.getCollectionUUID()); + collectQueryStatsMongos(opCtx, std::move(CurOp::get(opCtx)->debug().queryStatsInfo.key)); + + return CursorId(0); +} } // namespace const size_t ClusterFind::kMaxRetries = 10; @@ -506,16 +524,9 @@ CursorId ClusterFind::runQuery(OperationContext* opCtx, for (size_t retries = 1; retries <= kMaxRetries; ++retries) { auto swCM = getCollectionRoutingInfoForTxnCmd(opCtx, query.nss()); if (swCM == ErrorCodes::NamespaceNotFound) { - uassert(CollectionUUIDMismatchInfo(query.nss().db().toString(), - *findCommand.getCollectionUUID(), - query.nss().coll().toString(), - boost::none), - "Database does not exist", - !findCommand.getCollectionUUID()); - // If the database doesn't exist, we successfully return an empty result set without // creating a cursor. - return CursorId(0); + return earlyExitWithNoResults(opCtx, query, findCommand); } const auto cm = uassertStatusOK(std::move(swCM)); @@ -842,17 +853,20 @@ StatusWith<CursorResponse> ClusterFind::runGetMore(OperationContext* opCtx, postBatchResumeToken = pinnedCursor.getValue()->getPostBatchResumeToken(); } + auto&& opDebug = CurOp::get(opCtx)->debug(); + // Set nReturned and whether the cursor has been exhausted. + opDebug.cursorExhausted = (idToReturn == 0); + opDebug.additiveMetrics.nBatches = 1; + CurOp::get(opCtx)->setEndOfOpMetrics(batch.size()); + const bool partialResultsReturned = pinnedCursor.getValue()->partialResultsReturned(); pinnedCursor.getValue()->setLeftoverMaxTimeMicros(opCtx->getRemainingMaxTimeMicros()); - pinnedCursor.getValue()->incNBatches(); + collectQueryStatsMongos(opCtx, pinnedCursor.getValue()); + // Upon successful completion, transfer ownership of the cursor back to the cursor manager. If // the cursor has been exhausted, the cursor manager will clean it up for us. pinnedCursor.getValue().returnCursor(cursorState); - // Set nReturned and whether the cursor has been exhausted. - CurOp::get(opCtx)->debug().cursorExhausted = (idToReturn == 0); - CurOp::get(opCtx)->debug().nreturned = batch.size(); - if (MONGO_unlikely(waitBeforeUnpinningOrDeletingCursorAfterGetMoreBatch.shouldFail())) { CurOpFailpointHelpers::waitWhileFailPointEnabled( &waitBeforeUnpinningOrDeletingCursorAfterGetMoreBatch, diff --git a/src/mongo/s/query/document_source_merge_cursors.cpp b/src/mongo/s/query/document_source_merge_cursors.cpp index f3af2bf0d99..02e4a1d24d5 100644 --- a/src/mongo/s/query/document_source_merge_cursors.cpp +++ b/src/mongo/s/query/document_source_merge_cursors.cpp @@ -58,10 +58,10 @@ DocumentSourceMergeCursors::DocumentSourceMergeCursors( } std::size_t DocumentSourceMergeCursors::getNumRemotes() const { - if (_armParams) { - return _armParams->getRemotes().size(); + if (_blockingResultsMerger) { + return _blockingResultsMerger->getNumRemotes(); } - return _blockingResultsMerger->getNumRemotes(); + return _armParams->getRemotes().size(); } BSONObj DocumentSourceMergeCursors::getHighWaterMark() { @@ -72,16 +72,34 @@ BSONObj DocumentSourceMergeCursors::getHighWaterMark() { } bool DocumentSourceMergeCursors::remotesExhausted() const { - if (_armParams) { + if (!_blockingResultsMerger) { // We haven't started iteration yet. return false; } return _blockingResultsMerger->remotesExhausted(); } +Status DocumentSourceMergeCursors::setAwaitDataTimeout(Milliseconds awaitDataTimeout) { + if (!_blockingResultsMerger) { + // In cases where a cursor was established with a batchSize of 0, the first getMore + // might specify a custom maxTimeMS (AKA await data timeout). In these cases we will not + // have iterated the cursor yet so will not have populated the merger, but need to + // remember/track the custom await data timeout. We will soon iterate the cursor, so we + // just populate the merger now and let it track the await data timeout itself. + populateMerger(); + } + return _blockingResultsMerger->setAwaitDataTimeout(awaitDataTimeout); +} + +void DocumentSourceMergeCursors::addNewShardCursors(std::vector<RemoteCursor>&& newCursors) { + tassert(9535000, "_blockingResultsMerger must be set", _blockingResultsMerger); + recordRemoteCursorShardIds(newCursors); + _blockingResultsMerger->addNewShardCursors(std::move(newCursors)); +} + void DocumentSourceMergeCursors::populateMerger() { - invariant(!_blockingResultsMerger); - invariant(_armParams); + tassert(9535001, "_blockingResultsMerger must not yet be set", !_blockingResultsMerger); + tassert(9535002, "_armParams must be set", _armParams); _blockingResultsMerger.emplace( pExpCtx->opCtx, @@ -97,7 +115,7 @@ void DocumentSourceMergeCursors::populateMerger() { } std::unique_ptr<RouterStageMerge> DocumentSourceMergeCursors::convertToRouterStage() { - invariant(!_blockingResultsMerger, "Expected conversion to happen before execution"); + tassert(9535003, "Expected conversion to happen before execution", !_blockingResultsMerger); return std::make_unique<RouterStageMerge>( pExpCtx->opCtx, pExpCtx->mongoProcessInterface->taskExecutor, std::move(*_armParams)); } @@ -114,11 +132,13 @@ DocumentSource::GetNextResult DocumentSourceMergeCursors::doGetNext() { return Document::fromBsonWithMetaData(*next.getResult()); } -Value DocumentSourceMergeCursors::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - invariant(!_blockingResultsMerger); - invariant(_armParams); - return Value(Document{{kStageName, _armParams->toBSON()}}); +Value DocumentSourceMergeCursors::serialize(const SerializationOptions& opts) const { + if (_blockingResultsMerger) { + return Value(Document{ + {kStageName, _blockingResultsMerger->asyncResultsMergerParams().toBSON(opts)}}); + } + tassert(9535004, "_armParams must be set", _armParams); + return Value(Document{{kStageName, _armParams->toBSON(opts)}}); } boost::intrusive_ptr<DocumentSource> DocumentSourceMergeCursors::createFromBson( @@ -150,7 +170,7 @@ void DocumentSourceMergeCursors::reattachToOperationContext(OperationContext* op void DocumentSourceMergeCursors::doDispose() { if (_blockingResultsMerger) { - invariant(!_ownCursors); + tassert(9535005, "_ownCursors must not be set", !_ownCursors); _blockingResultsMerger->kill(pExpCtx->opCtx); } else if (_ownCursors) { populateMerger(); @@ -158,7 +178,6 @@ void DocumentSourceMergeCursors::doDispose() { } } - void DocumentSourceMergeCursors::recordRemoteCursorShardIds( const std::vector<RemoteCursor>& remoteCursors) { for (const auto& remoteCursor : remoteCursors) { diff --git a/src/mongo/s/query/document_source_merge_cursors.h b/src/mongo/s/query/document_source_merge_cursors.h index 33050bf45ab..925010afa42 100644 --- a/src/mongo/s/query/document_source_merge_cursors.h +++ b/src/mongo/s/query/document_source_merge_cursors.h @@ -30,11 +30,15 @@ #pragma once #include <memory> +#include <set> +#include <variant> +#include <vector> #include "mongo/db/pipeline/document_source.h" #include "mongo/executor/task_executor.h" #include "mongo/s/query/blocking_results_merger.h" #include "mongo/s/query/router_stage_merge.h" +#include "mongo/util/duration.h" namespace mongo { @@ -80,7 +84,7 @@ public: /** * Serializes this stage to be sent to perform the merging on a different host. */ - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { StageConstraints constraints(StreamType::kStreaming, @@ -118,27 +122,13 @@ public: bool remotesExhausted() const; - Status setAwaitDataTimeout(Milliseconds awaitDataTimeout) { - if (!_blockingResultsMerger) { - // In cases where a cursor was established with a batchSize of 0, the first getMore - // might specify a custom maxTimeMS (AKA await data timeout). In these cases we will not - // have iterated the cursor yet so will not have populated the merger, but need to - // remember/track the custom await data timeout. We will soon iterate the cursor, so we - // just populate the merger now and let it track the await data timeout itself. - populateMerger(); - } - return _blockingResultsMerger->setAwaitDataTimeout(awaitDataTimeout); - } + Status setAwaitDataTimeout(Milliseconds awaitDataTimeout); /** * Adds the specified shard cursors to the set of cursors to be merged. The results from the * new cursors will be returned as normal through getNext(). */ - void addNewShardCursors(std::vector<RemoteCursor>&& newCursors) { - invariant(_blockingResultsMerger); - recordRemoteCursorShardIds(newCursors); - _blockingResultsMerger->addNewShardCursors(std::move(newCursors)); - } + void addNewShardCursors(std::vector<RemoteCursor>&& newCursors); /** * Marks the remote cursors as unowned, meaning that they won't be killed upon disposing of this @@ -170,7 +160,7 @@ private: // When we have parsed the params out of a BSONObj, the object needs to stay around while the // params are in use. We store them here. - boost::optional<BSONObj> _armParamsObj; + const boost::optional<BSONObj> _armParamsObj; // '_blockingResultsMerger' is lazily populated. Until we need to use it, '_armParams' will be // populated with the parameters. Once we start using '_blockingResultsMerger', '_armParams' @@ -180,7 +170,11 @@ private: // cursors within '_blockingResultsMerger' to be killed prematurely. For example, if this stage // is parsed on mongos then forwarded to the shards, it should not kill the cursors when it goes // out of scope on mongos. + // Note that there is a single case in which neither _armParams nor _blockingResultsMerger are + // set, and this after convertToRouterStage() is called. After that call the DocumentSource will + // remain in an unusable state. boost::optional<AsyncResultsMergerParams> _armParams; + // Can only be populated if _armParams is not set. Not populated initially. boost::optional<BlockingResultsMerger> _blockingResultsMerger; // Indicates whether the cursors stored in _armParams are "owned", meaning the cursors should be diff --git a/src/mongo/s/query/establish_cursors.cpp b/src/mongo/s/query/establish_cursors.cpp index 82ec1df2809..18196d76751 100644 --- a/src/mongo/s/query/establish_cursors.cpp +++ b/src/mongo/s/query/establish_cursors.cpp @@ -42,8 +42,11 @@ #include "mongo/db/cursor_id.h" #include "mongo/db/query/cursor_response.h" #include "mongo/db/query/kill_cursors_gen.h" +#include "mongo/db/query/query_knobs_gen.h" +#include "mongo/executor/async_multicaster.h" #include "mongo/executor/remote_command_request.h" #include "mongo/executor/remote_command_response.h" +#include "mongo/executor/task_executor.h" #include "mongo/logv2/log.h" #include "mongo/s/grid.h" #include "mongo/s/multi_statement_transaction_requests_sender.h" @@ -102,12 +105,13 @@ public: return std::exchange(_remoteCursors, {}); }; + static void killOpOnShards(ServiceContext* srvCtx, + std::shared_ptr<executor::TaskExecutor> executor, + OperationKey opKey, + std::set<HostAndPort> remotes) noexcept; + private: void _handleFailure(const AsyncRequestsSender::Response& response, Status status) noexcept; - static void _killOpOnShards(ServiceContext* srvCtx, - std::shared_ptr<executor::TaskExecutor> executor, - OperationKey opKey, - std::set<HostAndPort> remotes) noexcept; /** * Favors the status with 'CollectionUUIDMismatch' error to be saved in '_maybeFailure' to be @@ -129,22 +133,26 @@ private: std::vector<HostAndPort> _remotesToClean; }; +// Attach our OperationKey to a request. This will allow us to kill any outstanding +// requests in case we're interrupted or one of the remotes returns an error. Note that although +// the opCtx may have an OperationKey set on it already, do not inherit it here because we may +// target ourselves which implies the same node receiving multiple operations with the same +// opKey. +BSONObj appendOpKey(const OperationKey& opKey, const BSONObj& request) { + BSONObjBuilder newCmd(request); + opKey.appendToBuilder(&newCmd, "clientOperationKey"); + return newCmd.obj(); +} + void CursorEstablisher::sendRequests(const ReadPreferenceSetting& readPref, const std::vector<std::pair<ShardId, BSONObj>>& remotes, Shard::RetryPolicy retryPolicy) { // Construct the requests std::vector<AsyncRequestsSender::Request> requests; - // Attach our OperationKey to each remote request. This will allow us to kill any outstanding - // requests in case we're interrupted or one of the remotes returns an error. Note that although - // the opCtx may have an OperationKey set on it already, do not inherit it here because we may - // target ourselves which implies the same node receiving multiple operations with the same - // opKey. // TODO SERVER-47261 management of the opKey should move to the ARS. for (const auto& remote : remotes) { - BSONObjBuilder requestWithOpKey(remote.second); - _opKey.appendToBuilder(&requestWithOpKey, "clientOperationKey"); - requests.emplace_back(remote.first, requestWithOpKey.obj()); + requests.emplace_back(remote.first, appendOpKey(_opKey, remote.second)); } LOGV2_DEBUG(4625502, @@ -182,11 +190,9 @@ void CursorEstablisher::waitForResponse() noexcept { hadValidCursor = true; - RemoteCursor remoteCursor; - remoteCursor.setCursorResponse(std::move(cursor.getValue())); - remoteCursor.setShardId(response.shardId); - remoteCursor.setHostAndPort(*response.shardHostAndPort); - _remoteCursors.emplace_back(std::move(remoteCursor)); + _remoteCursors.emplace_back(RemoteCursor(response.shardId.toString(), + *response.shardHostAndPort, + std::move(cursor.getValue()))); } if (response.shardHostAndPort && !hadValidCursor) { @@ -199,16 +205,41 @@ void CursorEstablisher::waitForResponse() noexcept { } } +// Schedule killOperations against all cursors that were established. Make sure to +// capture arguments by value since the cleanup work may get scheduled after +// returning from this function. +StatusWith<executor::TaskExecutor::CallbackHandle> scheduleCursorCleanup( + std::shared_ptr<executor::TaskExecutor> executor, + ServiceContext* svcCtx, + OperationKey opKey, + std::set<HostAndPort>&& remotesToClean) { + return executor->scheduleWork([svcCtx = svcCtx, + executor = executor, + opKey = opKey, + remotesToClean = std::move(remotesToClean)]( + const executor::TaskExecutor::CallbackArgs& args) mutable { + if (!args.status.isOK()) { + LOGV2_WARNING( + 7355702, "Failed to schedule remote cursor cleanup", "error"_attr = args.status); + return; + } + CursorEstablisher::killOpOnShards( + svcCtx, std::move(executor), std::move(opKey), std::move(remotesToClean)); + }); +} + void CursorEstablisher::checkForFailedRequests() { if (!_maybeFailure) { // If we saw no failures, there is nothing to do. return; } - LOGV2(4625501, - "Unable to establish remote cursors", - "error"_attr = *_maybeFailure, - "nRemotes"_attr = _remotesToClean.size()); + if (!(_maybeFailure->code() == ErrorCodes::CommandOnShardedViewNotSupportedOnMongod)) { + LOGV2(4625501, + "Unable to establish remote cursors", + "error"_attr = *_maybeFailure, + "nRemotes"_attr = _remotesToClean.size()); + } if (_remotesToClean.empty()) { // If we don't have any remotes to clean, throw early. @@ -218,21 +249,8 @@ void CursorEstablisher::checkForFailedRequests() { // Filter out duplicate hosts. auto remotes = std::set<HostAndPort>(_remotesToClean.begin(), _remotesToClean.end()); - // Schedule killOperations against all cursors that were established. Make sure to - // capture arguments by value since the cleanup work may get scheduled after - // returning from this function. - uassertStatusOK(_executor->scheduleWork( - [svcCtx = _opCtx->getServiceContext(), - executor = _executor, - opKey = _opKey, - remotes = std::move(remotes)](const executor::TaskExecutor::CallbackArgs& args) mutable { - if (!args.status.isOK()) { - LOGV2_WARNING( - 48038, "Failed to schedule remote cursor cleanup", "error"_attr = args.status); - return; - } - _killOpOnShards(svcCtx, std::move(executor), std::move(opKey), std::move(remotes)); - })); + uassertStatusOK( + scheduleCursorCleanup(_executor, _opCtx->getServiceContext(), _opKey, std::move(remotes))); // Throw our failure. uassertStatusOK(*_maybeFailure); @@ -294,10 +312,10 @@ void CursorEstablisher::_handleFailure(const AsyncRequestsSender::Response& resp _maybeFailure = std::move(status); } -void CursorEstablisher::_killOpOnShards(ServiceContext* srvCtx, - std::shared_ptr<executor::TaskExecutor> executor, - OperationKey opKey, - std::set<HostAndPort> remotes) noexcept try { +void CursorEstablisher::killOpOnShards(ServiceContext* srvCtx, + std::shared_ptr<executor::TaskExecutor> executor, + OperationKey opKey, + std::set<HostAndPort> remotes) noexcept try { ThreadClient tc("establishCursors cleanup", srvCtx); auto opCtx = tc->makeOperationContext(); @@ -327,6 +345,16 @@ void CursorEstablisher::_killOpOnShards(ServiceContext* srvCtx, } catch (const AssertionException& ex) { LOGV2_DEBUG(4625503, 2, "Failed to cleanup remote operations", "error"_attr = ex.toStatus()); } +/** + * Returns a copy of 'cmdObj' with the $readPreference mode set to secondaryPreferred. + */ +BSONObj appendReadPreferenceNearest(BSONObj cmdObj) { + BSONObjBuilder cmdWithReadPrefBob(std::move(cmdObj)); + cmdWithReadPrefBob.append("$readPreference", + BSON("mode" + << "nearest")); + return cmdWithReadPrefBob.obj(); +} } // namespace @@ -358,4 +386,101 @@ void killRemoteCursor(OperationContext* opCtx, executor->scheduleRemoteCommand(request, [](auto const&) {}).getStatus().ignore(); } +std::pair<std::vector<HostAndPort>, StringMap<ShardId>> getHostInfos( + OperationContext* opCtx, const std::set<ShardId>& shardIds) { + std::vector<HostAndPort> servers; + StringMap<ShardId> hostToShardId; + + // Get the host/port of every node in each shard. + auto registry = Grid::get(opCtx)->shardRegistry(); + for (const auto& shardId : shardIds) { + auto shard = uassertStatusOK(registry->getShard(opCtx, shardId)); + auto cs = shard->getConnString(); + auto shardServers = cs.getServers(); + for (auto& host : shardServers) { + hostToShardId.emplace(host.toString(), shardId); + } + servers.insert(servers.end(), shardServers.begin(), shardServers.end()); + } + return {std::move(servers), hostToShardId}; +} + +std::vector<RemoteCursor> establishCursorsOnAllHosts( + OperationContext* opCtx, + std::shared_ptr<executor::TaskExecutor> executor, + const NamespaceString& nss, + const std::set<ShardId>& shardIds, + BSONObj cmdObj, + bool allowPartialResults, + Shard::RetryPolicy retryPolicy) { + auto [servers, hostToShardId] = getHostInfos(opCtx, shardIds); + OperationKey opKey = UUID::gen(); + + // Operation key will allow us to kill any outstanding requests in case we're interrupted. + // Secondaries will reject aggregation commands with a default read preference (primary). The + // actual semantics of read preference don't make as much sense when broadcasting to all + // shards, but we will set read preference to 'nearest' since it does not imply preference for + // primary or secondary. + BSONObj cmd = appendOpKey(opKey, appendReadPreferenceNearest(cmdObj)); + + executor::AsyncMulticaster::Options options; + options.maxConcurrency = internalQueryAggMulticastMaxConcurrency; + auto results = executor::AsyncMulticaster(executor, options) + .multicast(servers, + nss.db().toString(), + cmd, + opCtx, + Milliseconds(internalQueryAggMulticastTimeoutMS)); + std::vector<RemoteCursor> remoteCursors; + std::set<HostAndPort> remotesToClean; + + boost::optional<Status> failure; + + for (auto&& [hostAndPort, result] : results) { + if (result.isOK()) { + auto cursors = CursorResponse::parseFromBSONMany(result.data); + bool hadValidCursor = false; + + auto it = hostToShardId.find(hostAndPort.toString()); + tassert(7355701, "Host must have shard ID.", it != hostToShardId.end()); + auto shardId = it->second; + + for (auto& cursor : cursors) { + if (!cursor.isOK()) { + failure = cursor.getStatus(); + continue; + } + hadValidCursor = true; + + remoteCursors.emplace_back( + RemoteCursor(shardId.toString(), hostAndPort, std::move(cursor.getValue()))); + } + + if (hadValidCursor) { + remotesToClean.insert(hostAndPort); + } + } else { + LOGV2_DEBUG(7355700, + 3, + "Experienced a failure while establishing cursors", + "error"_attr = result.status); + failure = result.status; + } + } + if (failure.has_value() && !allowPartialResults) { + LOGV2(7355705, + "Unable to establish remote cursors", + "error"_attr = *failure, + "nRemotes"_attr = remoteCursors.size()); + + if (!remotesToClean.empty()) { + uassertStatusOK(scheduleCursorCleanup( + executor, opCtx->getServiceContext(), opKey, std::move(remotesToClean))); + } + + uassertStatusOK(failure.value()); + } + return remoteCursors; +} + } // namespace mongo diff --git a/src/mongo/s/query/establish_cursors.h b/src/mongo/s/query/establish_cursors.h index 3a904adcadd..cd19af7eea9 100644 --- a/src/mongo/s/query/establish_cursors.h +++ b/src/mongo/s/query/establish_cursors.h @@ -73,6 +73,26 @@ std::vector<RemoteCursor> establishCursors( Shard::RetryPolicy retryPolicy = Shard::RetryPolicy::kIdempotent); /** + * Establishes cursors on every host in the remote shards by issuing requests in parallel with the + * AsyncMulticaster. + * + * If any of the cursors fail to be established, this function performs cleanup by sending + * killCursors to any cursors that were established, then throws the error. If the namespace + * represents a view, an exception containing a ResolvedView is thrown. + * + * On success, the ownership of the cursors is transferred to the caller. This means the caller is + * now responsible for either exhausting the cursors or sending killCursors to them. + */ +std::vector<RemoteCursor> establishCursorsOnAllHosts( + OperationContext* opCtx, + std::shared_ptr<executor::TaskExecutor> executor, + const NamespaceString& nss, + const std::set<ShardId>& shardIds, + BSONObj cmdObj, + bool allowPartialResults, + Shard::RetryPolicy retryPolicy = Shard::RetryPolicy::kIdempotent); + +/** * Schedules a remote killCursor command for 'cursor'. * * Note that this method is optimistic and does not check the return status for the killCursors diff --git a/src/mongo/s/query/store_possible_cursor.cpp b/src/mongo/s/query/store_possible_cursor.cpp index c778daa9d84..723cafff2e4 100644 --- a/src/mongo/s/query/store_possible_cursor.cpp +++ b/src/mongo/s/query/store_possible_cursor.cpp @@ -88,15 +88,17 @@ StatusWith<BSONObj> storePossibleCursor(OperationContext* opCtx, return incomingCursorResponse.getStatus(); } - CurOp::get(opCtx)->debug().nreturned = incomingCursorResponse.getValue().getBatch().size(); - + auto&& opDebug = CurOp::get(opCtx)->debug(); + opDebug.additiveMetrics.nBatches = 1; // If nShards has already been set, then we are storing the forwarding $mergeCursors cursor from // a split aggregation pipeline, and the shards half of that pipeline may have targeted multiple // shards. In that case, leave the current value as-is. - CurOp::get(opCtx)->debug().nShards = std::max(CurOp::get(opCtx)->debug().nShards, 1); + opDebug.nShards = std::max(opDebug.nShards, 1); + CurOp::get(opCtx)->setEndOfOpMetrics(incomingCursorResponse.getValue().getBatch().size()); if (incomingCursorResponse.getValue().getCursorId() == CursorId(0)) { - CurOp::get(opCtx)->debug().cursorExhausted = true; + opDebug.cursorExhausted = true; + collectQueryStatsMongos(opCtx, std::move(opDebug.queryStatsInfo.key)); return cmdResult; } @@ -128,7 +130,7 @@ StatusWith<BSONObj> storePossibleCursor(OperationContext* opCtx, } auto ccc = ClusterClientCursorImpl::make(opCtx, std::move(executor), std::move(params)); - ccc->incNBatches(); + collectQueryStatsMongos(opCtx, ccc); // We don't expect to use this cursor until a subsequent getMore, so detach from the current // OperationContext until then. ccc->detachFromOperationContext(); @@ -144,7 +146,7 @@ StatusWith<BSONObj> storePossibleCursor(OperationContext* opCtx, return clusterCursorId.getStatus(); } - CurOp::get(opCtx)->debug().cursorid = clusterCursorId.getValue(); + opDebug.cursorid = clusterCursorId.getValue(); CursorResponse outgoingCursorResponse( requestedNss, diff --git a/src/mongo/s/resharding/common_types.idl b/src/mongo/s/resharding/common_types.idl index fb0547b7091..456f96bf811 100644 --- a/src/mongo/s/resharding/common_types.idl +++ b/src/mongo/s/resharding/common_types.idl @@ -271,10 +271,13 @@ structs: description: "A struct representing the information needed for a resharding pipeline to determine which documents belong to a particular shard." strict: true + query_shape_component: true fields: recipientShardId: type: shard_id description: "The id of the recipient shard." + query_shape: anonymize reshardingKey: type: KeyPattern description: "The index specification document to use as the new shard key." + query_shape: custom diff --git a/src/mongo/s/service_entry_point_mongos.cpp b/src/mongo/s/service_entry_point_mongos.cpp index 12ebf1299bd..7c1e38d008c 100644 --- a/src/mongo/s/service_entry_point_mongos.cpp +++ b/src/mongo/s/service_entry_point_mongos.cpp @@ -203,9 +203,17 @@ Future<DbResponse> HandleRequest::run() { } Future<DbResponse> ServiceEntryPointMongos::handleRequestImpl(OperationContext* opCtx, - const Message& message) noexcept { + const Message& message) try { auto hr = std::make_shared<HandleRequest>(opCtx, message); return hr->run(); +} catch (const DBException& ex) { + auto status = ex.toStatus(); + LOGV2(9431602, "Failed to handle request", "error"_attr = redact(status)); + return status; +} catch (...) { + auto error = exceptionToStatus(); + LOGV2_FATAL( + 9431601, "Request handling produced unhandled exception", "error"_attr = redact(error)); } Future<DbResponse> ServiceEntryPointMongos::handleRequest(OperationContext* opCtx, @@ -219,7 +227,7 @@ void ServiceEntryPointMongos::onClientConnect(Client* client) { } } -void ServiceEntryPointMongos::onClientDisconnect(Client* client) { +void ServiceEntryPointMongos::onClientDisconnect(Client* client) try { if (load_balancer_support::isFromLoadBalancer(client)) { _loadBalancedConnections.decrement(); @@ -259,6 +267,11 @@ void ServiceEntryPointMongos::onClientDisconnect(Client* client) { "aborting in-progress transaction because load-balanced client disconnected"}); } } +} catch (const DBException& ex) { + LOGV2_DEBUG(8969800, + 2, + "Encountered error while performing client connection cleanup", + "error"_attr = ex.toStatus()); } void ServiceEntryPointMongos::appendStats(BSONObjBuilder* bob) const { diff --git a/src/mongo/s/service_entry_point_mongos.h b/src/mongo/s/service_entry_point_mongos.h index c5c6530d2a9..ccda290e475 100644 --- a/src/mongo/s/service_entry_point_mongos.h +++ b/src/mongo/s/service_entry_point_mongos.h @@ -45,8 +45,7 @@ class ServiceEntryPointMongos final : public ServiceEntryPointImpl { public: using ServiceEntryPointImpl::ServiceEntryPointImpl; - static Future<DbResponse> handleRequestImpl(OperationContext* opCtx, - const Message& request) noexcept; + static Future<DbResponse> handleRequestImpl(OperationContext* opCtx, const Message& request); Future<DbResponse> handleRequest(OperationContext* opCtx, const Message& request) noexcept override; diff --git a/src/mongo/s/sessions_collection_sharded_test.cpp b/src/mongo/s/sessions_collection_sharded_test.cpp index d82a4edfbfa..b8cd82617e5 100644 --- a/src/mongo/s/sessions_collection_sharded_test.cpp +++ b/src/mongo/s/sessions_collection_sharded_test.cpp @@ -34,6 +34,7 @@ #include "mongo/client/remote_command_targeter_mock.h" #include "mongo/db/commands.h" #include "mongo/db/logical_session_id.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/catalog_cache_test_fixture.h" #include "mongo/s/client/shard_registry.h" diff --git a/src/mongo/s/transaction_router_test.cpp b/src/mongo/s/transaction_router_test.cpp index 74ce1210978..d72d1aea6ff 100644 --- a/src/mongo/s/transaction_router_test.cpp +++ b/src/mongo/s/transaction_router_test.cpp @@ -56,15 +56,6 @@ #include "mongo/util/net/socket_utils.h" #include "mongo/util/tick_source_mock.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString(); \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace mongo { namespace { diff --git a/src/mongo/s/write_ops/batch_write_exec.cpp b/src/mongo/s/write_ops/batch_write_exec.cpp index 45b89579dbb..980d376b607 100644 --- a/src/mongo/s/write_ops/batch_write_exec.cpp +++ b/src/mongo/s/write_ops/batch_write_exec.cpp @@ -104,6 +104,25 @@ bool hasTransientTransactionError(const BatchedCommandResponse& response) { // applies when no writes are occurring and metadata is not changing on reload. const int kMaxRoundsWithoutProgress(5); +/** + * Provides the write concern with which child batches have to be internally submitted. + */ +boost::optional<WriteConcernOptions> getWriteConcernForChildBatch(OperationContext* opCtx) { + // Per-operation write concern is not supported in transactions. + if (TransactionRouter::get(opCtx)) { + return boost::none; + } + + // Retrieve the WC specified by the remote client; in case of "fire and forget" request, the WC + // needs to be upgraded to "w: 1" for the sharding protocol to correctly handle internal + // writeErrors. + auto wc = opCtx->getWriteConcern(); + if (!wc.requiresWriteAcknowledgement()) { + wc.w = 1; + } + + return wc; +} } // namespace void BatchWriteExec::executeBatch(OperationContext* opCtx, @@ -195,6 +214,7 @@ void BatchWriteExec::executeBatch(OperationContext* opCtx, // std::vector<AsyncRequestsSender::Request> requests; + const auto wcSettingForChildBatch = getWriteConcernForChildBatch(opCtx); // Get as many batches as we can at once for (auto&& childBatch : childBatches) { @@ -217,6 +237,10 @@ void BatchWriteExec::executeBatch(OperationContext* opCtx, BSONObjBuilder requestBuilder; shardBatchRequest.serialize(&requestBuilder); logical_session_id_helpers::serializeLsidAndTxnNumber(opCtx, &requestBuilder); + if (wcSettingForChildBatch) { + requestBuilder.append(WriteConcernOptions::kWriteConcernField, + wcSettingForChildBatch->toBSON()); + } return requestBuilder.obj(); }(); diff --git a/src/mongo/s/write_ops/batch_write_exec_test.cpp b/src/mongo/s/write_ops/batch_write_exec_test.cpp index a2dc99859cc..3e90e5b65d3 100644 --- a/src/mongo/s/write_ops/batch_write_exec_test.cpp +++ b/src/mongo/s/write_ops/batch_write_exec_test.cpp @@ -397,7 +397,6 @@ TEST_F(BatchWriteExecTest, SingleOpUnordered) { insertOp.setDocuments({BSON("x" << 1)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); // Do single-target, single doc batch write op auto future = launchAsync([&] { @@ -436,7 +435,6 @@ TEST_F(BatchWriteExecTest, SingleUpdateTargetsShardWithLet) { << "100")))}); return updateOp; }()); - updateRequest.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); const static Timestamp timestamp(2); @@ -522,7 +520,6 @@ TEST_F(BatchWriteExecTest, SingleDeleteTargetsShardWithLet) { deleteOp.setDeletes(std::vector{write_ops::DeleteOpEntry(q, false)}); return deleteOp; }()); - deleteRequest.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); @@ -608,7 +605,6 @@ TEST_F(BatchWriteExecTest, MultiOpLargeOrdered) { insertOp.setDocuments(docsToInsert); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -641,7 +637,6 @@ TEST_F(BatchWriteExecTest, SingleOpUnorderedError) { insertOp.setDocuments({BSON("x" << 1)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -682,7 +677,6 @@ TEST_F(BatchWriteExecTest, MultiOpLargeUnorderedWithStaleShardVersionError) { insertOp.setDocuments(docsToInsert); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -714,7 +708,6 @@ TEST_F(BatchWriteExecTest, StaleShardVersionReturnedFromBatchWithSingleMultiWrit write_ops::UpdateModification::parseFromClassicUpdate(BSON("Key" << 100)))}); return updateOp; }()); - request.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); const static Timestamp timestamp(2); @@ -811,7 +804,6 @@ TEST_F(BatchWriteExecTest, MultiOpLargeUnorderedWithCannotRefreshError) { insertOp.setDocuments(docsToInsert); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -848,7 +840,6 @@ TEST_F(BatchWriteExecTest, write_ops::UpdateModification::parseFromClassicUpdate(BSON("y" << 2)))}); return updateOp; }()); - request.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); const static Timestamp timestamp(2); @@ -952,7 +943,6 @@ TEST_F(BatchWriteExecTest, RetryableErrorReturnedFromMultiWriteWithShard1Firs) { write_ops::UpdateModification::parseFromClassicUpdate(BSON("y" << 2)))}); return updateOp; }()); - request.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); const static Timestamp timestamp(2); @@ -1066,7 +1056,6 @@ TEST_F(BatchWriteExecTest, RetryableErrorReturnedFromMultiWriteWithShard1FirstOK write_ops::UpdateModification::parseFromClassicUpdate(BSON("y" << 2)))}); return updateOp; }()); - request.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); const static Timestamp timestamp(2); @@ -1176,7 +1165,6 @@ TEST_F(BatchWriteExecTest, RetryableErrorReturnedFromWriteWithShard1SSVShard2OK) write_ops::UpdateModification::parseFromClassicUpdate(BSON("x" << 1)))}); return updateOp; }()); - request.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); const static Timestamp timestamp(2); @@ -1276,7 +1264,6 @@ TEST_F(BatchWriteExecTest, StaleShardOp) { insertOp.setDocuments({BSON("x" << 1)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); // Execute request auto future = launchAsync([&] { @@ -1308,7 +1295,6 @@ TEST_F(BatchWriteExecTest, MultiStaleShardOp) { insertOp.setDocuments({BSON("x" << 1)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -1346,7 +1332,6 @@ TEST_F(BatchWriteExecTest, TooManyStaleShardOp) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -1381,7 +1366,6 @@ TEST_F(BatchWriteExecTest, StaleDbOp) { insertOp.setDocuments({BSON("x" << 1)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); // Execute request auto future = launchAsync([&] { @@ -1413,7 +1397,6 @@ TEST_F(BatchWriteExecTest, MultiStaleDbOp) { insertOp.setDocuments({BSON("x" << 1)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -1451,7 +1434,6 @@ TEST_F(BatchWriteExecTest, TooManyStaleDbOp) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -1486,7 +1468,6 @@ TEST_F(BatchWriteExecTest, MultiCannotRefreshShardOp) { insertOp.setDocuments({BSON("x" << 1)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -1522,7 +1503,6 @@ TEST_F(BatchWriteExecTest, TooManyCannotRefreshShardOp) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -1566,7 +1546,6 @@ TEST_F(BatchWriteExecTest, RetryableWritesLargeBatch) { insertOp.setDocuments(docsToInsert); return insertOp; }()); - request.setWriteConcern(BSONObj()); operationContext()->setLogicalSessionId(makeLogicalSessionIdForTest()); operationContext()->setTxnNumber(5); @@ -1601,7 +1580,6 @@ TEST_F(BatchWriteExecTest, RetryableErrorNoTxnNumber) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); BatchedCommandResponse retryableErrResponse; retryableErrResponse.setStatus({ErrorCodes::NotWritablePrimary, "mock retryable error"}); @@ -1640,7 +1618,6 @@ TEST_F(BatchWriteExecTest, RetryableErrorTxnNumber) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); operationContext()->setLogicalSessionId(makeLogicalSessionIdForTest()); operationContext()->setTxnNumber(5); @@ -1678,7 +1655,6 @@ TEST_F(BatchWriteExecTest, NonRetryableErrorTxnNumber) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); operationContext()->setLogicalSessionId(makeLogicalSessionIdForTest()); operationContext()->setTxnNumber(5); @@ -1720,7 +1696,6 @@ TEST_F(BatchWriteExecTest, StaleEpochIsNotRetryable) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); operationContext()->setLogicalSessionId(makeLogicalSessionIdForTest()); operationContext()->setTxnNumber(5); @@ -1748,6 +1723,249 @@ TEST_F(BatchWriteExecTest, StaleEpochIsNotRetryable) { future.default_timed_get(); } +TEST_F(BatchWriteExecTest, FireAndForgetBatchInsertGetsReplyWithOnlyOkStatus) { + const int kNumDocsToInsert = 5; + const std::string kDocValue("sample"); + + std::vector<BSONObj> docsToInsert; + docsToInsert.reserve(kNumDocsToInsert); + for (int i = 0; i < kNumDocsToInsert; i++) { + docsToInsert.push_back(BSON("_id" << i << "otherField" << kDocValue)); + } + + BatchedCommandRequest request([&] { + write_ops::InsertCommandRequest insertOp(nss); + insertOp.setWriteCommandRequestBase([] { + write_ops::WriteCommandRequestBase writeCommandBase; + writeCommandBase.setOrdered(true); + return writeCommandBase; + }()); + insertOp.setDocuments(docsToInsert); + return insertOp; + }()); + + auto future = launchAsync([&] { + BatchedCommandResponse response; + BatchWriteExecStats stats; + // Set Unacknowledged WC for a "fire & forget" request + auto opCtx = operationContext(); + opCtx->setWriteConcern( + WriteConcernOptions::parse(WriteConcernOptions::Unacknowledged).getValue()); + BatchWriteExec::executeBatch(opCtx, singleShardNSTargeter, request, &response, &stats); + + // The reply should only contain an OK status, without any further detail on + // the ops actually executed across the cluster. + BatchedCommandResponse expectedReplyToFireAndForgetRequest; + expectedReplyToFireAndForgetRequest.setStatus(Status::OK()); + ASSERT_EQUALS(response.toBSON().woCompare(expectedReplyToFireAndForgetRequest.toBSON()), 0); + }); + + expectInsertsReturnSuccess(docsToInsert.begin(), docsToInsert.end()); + + future.default_timed_get(); +} + +TEST_F(BatchWriteExecTest, FireAndForgetBatchUpdateGetsReplyWithOnlyOkStatus) { + BatchedCommandRequest request([&] { + write_ops::UpdateCommandRequest updateOp(nss); + updateOp.setWriteCommandRequestBase([] { + write_ops::WriteCommandRequestBase writeCommandBase; + writeCommandBase.setOrdered(false); + return writeCommandBase; + }()); + updateOp.setUpdates(std::vector{write_ops::UpdateOpEntry( + BSON("_id" << 100), + write_ops::UpdateModification::parseFromClassicUpdate(BSON("Key" << 100)))}); + return updateOp; + }()); + + const static auto epoch = OID::gen(); + const static Timestamp timestamp(2); + + // This allows the batch to target each write operation to perform this test + class MultiShardTargeter : public MockNSTargeter { + public: + using MockNSTargeter::MockNSTargeter; + + std::vector<ShardEndpoint> targetUpdate(OperationContext* opCtx, + const BatchItemRef& itemRef) const override { + if (targetAll) { + return std::vector{ + ShardEndpoint( + kShardName1, ChunkVersion(100, 200, epoch, timestamp), boost::none), + ShardEndpoint( + kShardName2, ChunkVersion(101, 200, epoch, timestamp), boost::none)}; + } else { + return std::vector{ShardEndpoint( + kShardName2, ChunkVersion(101, 200, epoch, timestamp), boost::none)}; + } + } + + bool targetAll = true; + }; + + MultiShardTargeter multiShardNSTargeter( + nss, + {MockRange( + ShardEndpoint(kShardName1, ChunkVersion(100, 200, epoch, timestamp), boost::none), + BSON("sk" << MINKEY), + BSON("sk" << 10)), + MockRange( + ShardEndpoint(kShardName2, ChunkVersion(101, 200, epoch, timestamp), boost::none), + BSON("sk" << 10), + BSON("sk" << MAXKEY))}); + auto future = launchAsync([&] { + // Set Unacknowledged WC for a "fire & forget" request + auto opCtx = operationContext(); + opCtx->setWriteConcern( + WriteConcernOptions::parse(WriteConcernOptions::Unacknowledged).getValue()); + + BatchedCommandResponse response; + BatchWriteExecStats stats; + BatchWriteExec::executeBatch(opCtx, multiShardNSTargeter, request, &response, &stats); + return response; + }); + + onCommandForPoolExecutor([&](const RemoteCommandRequest& request) { + ASSERT_EQ(kTestShardHost1, request.target); + + BatchedCommandResponse response; + response.setStatus(Status::OK()); + response.setNModified(1); + + return response.toBSON(); + }); + + onCommandForPoolExecutor([&](const RemoteCommandRequest& request) { + ASSERT_EQ(kTestShardHost2, request.target); + + BatchedCommandResponse response; + response.setStatus(Status::OK()); + response.setNModified(0); + response.addToErrDetails( + write_ops::WriteError(0, + Status(StaleConfigInfo(nss, + ChunkVersion(101, 200, epoch, timestamp), + ChunkVersion(105, 200, epoch, timestamp), + ShardId(kShardName2)), + "Stale error"))); + return response.toBSON(); + }); + + onCommandForPoolExecutor([&](const RemoteCommandRequest& request) { + ASSERT_EQ(kTestShardHost2, request.target); + + BatchedCommandResponse response; + response.setStatus(Status::OK()); + response.setNModified(2); + + return response.toBSON(); + }); + + // The reply should only contain an OK status, without any further detail on + // the ops actually executed across the cluster. + // (Despite of a "fire and forget" request, child batches still need to be internally processed + // before returning a reply). + auto response = future.default_timed_get(); + BatchedCommandResponse expectedReplyToFireAndForgetRequest; + expectedReplyToFireAndForgetRequest.setStatus(Status::OK()); + ASSERT_EQUALS(response.toBSON().woCompare(expectedReplyToFireAndForgetRequest.toBSON()), 0); +} + +TEST_F(BatchWriteExecTest, FireAndForgetBatchDeleteGetsReplyWithOnlyOkStatus) { + // Try to update the single doc where a let param is used in the shard key. + const auto let = BSON("y" << 100); + const auto rtc = LegacyRuntimeConstants{Date_t::now(), Timestamp(1, 1)}; + const auto q = BSON("x" + << "$$y"); + BatchedCommandRequest deleteRequest([&] { + write_ops::DeleteCommandRequest deleteOp(nss); + deleteOp.setWriteCommandRequestBase([] { + write_ops::WriteCommandRequestBase writeCommandBase; + writeCommandBase.setOrdered(false); + return writeCommandBase; + }()); + deleteOp.setLet(let); + deleteOp.setLegacyRuntimeConstants(rtc); + deleteOp.setDeletes(std::vector{write_ops::DeleteOpEntry(q, false)}); + return deleteOp; + }()); + + const static auto epoch = OID::gen(); + + class MultiShardTargeter : public MockNSTargeter { + public: + using MockNSTargeter::MockNSTargeter; + + protected: + std::vector<ShardEndpoint> targetDelete(OperationContext* opCtx, + const BatchItemRef& itemRef) const override { + return std::vector{ShardEndpoint( + kShardName2, ChunkVersion(101, 200, epoch, Timestamp(1, 1)), boost::none)}; + } + }; + + MultiShardTargeter multiShardNSTargeter( + nss, + {MockRange(ShardEndpoint( + kShardName1, ChunkVersion(100, 200, epoch, Timestamp(1, 1)), boost::none), + BSON("x" << MINKEY), + BSON("x" << 0)), + MockRange(ShardEndpoint( + kShardName2, ChunkVersion(101, 200, epoch, Timestamp(1, 1)), boost::none), + BSON("x" << 0), + BSON("x" << MAXKEY))}); + + auto future = launchAsync([&] { + BatchedCommandResponse response; + BatchWriteExecStats stats; + + // Set Unacknowledged WC for a "fire & forget" request + auto opCtx = operationContext(); + opCtx->setWriteConcern( + WriteConcernOptions::parse(WriteConcernOptions::Unacknowledged).getValue()); + + BatchWriteExec::executeBatch(opCtx, multiShardNSTargeter, deleteRequest, &response, &stats); + + return response; + }); + + // The update will hit the first shard. + onCommandForPoolExecutor( + [&](const RemoteCommandRequest& request) { + ASSERT_EQ(kTestShardHost2, request.target); + + BatchedCommandResponse response; + response.setStatus(Status::OK()); + + // Check that let params are propagated to shards. + const auto opMsgRequest(OpMsgRequest::fromDBAndBody(request.dbname, request.cmdObj)); + const auto actualBatchedUpdate(BatchedCommandRequest::parseDelete(opMsgRequest)); + ASSERT_BSONOBJ_EQ(let, actualBatchedUpdate.getLet().value_or(BSONObj())); + ASSERT_EQUALS(actualBatchedUpdate.getLegacyRuntimeConstants()->getLocalNow(), + rtc.getLocalNow()); + ASSERT_EQUALS(actualBatchedUpdate.getLegacyRuntimeConstants()->getClusterTime(), + rtc.getClusterTime()); + + // Check that let params are only forwarded and not evaluated. + auto expectedQ = BSON("x" + << "$$y"); + for (auto&& u : actualBatchedUpdate.getDeleteRequest().getDeletes()) + ASSERT_BSONOBJ_EQ(expectedQ, u.getQ()); + + return response.toBSON(); + }); + + // The reply should only contain an OK status, without any further detail on + // the ops actually executed across the cluster. + // (Despite of a "fire and forget" request, child batches still need to be internally processed + // before returning a reply). + auto response = future.default_timed_get(); + BatchedCommandResponse expectedReplyToFireAndForgetRequest; + expectedReplyToFireAndForgetRequest.setStatus(Status::OK()); + ASSERT_EQUALS(response.toBSON().woCompare(expectedReplyToFireAndForgetRequest.toBSON()), 0); +} + TEST_F(BatchWriteExecTest, TenantMigrationAbortedErrorOrderedOp) { const std::vector<BSONObj> expected{BSON("x" << 1), BSON("x" << 2), BSON("x" << 3)}; BatchedCommandRequest request([&] { @@ -1760,7 +1978,6 @@ TEST_F(BatchWriteExecTest, TenantMigrationAbortedErrorOrderedOp) { insertOp.setDocuments(expected); return insertOp; }()); - request.setWriteConcern(BSONObj()); // Execute request auto future = launchAsync([&] { @@ -1791,7 +2008,6 @@ TEST_F(BatchWriteExecTest, TenantMigrationAbortedErrorUnorderedOp) { insertOp.setDocuments(expected); return insertOp; }()); - request.setWriteConcern(BSONObj()); // Execute request auto future = launchAsync([&] { @@ -1822,7 +2038,6 @@ TEST_F(BatchWriteExecTest, MultipleTenantMigrationAbortedErrorUnorderedOp) { insertOp.setDocuments(expected); return insertOp; }()); - request.setWriteConcern(BSONObj()); const int numTenantMigrationAbortedErrors = 3; @@ -1857,7 +2072,6 @@ TEST_F(BatchWriteExecTest, MultipleTenantMigrationAbortedErrorOrderedOp) { insertOp.setDocuments(expected); return insertOp; }()); - request.setWriteConcern(BSONObj()); const int numTenantMigrationAbortedErrors = 3; @@ -1892,7 +2106,6 @@ TEST_F(BatchWriteExecTest, PartialTenantMigrationAbortedErrorOrderedOp) { insertOp.setDocuments(expected); return insertOp; }()); - request.setWriteConcern(BSONObj()); // Execute request auto future = launchAsync([&] { @@ -1925,7 +2138,6 @@ TEST_F(BatchWriteExecTest, PartialTenantMigrationErrorUnorderedOp) { insertOp.setDocuments(expected); return insertOp; }()); - request.setWriteConcern(BSONObj()); // Execute request auto future = launchAsync([&] { @@ -2010,7 +2222,6 @@ TEST_F(BatchWriteExecTargeterErrorTest, TargetedFailedAndErrorResponse) { write_ops::UpdateModification::parseFromClassicUpdate(BSON("Key" << 100)))}); return updateOp; }()); - request.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); const static Timestamp timestamp(2); @@ -2146,7 +2357,6 @@ TEST_F(BatchWriteExecTransactionTargeterErrorTest, TargetedFailedAndErrorRespons write_ops::UpdateModification::parseFromClassicUpdate(BSON("Key" << 100)))}); return updateOp; }()); - request.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); const static Timestamp timestamp(2); @@ -2290,7 +2500,6 @@ TEST_F(BatchWriteExecTransactionMultiShardTest, TargetedSucceededAndErrorRespons write_ops::UpdateModification::parseFromClassicUpdate(BSON("Key" << 100)))}); return updateOp; }()); - request.setWriteConcern(BSONObj()); const static auto epoch = OID::gen(); const static Timestamp timestamp(2); @@ -2481,7 +2690,6 @@ TEST_F(BatchWriteExecTransactionTest, ErrorInBatchThrows_CommandError) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -2513,7 +2721,6 @@ TEST_F(BatchWriteExecTransactionTest, ErrorInBatchSets_WriteError) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -2543,7 +2750,6 @@ TEST_F(BatchWriteExecTransactionTest, ErrorInBatchSets_WriteErrorOrdered) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -2573,7 +2779,6 @@ TEST_F(BatchWriteExecTransactionTest, ErrorInBatchSets_WriteErrorFromBusyCache) insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -2603,7 +2808,6 @@ TEST_F(BatchWriteExecTransactionTest, ErrorInBatchSets_WriteErrorOrderedFromBusy insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -2633,7 +2837,6 @@ TEST_F(BatchWriteExecTransactionTest, ErrorInBatchSets_TransientTxnError) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -2659,7 +2862,6 @@ TEST_F(BatchWriteExecTransactionTest, ErrorInBatchSets_DispatchError) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; @@ -2691,7 +2893,6 @@ TEST_F(BatchWriteExecTransactionTest, ErrorInBatchSets_TransientDispatchError) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSONObj()); auto future = launchAsync([&] { BatchedCommandResponse response; diff --git a/src/mongo/s/write_ops/batch_write_op.cpp b/src/mongo/s/write_ops/batch_write_op.cpp index f93e5302e4f..ed67f6c0088 100644 --- a/src/mongo/s/write_ops/batch_write_op.cpp +++ b/src/mongo/s/write_ops/batch_write_op.cpp @@ -568,6 +568,9 @@ BatchedCommandRequest BatchWriteOp::buildBatchRequest(const TargetedWriteBatch& wcb.setStmtIds(std::move(stmtIdsForOp)); } + wcb.setBypassEmptyTsReplacement( + _clientRequest.getWriteCommandRequestBase().getBypassEmptyTsReplacement()); + return wcb; }()); @@ -580,19 +583,6 @@ BatchedCommandRequest BatchWriteOp::buildBatchRequest(const TargetedWriteBatch& if (dbVersion) request.setDbVersion(*dbVersion); - if (_clientRequest.hasWriteConcern()) { - if (_clientRequest.requiresWriteAcknowledgement()) { - request.setWriteConcern(_clientRequest.getWriteConcern()); - } else { - // Mongos needs to send to the shard with w > 0 so it will be able to see the - // writeErrors - request.setWriteConcern(upgradeWriteConcern(_clientRequest.getWriteConcern())); - } - } else if (!TransactionRouter::get(_opCtx)) { - // Apply the WC from the opCtx (except if in a transaction). - request.setWriteConcern(_opCtx->getWriteConcern().toBSON()); - } - return request; } diff --git a/src/mongo/s/write_ops/batch_write_op_test.cpp b/src/mongo/s/write_ops/batch_write_op_test.cpp index 80ea3818494..b21e64c6d53 100644 --- a/src/mongo/s/write_ops/batch_write_op_test.cpp +++ b/src/mongo/s/write_ops/batch_write_op_test.cpp @@ -235,7 +235,6 @@ TEST_F(BatchWriteOpTest, SingleWriteConcernErrorOrdered) { insertOp.setDocuments({BSON("x" << 1)}); return insertOp; }()); - request.setWriteConcern(BSON("w" << 3)); BatchWriteOp batchOp(_opCtx, request); @@ -247,7 +246,6 @@ TEST_F(BatchWriteOpTest, SingleWriteConcernErrorOrdered) { BatchedCommandRequest targetBatch = batchOp.buildBatchRequest(*targeted.begin()->second, targeter); - ASSERT(targetBatch.getWriteConcern().woCompare(request.getWriteConcern()) == 0); BatchedCommandResponse response; buildResponse(1, &response); @@ -1029,7 +1027,6 @@ TEST_F(BatchWriteOpTest, MultiOpErrorAndWriteConcernErrorUnordered) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 1)}); return insertOp; }()); - request.setWriteConcern(BSON("w" << 3)); BatchWriteOp batchOp(_opCtx, request); @@ -1072,7 +1069,6 @@ TEST_F(BatchWriteOpTest, SingleOpErrorAndWriteConcernErrorOrdered) { updateOp.setUpdates({buildUpdate(BSON("x" << GTE << -1 << LT << 2), true)}); return updateOp; }()); - request.setWriteConcern(BSON("w" << 3)); BatchWriteOp batchOp(_opCtx, request); @@ -1415,7 +1411,6 @@ TEST_F(BatchWriteOpTest, MultiOpTwoWCErrors) { insertOp.setDocuments({BSON("x" << -1), BSON("x" << 2)}); return insertOp; }()); - request.setWriteConcern(BSON("w" << 3)); BatchWriteOp batchOp(_opCtx, request); diff --git a/src/mongo/s/write_ops/batched_command_request.cpp b/src/mongo/s/write_ops/batched_command_request.cpp index e6b3c970592..185857d6acc 100644 --- a/src/mongo/s/write_ops/batched_command_request.cpp +++ b/src/mongo/s/write_ops/batched_command_request.cpp @@ -38,8 +38,6 @@ namespace mongo { namespace { -const auto kWriteConcern = "writeConcern"_sd; - template <class T> BatchedCommandRequest constructBatchedCommandRequest(const OpMsgRequest& request) { auto batchRequest = BatchedCommandRequest{T::parse(request)}; @@ -53,16 +51,6 @@ BatchedCommandRequest constructBatchedCommandRequest(const OpMsgRequest& request batchRequest.setShardVersion(shardVersion); } - auto writeConcernField = request.body[kWriteConcern]; - if (!writeConcernField.eoo()) { - auto wcObj = writeConcernField.Obj(); - // Client write concerns without 'w' fields should be filled with the default write concern, - // which should be populated later to the operation context during the command setup phase. - if (wcObj.hasElement("w")) { - batchRequest.setWriteConcern(wcObj); - } - } - // The 'isTimeseriesNamespace' is an internal parameter used for communication between mongos // and mongod. auto isTimeseriesNamespace = @@ -178,19 +166,9 @@ const boost::optional<BSONObj>& BatchedCommandRequest::getLet() const { return _visit(Visitor{}); }; -bool BatchedCommandRequest::requiresWriteAcknowledgement() const { - if (!hasWriteConcern()) { - return true; - } - - BSONObj writeConcern = getWriteConcern(); - BSONElement wElem = writeConcern["w"]; - if (!wElem.isNumber() || wElem.Number() != 0) { - return true; - } - - return false; -} +const OptionalBool& BatchedCommandRequest::getBypassEmptyTsReplacement() const { + return _visit([](auto&& op) -> decltype(auto) { return op.getBypassEmptyTsReplacement(); }); +}; const write_ops::WriteCommandRequestBase& BatchedCommandRequest::getWriteCommandRequestBase() const { @@ -211,10 +189,6 @@ void BatchedCommandRequest::serialize(BSONObjBuilder* builder) const { if (_dbVersion) { builder->append("databaseVersion", _dbVersion->toBSON()); } - - if (_writeConcern) { - builder->append(kWriteConcern, *_writeConcern); - } } BSONObj BatchedCommandRequest::toBSON() const { diff --git a/src/mongo/s/write_ops/batched_command_request.h b/src/mongo/s/write_ops/batched_command_request.h index 1834fe4286b..0bcb51a3556 100644 --- a/src/mongo/s/write_ops/batched_command_request.h +++ b/src/mongo/s/write_ops/batched_command_request.h @@ -115,25 +115,6 @@ public: std::size_t sizeWriteOps() const; - void setWriteConcern(const BSONObj& writeConcern) { - _writeConcern = writeConcern.getOwned(); - } - - void unsetWriteConcern() { - _writeConcern = boost::none; - } - - bool hasWriteConcern() const { - return _writeConcern.is_initialized(); - } - - const BSONObj& getWriteConcern() const { - invariant(_writeConcern); - return *_writeConcern; - } - - bool requiresWriteAcknowledgement() const; - void setShardVersion(ChunkVersion shardVersion) { _shardVersion = std::move(shardVersion); } @@ -168,6 +149,7 @@ public: const boost::optional<LegacyRuntimeConstants>& getLegacyRuntimeConstants() const; const boost::optional<BSONObj>& getLet() const; + const OptionalBool& getBypassEmptyTsReplacement() const; const write_ops::WriteCommandRequestBase& getWriteCommandRequestBase() const; void setWriteCommandRequestBase(write_ops::WriteCommandRequestBase writeCommandBase); @@ -250,8 +232,6 @@ private: boost::optional<ChunkVersion> _shardVersion; boost::optional<DatabaseVersion> _dbVersion; - - boost::optional<BSONObj> _writeConcern; }; /** diff --git a/src/mongo/s/write_ops/batched_command_request_test.cpp b/src/mongo/s/write_ops/batched_command_request_test.cpp index 9a5e968f10d..be0728b1533 100644 --- a/src/mongo/s/write_ops/batched_command_request_test.cpp +++ b/src/mongo/s/write_ops/batched_command_request_test.cpp @@ -92,14 +92,12 @@ TEST(BatchedCommandRequest, InsertCloneWithIds) { insertOp.setDocuments({BSON("x" << 1), BSON("x" << 2)}); return insertOp; }()); - batchedRequest.setWriteConcern(BSON("w" << 2)); const auto clonedRequest(BatchedCommandRequest::cloneInsertWithIds(std::move(batchedRequest))); ASSERT_EQ("xyz.abc", clonedRequest.getNS().ns()); ASSERT(clonedRequest.getWriteCommandRequestBase().getOrdered()); ASSERT(clonedRequest.getWriteCommandRequestBase().getBypassDocumentValidation()); - ASSERT_BSONOBJ_EQ(BSON("w" << 2), clonedRequest.getWriteConcern()); const auto& insertDocs = clonedRequest.getInsertRequest().getDocuments(); ASSERT_EQ(2u, insertDocs.size()); diff --git a/src/mongo/s/write_ops/write_op.cpp b/src/mongo/s/write_ops/write_op.cpp index 236c7efbc94..6b33d3b9312 100644 --- a/src/mongo/s/write_ops/write_op.cpp +++ b/src/mongo/s/write_ops/write_op.cpp @@ -29,7 +29,24 @@ #include "mongo/s/write_ops/write_op.h" + +#include <absl/container/flat_hash_set.h> +#include <algorithm> +#include <boost/move/utility_core.hpp> +#include <boost/none.hpp> +#include <boost/optional/optional.hpp> +#include <ostream> +#include <string> + +#include "mongo/base/error_codes.h" +#include "mongo/base/status.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/catalog/collection_uuid_mismatch_info.h" +#include "mongo/db/stats/counters.h" +#include "mongo/s/sharding_feature_flags_gen.h" #include "mongo/s/transaction_router.h" +#include "mongo/s/write_ops/batch_write_op.h" +#include "mongo/s/write_ops/batched_command_request.h" #include "mongo/util/assert_util.h" namespace mongo { @@ -79,6 +96,16 @@ write_ops::WriteError combineOpErrors(const std::vector<ChildWriteOp const*>& er Status(MultipleErrorsOccurredInfo(errB.arr()), msg.str())); } +bool isSafeToIgnoreErrorInPartiallyAppliedOp(write_ops::WriteError& error) { + // UUID mismatch errors are safe to ignore if the actualCollection is null in conjuntion with + // other successful operations. This is true because it means we wrongly targeted a non-owning + // shard with the operation and we wouldn't have applied any modifications anyway. + // + // Note this is only safe if we're using ShardVersion::IGNORED since we're ignoring any + // placement concern and broadcasting to all shards. + return error.getStatus().code() == ErrorCodes::CollectionUUIDMismatch && + !error.getStatus().extraInfo<CollectionUUIDMismatchInfo>()->actualCollection(); +} } // namespace const BatchItemRef& WriteOp::getWriteItem() const { @@ -182,7 +209,29 @@ void WriteOp::_updateOpState() { _state = WriteOpState_Ready; } else if (!childErrors.empty()) { _error = combineOpErrors(childErrors); - _state = WriteOpState_Error; + bool isTargetingAllShardsWithSVIgnored = + childErrors.front() + ->endpoint->shardVersion + .map([&](const auto& cv) { return ChunkVersion::isIgnoredVersion(cv); }) + .get_value_or(false); + // There are errors that are safe to ignore if they were correctly applied to other shards + // and we're using ShardVersion::IGNORED. They are safe to ignore as they can be interpreted + // as no-ops if the shard response had been instead a successful result since they wouldn't + // have modified any data. As a result, we can swallow the errors and treat them as a + // successful operation. + if (isTargetingAllShardsWithSVIgnored && isSafeToIgnoreErrorInPartiallyAppliedOp(*_error) && + !_successfulShardSet.empty()) { + if (!hasPendingChild) { + _error.reset(); + _state = WriteOpState_Completed; + } else { + // As this error is acceptable we wait until all other operations finish to take a + // decision. + return; + } + } else { + _state = WriteOpState_Error; + } } else if (hasPendingChild && _inTxn) { // Return early here since this means that there were no errors while in txn // but there are still ops that have not yet finished. diff --git a/src/mongo/shell/SConscript b/src/mongo/shell/SConscript index d8f54958a73..3b742237bc6 100644 --- a/src/mongo/shell/SConscript +++ b/src/mongo/shell/SConscript @@ -203,6 +203,7 @@ if get_option('ssl') == 'on': "fle_shell_options.idl", ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/client/clientdriver_minimal', '$BUILD_DIR/mongo/crypto/aead_encryption', '$BUILD_DIR/mongo/crypto/encrypted_field_config', diff --git a/src/mongo/shell/check_log.js b/src/mongo/shell/check_log.js index 1a7c4312c0b..e31dd863519 100644 --- a/src/mongo/shell/check_log.js +++ b/src/mongo/shell/check_log.js @@ -30,6 +30,32 @@ checkLog = (function() { * is found in the logs. Note: this function does not throw an exception, so the return * value should not be ignored. */ + const getLogMessage = function(conn, msg) { + const logMessages = getGlobalLog(conn); + if (logMessages === null) { + return null; + } + if (msg instanceof RegExp) { + for (let logMsg of logMessages) { + if (logMsg.search(msg) != -1) { + return logMsg; + } + } + } else { + for (let logMsg of logMessages) { + if (logMsg.includes(msg)) { + return logMsg; + } + } + } + return null; + }; + + /* + * Calls the 'getLog' function on the provided connection 'conn' to see if the provided msg + * is found in the logs. Note: this function does not throw an exception, so the return + * value should not be ignored. + */ const checkContainsOnce = function(conn, msg) { const logMessages = getGlobalLog(conn); if (logMessages === null) { @@ -184,6 +210,29 @@ checkLog = (function() { {runHangAnalyzer: false}); }; + /* + * Calls the 'getLog' function at regular intervals on the provided connection 'conn' until + * the provided 'msg' is found in the logs and returned, or it times out. Throws an exception on + * timeout. + */ + let containsLog = function(conn, msg, timeoutMillis = 5 * 60 * 1000, retryIntervalMS = 300) { + // Don't run the hang analyzer because we don't expect contains() to always succeed. + let logMsg = null; + assert.soon( + function() { + logMsg = getLogMessage(conn, msg); + if (logMsg) { + return true; + } + return false; + }, + 'Could not find log entries containing the following message: ' + msg, + timeoutMillis, + retryIntervalMS, + {runHangAnalyzer: false}); + return logMsg; + }; + let containsJson = function(conn, id, attrsDict, timeoutMillis = 5 * 60 * 1000) { // Don't run the hang analyzer because we don't expect contains() to always succeed. assert.soon( @@ -423,12 +472,14 @@ checkLog = (function() { return { getGlobalLog: getGlobalLog, + getLogMessage: getLogMessage, checkContainsOnce: checkContainsOnce, checkContainsOnceJson: checkContainsOnceJson, checkContainsWithCountJson: checkContainsWithCountJson, checkContainsWithAtLeastCountJson: checkContainsWithAtLeastCountJson, checkContainsOnceJsonStringMatch: checkContainsOnceJsonStringMatch, contains: contains, + containsLog: containsLog, containsJson: containsJson, containsRelaxedJson: containsRelaxedJson, containsWithCount: containsWithCount, diff --git a/src/mongo/shell/data_consistency_checker.js b/src/mongo/shell/data_consistency_checker.js index 3a1621b11de..b3303c76857 100644 --- a/src/mongo/shell/data_consistency_checker.js +++ b/src/mongo/shell/data_consistency_checker.js @@ -370,6 +370,28 @@ var {DataConsistencyChecker} = (function() { delete syncingInfo.idIndex.ns; } + // If the servers are using encryption and they specify an encryption option + // in versions <7.2 this is stored on the primary but not the secondary. + // This is not an actual failure since the data is correct on all nodes. We + // can safely ignore this element in the configString. + const encryptionRegex = /encryption=\(?[^)]*\),?/; + + if (sourceInfo.options && sourceInfo.options.storageEngine && + sourceInfo.options.storageEngine.wiredTiger && + sourceInfo.options.storageEngine.wiredTiger.configString) { + sourceInfo.options.storageEngine.wiredTiger.configString = + sourceInfo.options.storageEngine.wiredTiger.configString.replace( + encryptionRegex, ""); + } + + if (syncingInfo.options && syncingInfo.options.storageEngine && + syncingInfo.options.storageEngine.wiredTiger && + syncingInfo.options.storageEngine.wiredTiger.configString) { + syncingInfo.options.storageEngine.wiredTiger.configString = + syncingInfo.options.storageEngine.wiredTiger.configString.replace( + encryptionRegex, ""); + } + if (!bsonBinaryEqual(syncingInfo, sourceInfo)) { prettyPrint( `the two nodes have different attributes for the collection or view ${ diff --git a/src/mongo/shell/encrypted_dbclient_base.cpp b/src/mongo/shell/encrypted_dbclient_base.cpp index b9151685a93..391ef009fd3 100644 --- a/src/mongo/shell/encrypted_dbclient_base.cpp +++ b/src/mongo/shell/encrypted_dbclient_base.cpp @@ -301,7 +301,7 @@ BSONObj EncryptedDBClientBase::validateBSONElement(ConstDataRange out, uint8_t b builder.appendNum(static_cast<uint32_t>(docLength)); builder.appendChar(static_cast<uint8_t>(bsonType)); - builder.appendStr(valueString, true); + builder.appendCStr(valueString); builder.appendBuf(out.data(), out.length()); builder.appendChar('\0'); @@ -471,7 +471,7 @@ void EncryptedDBClientBase::encrypt(mozjs::MozJSImplScope* scope, } plaintextBuilder.appendNum(static_cast<uint32_t>(valueStr.size() + 1)); - plaintextBuilder.appendStr(valueStr, true); + plaintextBuilder.appendStrBytesAndNul(valueStr); bsonType = BSONType::String; } else if (args.get(1).isNumber()) { diff --git a/src/mongo/shell/query.js b/src/mongo/shell/query.js index a5d920cb109..b43e13c23e8 100644 --- a/src/mongo/shell/query.js +++ b/src/mongo/shell/query.js @@ -98,6 +98,10 @@ DBQuery.prototype._canUseCommandCursor = function() { (this._options & DBQuery.Option.exhaust) === 0; }; +DBQuery.prototype._isTailableCursor = function() { + return (this._options & DBQuery.Option.tailable) !== 0; +}; + /** * This method is exposed only for the purpose of testing and should not be used in most contexts. * @@ -301,7 +305,9 @@ DBQuery.prototype.skip = function(skip) { DBQuery.prototype.hasNext = function() { this._exec(); - if (this._limit > 0 && this._cursorSeen >= this._limit) { + // Return when limit is reached for tailable cursors. For other cursor options, like an exhaust + // cursor, the server manages closing. + if (this._isTailableCursor() && this._limit > 0 && this._cursorSeen >= this._limit) { this._cursor.close(); return false; } diff --git a/src/mongo/shell/servers.js b/src/mongo/shell/servers.js index 1040b3c991d..ece24b66a6b 100644 --- a/src/mongo/shell/servers.js +++ b/src/mongo/shell/servers.js @@ -25,17 +25,6 @@ var _parsePath = function() { return dbpath; }; -var _parsePort = function() { - var port = ""; - for (var i = 0; i < arguments.length; ++i) - if (arguments[i] == "--port") - port = arguments[i + 1]; - - if (port == "") - throw Error("No port specified"); - return port; -}; - var createMongoArgs = function(binaryName, args) { if (!Array.isArray(args)) { throw new Error("The second argument to createMongoArgs must be an array"); @@ -82,6 +71,17 @@ MongoRunner.mongosPath = "mongos"; MongoRunner.mongoqPath = "mongoqd"; MongoRunner.mongoShellPath = "mongo"; +MongoRunner.parsePort = function() { + var port = ""; + for (var i = 0; i < arguments.length; ++i) + if (arguments[i] == "--port") + port = arguments[i + 1]; + + if (port == "") + throw Error("No port specified"); + return port; +}; + MongoRunner.VersionSub = function(pattern, version) { this.pattern = pattern; this.version = version; @@ -1640,7 +1640,7 @@ MongoRunner._startWithArgs = function(argArray, env, waitForConnect) { // TODO: Make there only be one codepath for starting mongo processes argArray = appendSetParameterArgs(argArray); - var port = _parsePort.apply(null, argArray); + var port = MongoRunner.parsePort.apply(null, argArray); var pid = -1; if (env === undefined) { pid = _startMongoProgram.apply(null, argArray); @@ -1675,7 +1675,7 @@ MongoRunner._startWithArgs = function(argArray, env, waitForConnect) { * command line arguments to the program. */ startMongoProgram = function() { - var port = _parsePort.apply(null, arguments); + var port = MongoRunner.parsePort.apply(null, arguments); // Enable test commands. // TODO: Make this work better with multi-version testing so that we can support diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js index 9136fa49333..1319746ea1a 100644 --- a/src/mongo/shell/shardingtest.js +++ b/src/mongo/shell/shardingtest.js @@ -1034,8 +1034,7 @@ var ShardingTest = function(params) { assert(isObject(params), 'ShardingTest configuration must be a JSON object'); var testName = params.name || jsTest.name(); - var otherParams = Object.merge(params, params.other || {}); - + var otherParams = Object.deepMerge(params, params.other || {}); var numShards = otherParams.hasOwnProperty('shards') ? otherParams.shards : 2; var mongosVerboseLevel = otherParams.hasOwnProperty('verbose') ? otherParams.verbose : 1; var numMongos = otherParams.hasOwnProperty('mongos') ? otherParams.mongos : 1; diff --git a/src/mongo/shell/types.js b/src/mongo/shell/types.js index c06601f9161..ce8fd02b172 100644 --- a/src/mongo/shell/types.js +++ b/src/mongo/shell/types.js @@ -292,6 +292,36 @@ Object.merge = function(dst, src, deep) { return Object.extend(clone, src, deep); }; +// If there is a conflict in values of a key for two objects being merged, the second value will +// override the first one in the merged object +Object.deepMerge = function(...objects) { + const isObject = obj => obj && typeof obj === 'object'; + + // Create new object prev to hold combination of all object fields. + return objects.reduce((prev, obj) => { + if (obj === undefined) { + obj = {}; + } + Object.keys(obj).forEach(key => { + const pVal = prev[key]; // Get the values for key from the two objects being merged. + const oVal = obj[key]; + + if (Array.isArray(pVal) && + Array.isArray(oVal)) { // If both are arrays then concatenate them into a new + // array and add it to prev. + prev[key] = pVal.concat(...oVal); + } else if (isObject(pVal) && + isObject(oVal)) { // If both are objects then recursively merge again. + prev[key] = Object.deepMerge(pVal, oVal); + } else { // In all other cases set prev[key] to obj[key]. + prev[key] = oVal; + } + }); + + return prev; + }, {}); +}; + Object.keySet = function(o) { var ret = new Array(); for (var i in o) { diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js index da2c29fa493..e1e09a83a7b 100644 --- a/src/mongo/shell/utils.js +++ b/src/mongo/shell/utils.js @@ -29,6 +29,13 @@ function _getErrorWithCode(codeOrObj, message) { if (codeOrObj.hasOwnProperty("writeErrors")) { e.writeErrors = codeOrObj.writeErrors; + } else if ((codeOrObj instanceof BulkWriteResult || codeOrObj instanceof BulkWriteError) && + codeOrObj.hasWriteErrors()) { + e.writeErrors = codeOrObj.getWriteErrors(); + } + + if (codeOrObj instanceof WriteResult && codeOrObj.hasWriteError()) { + e.writeErrors = [codeOrObj.getWriteError()]; } if (codeOrObj.hasOwnProperty("errorLabels")) { @@ -85,7 +92,11 @@ function isNetworkError(errorOrResponse) { "error doing query", "socket exception", "SocketException", - "HostNotFound" + "HostNotFound", + "HostUnreachable", + "NetworkTimeout", + "ConnectionPoolExpired", + "ConnectionError" ]; // Then check if it's an Error, if so see if any of the known network error strings appear diff --git a/src/mongo/transport/transport_layer_asio.cpp b/src/mongo/transport/transport_layer_asio.cpp index ca35c4a5337..5e9f8595278 100644 --- a/src/mongo/transport/transport_layer_asio.cpp +++ b/src/mongo/transport/transport_layer_asio.cpp @@ -1476,6 +1476,10 @@ TransportLayerASIO::_createSSLContext(std::shared_ptr<SSLManagerInterface>& mana newSSLContext->ingress->native_handle(), asyncOCSPStaple); if (!resp.isOK()) { + // The stapleOCSPResponse call above may have started a periodic OCSP fetch job + // on a separate thread which keeps a copy of the manager shared pointer. + // This stops that thread so that the transient manager can be destructed. + newSSLContext->manager->stopJobs(); return Status(ErrorCodes::InvalidSSLConfiguration, str::stream() << "Can not staple OCSP Response. Reason: " << resp.reason()); diff --git a/src/mongo/unittest/bson_test_util.cpp b/src/mongo/unittest/bson_test_util.cpp index 11f504a8f99..542c902bef1 100644 --- a/src/mongo/unittest/bson_test_util.cpp +++ b/src/mongo/unittest/bson_test_util.cpp @@ -29,6 +29,7 @@ #include "mongo/platform/basic.h" +#include "mongo/bson/json.h" #include "mongo/unittest/bson_test_util.h" namespace mongo { diff --git a/src/mongo/unittest/bson_test_util.h b/src/mongo/unittest/bson_test_util.h index 2e7c928c339..ff4b58b5f7b 100644 --- a/src/mongo/unittest/bson_test_util.h +++ b/src/mongo/unittest/bson_test_util.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/bson/json.h" #include "mongo/bson/simple_bsonelement_comparator.h" #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/unittest/unittest.h" @@ -90,5 +91,9 @@ DECLARE_BSON_CMP_FUNC(BSONElement, GTE); DECLARE_BSON_CMP_FUNC(BSONElement, NE); #undef DECLARE_BSON_CMP_FUNC +// TODO SERVER-87736 this macro is misleading on the v6 .0 branch.We will keep it as "AUTO" for now, +// but there is no 'auto' support on this branch. +#define ASSERT_BSONOBJ_EQ_AUTO(expected, actual) ASSERT_BSONOBJ_EQ(fromjson(expected), actual) + } // namespace unittest } // namespace mongo diff --git a/src/mongo/unittest/unittest.cpp b/src/mongo/unittest/unittest.cpp index 94adc7f1046..ef9233554c2 100644 --- a/src/mongo/unittest/unittest.cpp +++ b/src/mongo/unittest/unittest.cpp @@ -193,27 +193,39 @@ public: void startCapturingLogMessages(); void stopCapturingLogMessages(); void stopCapturingLogMessagesIfNeeded(); - const std::vector<std::string>& getCapturedTextFormatLogMessages() const; + const synchronized_value<std::vector<std::string>>& getCapturedTextFormatLogMessages() const { + return _capturedLogMessages; + } std::vector<BSONObj> getCapturedBSONFormatLogMessages() const; int64_t countTextFormatLogLinesContaining(const std::string& needle); int64_t countBSONFormatLogLinesIsSubset(const BSONObj& needle); void printCapturedTextFormatLogLines() const; private: + class Listener : public logv2::LogLineListener { + public: + explicit Listener(synchronized_value<std::vector<std::string>>* sv) : _sv(sv) {} + void accept(const std::string& line) override { + (***_sv).push_back(line); + } + + private: + synchronized_value<std::vector<std::string>>* _sv; + }; + bool _isCapturingLogMessages{false}; // Captures Plain Text Log - std::vector<std::string> _capturedLogMessages; + synchronized_value<std::vector<std::string>> _capturedLogMessages; // Captured BSON - std::vector<std::string> _capturedBSONLogMessages; + synchronized_value<std::vector<std::string>> _capturedBSONLogMessages; // Capture Sink for Plain Text - boost::shared_ptr<boost::log::sinks::synchronous_sink<logv2::LogCaptureBackend>> _captureSink; + boost::shared_ptr<boost::log::sinks::unlocked_sink<logv2::LogCaptureBackend>> _captureSink; // Capture Sink for BSON - boost::shared_ptr<boost::log::sinks::synchronous_sink<logv2::LogCaptureBackend>> - _captureBSONSink; + boost::shared_ptr<boost::log::sinks::unlocked_sink<logv2::LogCaptureBackend>> _captureBSONSink; }; static CaptureLogs* getCaptureLogs() { @@ -248,16 +260,18 @@ namespace { void CaptureLogs::startCapturingLogMessages() { invariant(!_isCapturingLogMessages); - _capturedLogMessages.clear(); - _capturedBSONLogMessages.clear(); + (**_capturedLogMessages).clear(); + (**_capturedBSONLogMessages).clear(); if (!_captureSink) { - _captureSink = logv2::LogCaptureBackend::create(_capturedLogMessages, true); + _captureSink = logv2::LogCaptureBackend::create( + std::make_unique<Listener>(&_capturedLogMessages), true); _captureSink->set_filter( logv2::AllLogsFilter(logv2::LogManager::global().getGlobalDomain())); _captureSink->set_formatter(logv2::PlainFormatter()); - _captureBSONSink = logv2::LogCaptureBackend::create(_capturedBSONLogMessages, false); + _captureBSONSink = logv2::LogCaptureBackend::create( + std::make_unique<Listener>(&_capturedBSONLogMessages), false); _captureBSONSink->set_filter( logv2::AllLogsFilter(logv2::LogManager::global().getGlobalDomain())); @@ -283,14 +297,11 @@ void CaptureLogs::stopCapturingLogMessagesIfNeeded() { } } -const std::vector<std::string>& CaptureLogs::getCapturedTextFormatLogMessages() const { - return _capturedLogMessages; -} - std::vector<BSONObj> CaptureLogs::getCapturedBSONFormatLogMessages() const { std::vector<BSONObj> objs; - std::transform(_capturedBSONLogMessages.cbegin(), - _capturedBSONLogMessages.cend(), + auto logLinesLockGuard = *_capturedBSONLogMessages; + std::transform(logLinesLockGuard->cbegin(), + logLinesLockGuard->cend(), std::back_inserter(objs), [](const std::string& str) { return BSONObj(str.c_str()); }); return objs; @@ -298,7 +309,8 @@ std::vector<BSONObj> CaptureLogs::getCapturedBSONFormatLogMessages() const { void CaptureLogs::printCapturedTextFormatLogLines() const { LOGV2(23054, "****************************** Captured Lines (start) *****************************"); - for (const auto& line : getCapturedTextFormatLogMessages()) { + auto logLinesLockGuard = *getCapturedTextFormatLogMessages(); + for (const auto& line : *logLinesLockGuard) { LOGV2(23055, "{line}", "line"_attr = line); } LOGV2(23056, @@ -306,9 +318,10 @@ void CaptureLogs::printCapturedTextFormatLogLines() const { } int64_t CaptureLogs::countTextFormatLogLinesContaining(const std::string& needle) { - const auto& msgs = getCapturedTextFormatLogMessages(); - return std::count_if( - msgs.begin(), msgs.end(), [&](const std::string& s) { return stringContains(s, needle); }); + auto msgs = *getCapturedTextFormatLogMessages(); + return std::count_if(msgs->begin(), msgs->end(), [&](const std::string& s) { + return stringContains(s, needle); + }); } bool isSubset(BSONObj haystack, BSONObj needle) { @@ -360,8 +373,8 @@ void Test::startCapturingLogMessages() { void Test::stopCapturingLogMessages() { getCaptureLogs()->stopCapturingLogMessages(); } -const std::vector<std::string>& Test::getCapturedTextFormatLogMessages() const { - return getCaptureLogs()->getCapturedTextFormatLogMessages(); +std::vector<std::string> Test::getCapturedTextFormatLogMessages() const { + return getCaptureLogs()->getCapturedTextFormatLogMessages().get(); } std::vector<BSONObj> Test::getCapturedBSONFormatLogMessages() const { return getCaptureLogs()->getCapturedBSONFormatLogMessages(); diff --git a/src/mongo/unittest/unittest.h b/src/mongo/unittest/unittest.h index 514a8e7d8d9..8e82eb5e06b 100644 --- a/src/mongo/unittest/unittest.h +++ b/src/mongo/unittest/unittest.h @@ -53,6 +53,7 @@ #include "mongo/unittest/bson_test_util.h" #include "mongo/util/assert_util.h" #include "mongo/util/str.h" +#include "mongo/util/synchronized_value.h" /** * Fail unconditionally, reporting the given message. @@ -151,6 +152,21 @@ ASSERT_THROWS_WITH_CHECK(EXPRESSION, EXCEPTION_TYPE, ([](const EXCEPTION_TYPE&) {})) /** + * Verify that the evaluation of "EXPRESSION" does not throw any exceptions. + * + * If "EXPRESSION" throws an exception the test is considered a failure and further evaluation + * halts. + */ +#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ + try { \ + EXPRESSION; \ + } catch (const AssertionException& e) { \ + str::stream err; \ + err << "Threw an exception incorrectly: " << e.toString(); \ + FAIL(err); \ + } + +/** * Behaves like ASSERT_THROWS, above, but also fails if calling what() on the thrown exception * does not return a string equal to EXPECTED_WHAT. */ @@ -649,7 +665,7 @@ protected: * Gets a vector of strings, one log line per string, captured since * the last call to startCapturingLogMessages() in this test. */ - const std::vector<std::string>& getCapturedTextFormatLogMessages() const; + std::vector<std::string> getCapturedTextFormatLogMessages() const; std::vector<BSONObj> getCapturedBSONFormatLogMessages() const; /** diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index 0b794829149..429b0dfcb42 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -33,7 +33,7 @@ def fmtBuildInfo(data): fmtStr(env.subst(obj['value'])), fmtBool(obj['inBuildInfo']), fmtBool(obj['inVersion'])) - return ',\n'.join([fmtObj(obj) for _,obj in data.items()]) + return ',\n'.join([fmtObj(obj) for _, obj in data.items()]) buildInfoInitializer = fmtBuildInfo(env['MONGO_BUILDINFO_ENVIRONMENT_DATA']) @@ -564,6 +564,14 @@ env.Benchmark( ], ) +env.Benchmark( + target='tick_source_bm', + source=[ + 'tick_source_bm.cpp', + ], + LIBDEPS=[], +) + env.Library( target='future_util', source=[ @@ -771,6 +779,7 @@ icuEnv.CppUnitTest( 'processinfo', 'procparser' if env.TargetOSIs('linux') else [], 'progress_meter', + 'regex_util', 'safe_num', 'secure_zero_memory', 'summation', @@ -872,4 +881,3 @@ env.Benchmark( 'processinfo', ], ) - diff --git a/src/mongo/util/assert_util.h b/src/mongo/util/assert_util.h index d423fa3454f..d03605e6192 100644 --- a/src/mongo/util/assert_util.h +++ b/src/mongo/util/assert_util.h @@ -702,6 +702,19 @@ Status exceptionToStatus() noexcept; #define MONGO_UNREACHABLE_TASSERT(msgid) tasserted(msgid, "Hit a MONGO_UNREACHABLE_TASSERT!") /** + * Produces an invariant failure if executed. Subset of MONGO_UNREACHABLE, but specifically + * to indicate that the program has reached a function that is unimplemented and should be + * unreachable from production. + * Example: + * + * void myFuncToDo() { + * MONGO_UNIMPLEMENTED; + * } + */ +#define MONGO_UNIMPLEMENTED \ + ::mongo::invariantFailed("Hit a MONGO_UNIMPLEMENTED!", __FILE__, __LINE__); + +/** * Like `MONGO_UNIMPLEMENTED`, but triggers a `tassert` instead of an `invariant` */ #define MONGO_UNIMPLEMENTED_TASSERT(msgid) tasserted(msgid, "Hit a MONGO_UNIMPLEMENTED_TASSERT!") diff --git a/src/mongo/util/assert_util_test.cpp b/src/mongo/util/assert_util_test.cpp index d4740ea966e..68e45dff3e4 100644 --- a/src/mongo/util/assert_util_test.cpp +++ b/src/mongo/util/assert_util_test.cpp @@ -144,7 +144,7 @@ TEST(AssertUtils, UassertNamedCodeWithTwoCategories) { } MONGO_STATIC_ASSERT(!error_details::isNamedCode<19999>); -// ExceptionFor<ErrorCodes::Error(19999)> invalidType; // Must not compile. +// ExceptionFor<ErrorCodes::Error19999)> invalidType; // Must not compile. TEST(AssertUtils, UassertNumericCode) { ASSERT_CATCHES(19999, DBException); diff --git a/src/mongo/util/concurrency/ticketholder.cpp b/src/mongo/util/concurrency/ticketholder.cpp index 4de937b4bd6..f0ff7fdb4ce 100644 --- a/src/mongo/util/concurrency/ticketholder.cpp +++ b/src/mongo/util/concurrency/ticketholder.cpp @@ -40,13 +40,10 @@ #include <iostream> #include "mongo/logv2/log.h" -#include "mongo/util/fail_point.h" #include "mongo/util/str.h" namespace mongo { -MONGO_FAIL_POINT_DEFINE(hangTicketRelease); - TicketHolder::~TicketHolder() = default; #if defined(__linux__) @@ -147,12 +144,6 @@ boost::optional<Ticket> SemaphoreTicketHolder::waitForTicketUntil(OperationConte } void SemaphoreTicketHolder::release(AdmissionContext* admCtx, Ticket&& ticket) { - if (MONGO_unlikely(hangTicketRelease.shouldFail())) { - LOGV2(8435300, - "Hanging hangTicketRelease in release() due to 'hangTicketRelease' " - "failpoint"); - hangTicketRelease.pauseWhileSet(); - } check(sem_post(&_sem)); ticket.release(); } @@ -250,12 +241,6 @@ boost::optional<Ticket> SemaphoreTicketHolder::waitForTicketUntil(OperationConte } void SemaphoreTicketHolder::release(AdmissionContext* admCtx, Ticket&& ticket) { - if (MONGO_unlikely(hangTicketRelease.shouldFail())) { - LOGV2(8435301, - "Hanging hangTicketRelease in release() due to 'hangTicketRelease' " - "failpoint"); - hangTicketRelease.pauseWhileSet(); - } { stdx::lock_guard<Latch> lk(_mutex); _num++; diff --git a/src/mongo/util/future_test_utils.h b/src/mongo/util/future_test_utils.h index 8bbecfd1faa..ec7bd783030 100644 --- a/src/mongo/util/future_test_utils.h +++ b/src/mongo/util/future_test_utils.h @@ -71,12 +71,6 @@ class DummyInterruptible final : public Interruptible { // Must be implemented because it's called by Interruptible::waitForConditionOrInterrupt. return Status::OK(); } - IgnoreInterruptsState pushIgnoreInterrupts() override { - MONGO_UNREACHABLE; - } - void popIgnoreInterrupts(IgnoreInterruptsState iis) override { - MONGO_UNREACHABLE; - } DeadlineState pushArtificialDeadline(Date_t deadline, ErrorCodes::Error error) override { MONGO_UNREACHABLE; } diff --git a/src/mongo/util/interruptible.h b/src/mongo/util/interruptible.h index 55cc25944af..1e9cc070c64 100644 --- a/src/mongo/util/interruptible.h +++ b/src/mongo/util/interruptible.h @@ -107,28 +107,6 @@ protected: * Returns the equivalent of Date_t::now() + waitFor for the InterruptibleBase's clock */ virtual Date_t getExpirationDateForWaitForValue(Milliseconds waitFor) = 0; - - struct IgnoreInterruptsState { - bool ignoreInterrupts; - DeadlineState deadline; - }; - - /** - * Pushes an ignore interruption critical section into the InterruptibleBase. - * Until an associated popIgnoreInterrupts() is invoked, the InterruptibleBase should ignore - * interruptions related to explicit interruption or previously set deadlines. - * - * Note that new deadlines can be set after this is called, which will again introduce the - * possibility of interruption. - * - * Returns state needed to pop interruption. - */ - virtual IgnoreInterruptsState pushIgnoreInterrupts() = 0; - - /** - * Pops the ignored interruption critical section introduced by push. - */ - virtual void popIgnoreInterrupts(IgnoreInterruptsState iis) = 0; }; /** @@ -194,44 +172,6 @@ private: return DeadlineGuard(*this, deadline, error); } - /** - * An interruption guard provides a region where interruption is ignored. - * - * Note that this causes the deadline to be reset to Date_t::max(), but that it can also be - * subsequently reduced in size after the fact. - */ - class IgnoreInterruptionsGuard { - public: - IgnoreInterruptionsGuard(const IgnoreInterruptionsGuard&) = delete; - IgnoreInterruptionsGuard& operator=(const IgnoreInterruptionsGuard&) = delete; - - IgnoreInterruptionsGuard(IgnoreInterruptionsGuard&& other) - : _interruptible(other._interruptible), _oldState(other._oldState) { - other._interruptible = nullptr; - } - - IgnoreInterruptionsGuard& operator=(IgnoreInterruptionsGuard&&) = delete; - - ~IgnoreInterruptionsGuard() { - if (_interruptible) { - _interruptible->popIgnoreInterrupts(_oldState); - } - } - - private: - friend Interruptible; - - explicit IgnoreInterruptionsGuard(Interruptible& interruptible) - : _interruptible(&interruptible), _oldState(_interruptible->pushIgnoreInterrupts()) {} - - Interruptible* _interruptible; - IgnoreInterruptsState _oldState; - }; - - IgnoreInterruptionsGuard makeIgnoreInterruptionsGuard() { - return IgnoreInterruptionsGuard(*this); - } - public: class WaitListener; @@ -306,24 +246,6 @@ public: } /** - * Invokes the passed callback with an interruption guard active. Additionally handles the - * dance of try/catching the invocation and checking checkForInterrupt with the guard inactive - * (to allow a higher level timeout to override a lower level one, or for top level interruption - * to propagate) - */ - template <typename Callback> - decltype(auto) runWithoutInterruptionExceptAtGlobalShutdown(Callback&& cb) { - try { - const auto guard = makeIgnoreInterruptionsGuard(); - return std::forward<Callback>(cb)(); - } catch (const ExceptionForCat<ErrorCategory::ExceededTimeLimitError>&) { - // May throw replacement exception - checkForInterrupt(); - throw; - } - } - - /** * Raises a AssertionException if this operation is in a killed state. */ void checkForInterrupt() { @@ -591,19 +513,6 @@ class Interruptible::NotInterruptible final : public Interruptible { return Status::OK(); } - // It's invalid to call the deadline or ignore interruption guards on a possibly noop - // Interruptible. - // - // The noop Interruptible should only be invoked as a default arg at the bottom of the call - // stack (with types that won't modify it's invocation) - IgnoreInterruptsState pushIgnoreInterrupts() override { - MONGO_UNREACHABLE; - } - - void popIgnoreInterrupts(IgnoreInterruptsState) override { - MONGO_UNREACHABLE; - } - DeadlineState pushArtificialDeadline(Date_t deadline, ErrorCodes::Error error) override { MONGO_UNREACHABLE; } diff --git a/src/mongo/util/intrusive_counter.cpp b/src/mongo/util/intrusive_counter.cpp index e33cbc87da5..4c8efad7959 100644 --- a/src/mongo/util/intrusive_counter.cpp +++ b/src/mongo/util/intrusive_counter.cpp @@ -52,7 +52,8 @@ intrusive_ptr<const RCString> RCString::create(StringData s) { ptr->_size = s.size(); char* stringStart = reinterpret_cast<char*>(ptr.get()) + sizeof(RCString); - s.copyTo(stringStart, true); + s.copy(stringStart, s.size()); + stringStart[s.size()] = '\0'; return ptr; } diff --git a/src/mongo/util/net/SConscript b/src/mongo/util/net/SConscript index 272cd9740a1..f885a122325 100644 --- a/src/mongo/util/net/SConscript +++ b/src/mongo/util/net/SConscript @@ -252,9 +252,10 @@ if get_option('ssl') == 'on': env.CppUnitTest( target='util_net_ssl_test', source=[ - 'ssl_manager_test.cpp', - 'ssl_options_test.cpp', - 'sock_test.cpp', + "ssl_manager_test.cpp", + "ssl_options_test.cpp", + "sock_test.cpp", + "sock_test_utils.cpp", ], LIBDEPS=[ '$BUILD_DIR/mongo/client/connection_string', diff --git a/src/mongo/util/net/sock_test.cpp b/src/mongo/util/net/sock_test.cpp index ccb751ea2dd..84d91f5ba1b 100644 --- a/src/mongo/util/net/sock_test.cpp +++ b/src/mongo/util/net/sock_test.cpp @@ -31,176 +31,17 @@ #include "mongo/util/net/sock.h" -#ifndef _WIN32 -#include <netdb.h> -#include <sys/socket.h> -#include <sys/types.h> -#endif - #include "mongo/db/server_options.h" #include "mongo/stdx/thread.h" #include "mongo/unittest/unittest.h" #include "mongo/util/concurrency/notification.h" #include "mongo/util/fail_point.h" +#include "mongo/util/net/sock_test_utils.h" #include "mongo/util/net/socket_exception.h" namespace { using namespace mongo; -using std::shared_ptr; - -typedef std::shared_ptr<Socket> SocketPtr; -typedef std::pair<SocketPtr, SocketPtr> SocketPair; - -// On UNIX, make a connected pair of PF_LOCAL (aka PF_UNIX) sockets via the native 'socketpair' -// call. The 'type' parameter should be one of SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET, etc. -// For Win32, we don't have a native socketpair function, so we hack up a connected PF_INET -// pair on a random port. -SocketPair socketPair(int type, int protocol = 0); - -#if defined(_WIN32) -namespace detail { -void awaitAccept(SOCKET* acceptSock, SOCKET listenSock, Notification<void>& notify) { - *acceptSock = INVALID_SOCKET; - const SOCKET result = ::accept(listenSock, nullptr, 0); - if (result != INVALID_SOCKET) { - *acceptSock = result; - } - notify.set(); -} - -void awaitConnect(SOCKET* connectSock, const struct addrinfo& where, Notification<void>& notify) { - *connectSock = INVALID_SOCKET; - SOCKET newSock = ::socket(where.ai_family, where.ai_socktype, where.ai_protocol); - if (newSock != INVALID_SOCKET) { - int result = ::connect(newSock, where.ai_addr, where.ai_addrlen); - if (result == 0) { - *connectSock = newSock; - } - } - notify.set(); -} -} // namespace detail - -SocketPair socketPair(const int type, const int protocol) { - const int domain = PF_INET; - - // Create a listen socket and a connect socket. - const SOCKET listenSock = ::socket(domain, type, protocol); - if (listenSock == INVALID_SOCKET) - return SocketPair(); - - // Bind the listen socket on port zero, it will pick one for us, and start it listening - // for connections. - struct addrinfo hints, *res; - ::memset(&hints, 0, sizeof(hints)); - hints.ai_family = PF_INET; - hints.ai_socktype = type; - hints.ai_flags = AI_PASSIVE; - - int result = ::getaddrinfo(nullptr, "0", &hints, &res); - if (result != 0) { - closesocket(listenSock); - return SocketPair(); - } - - result = ::bind(listenSock, res->ai_addr, res->ai_addrlen); - if (result != 0) { - closesocket(listenSock); - ::freeaddrinfo(res); - return SocketPair(); - } - - // Read out the port to which we bound. - sockaddr_in bindAddr; - ::socklen_t len = sizeof(bindAddr); - ::memset(&bindAddr, 0, sizeof(bindAddr)); - result = ::getsockname(listenSock, reinterpret_cast<struct sockaddr*>(&bindAddr), &len); - if (result != 0) { - closesocket(listenSock); - ::freeaddrinfo(res); - return SocketPair(); - } - - result = ::listen(listenSock, 1); - if (result != 0) { - closesocket(listenSock); - ::freeaddrinfo(res); - return SocketPair(); - } - - struct addrinfo connectHints, *connectRes; - ::memset(&connectHints, 0, sizeof(connectHints)); - connectHints.ai_family = PF_INET; - connectHints.ai_socktype = type; - std::stringstream portStream; - portStream << ntohs(bindAddr.sin_port); - result = ::getaddrinfo(nullptr, portStream.str().c_str(), &connectHints, &connectRes); - if (result != 0) { - closesocket(listenSock); - ::freeaddrinfo(res); - return SocketPair(); - } - - // I'd prefer to avoid trying to do this non-blocking on Windows. Just spin up some - // threads to do the connect and acccept. - - Notification<void> accepted; - SOCKET acceptSock = INVALID_SOCKET; - stdx::thread acceptor([&] { detail::awaitAccept(&acceptSock, listenSock, accepted); }); - - Notification<void> connected; - SOCKET connectSock = INVALID_SOCKET; - stdx::thread connector([&] { detail::awaitConnect(&connectSock, *connectRes, connected); }); - - connected.get(); - connector.join(); - if (connectSock == INVALID_SOCKET) { - closesocket(listenSock); - ::freeaddrinfo(res); - ::freeaddrinfo(connectRes); - closesocket(acceptSock); - closesocket(connectSock); - return SocketPair(); - } - - accepted.get(); - acceptor.join(); - if (acceptSock == INVALID_SOCKET) { - closesocket(listenSock); - ::freeaddrinfo(res); - ::freeaddrinfo(connectRes); - closesocket(acceptSock); - closesocket(connectSock); - return SocketPair(); - } - - closesocket(listenSock); - ::freeaddrinfo(res); - ::freeaddrinfo(connectRes); - - SocketPtr first(new Socket(static_cast<int>(acceptSock), SockAddr())); - SocketPtr second(new Socket(static_cast<int>(connectSock), SockAddr())); - - return SocketPair(first, second); -} -#else -// We can just use ::socketpair and wrap up the result in a Socket. -SocketPair socketPair(const int type, const int protocol) { - // PF_LOCAL is the POSIX name for Unix domain sockets, while PF_UNIX - // is the name that BSD used. We use the BSD name because it is more - // widely supported (e.g. Solaris 10). - const int domain = PF_UNIX; - - int socks[2]; - const int result = ::socketpair(domain, type, protocol, socks); - if (result == 0) { - return SocketPair(SocketPtr(new Socket(socks[0], SockAddr())), - SocketPtr(new Socket(socks[1], SockAddr()))); - } - return SocketPair(); -} -#endif // This should match the name of the fail point declared in sock.cpp. const char kSocketFailPointName[] = "throwSockExcep"; diff --git a/src/mongo/util/net/sock_test_utils.cpp b/src/mongo/util/net/sock_test_utils.cpp new file mode 100644 index 00000000000..9164b8077ca --- /dev/null +++ b/src/mongo/util/net/sock_test_utils.cpp @@ -0,0 +1,199 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/util/net/sock_test_utils.h" + +#ifndef _WIN32 +#include <netdb.h> +#include <sys/socket.h> +#include <sys/types.h> +#endif + +#include "mongo/stdx/thread.h" +#include "mongo/util/concurrency/notification.h" +#include "mongo/util/net/socket_exception.h" + +namespace mongo { +namespace { + +#if defined(_WIN32) +namespace detail { +void awaitAccept(SOCKET* acceptSock, SOCKET listenSock, Notification<void>& notify) { + *acceptSock = INVALID_SOCKET; + const SOCKET result = ::accept(listenSock, nullptr, 0); + if (result != INVALID_SOCKET) { + *acceptSock = result; + } + notify.set(); +} + +void awaitConnect(SOCKET* connectSock, const struct addrinfo& where, Notification<void>& notify) { + *connectSock = INVALID_SOCKET; + SOCKET newSock = ::socket(where.ai_family, where.ai_socktype, where.ai_protocol); + if (newSock != INVALID_SOCKET) { + int result = ::connect(newSock, where.ai_addr, where.ai_addrlen); + if (result == 0) { + *connectSock = newSock; + } + } + notify.set(); +} +} // namespace detail + +SocketPair socketPairImpl(const int type, const int protocol) { + const int domain = PF_INET; + + // Create a listen socket and a connect socket. + const SOCKET listenSock = ::socket(domain, type, protocol); + if (listenSock == INVALID_SOCKET) + return SocketPair(); + + // Bind the listen socket on port zero, it will pick one for us, and start it listening + // for connections. + struct addrinfo hints, *res; + ::memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_INET; + hints.ai_socktype = type; + hints.ai_flags = AI_PASSIVE; + + int result = ::getaddrinfo(nullptr, "0", &hints, &res); + if (result != 0) { + closesocket(listenSock); + return SocketPair(); + } + + result = ::bind(listenSock, res->ai_addr, res->ai_addrlen); + if (result != 0) { + closesocket(listenSock); + ::freeaddrinfo(res); + return SocketPair(); + } + + // Read out the port to which we bound. + sockaddr_in bindAddr; + ::socklen_t len = sizeof(bindAddr); + ::memset(&bindAddr, 0, sizeof(bindAddr)); + result = ::getsockname(listenSock, reinterpret_cast<struct sockaddr*>(&bindAddr), &len); + if (result != 0) { + closesocket(listenSock); + ::freeaddrinfo(res); + return SocketPair(); + } + + result = ::listen(listenSock, 1); + if (result != 0) { + closesocket(listenSock); + ::freeaddrinfo(res); + return SocketPair(); + } + + struct addrinfo connectHints, *connectRes; + ::memset(&connectHints, 0, sizeof(connectHints)); + connectHints.ai_family = PF_INET; + connectHints.ai_socktype = type; + std::stringstream portStream; + portStream << ntohs(bindAddr.sin_port); + result = ::getaddrinfo(nullptr, portStream.str().c_str(), &connectHints, &connectRes); + if (result != 0) { + closesocket(listenSock); + ::freeaddrinfo(res); + return SocketPair(); + } + + // I'd prefer to avoid trying to do this non-blocking on Windows. Just spin up some + // threads to do the connect and acccept. + + Notification<void> accepted; + SOCKET acceptSock = INVALID_SOCKET; + stdx::thread acceptor([&] { detail::awaitAccept(&acceptSock, listenSock, accepted); }); + + Notification<void> connected; + SOCKET connectSock = INVALID_SOCKET; + stdx::thread connector([&] { detail::awaitConnect(&connectSock, *connectRes, connected); }); + + connected.get(); + connector.join(); + if (connectSock == INVALID_SOCKET) { + closesocket(listenSock); + ::freeaddrinfo(res); + ::freeaddrinfo(connectRes); + closesocket(acceptSock); + closesocket(connectSock); + return SocketPair(); + } + + accepted.get(); + acceptor.join(); + if (acceptSock == INVALID_SOCKET) { + closesocket(listenSock); + ::freeaddrinfo(res); + ::freeaddrinfo(connectRes); + closesocket(acceptSock); + closesocket(connectSock); + return SocketPair(); + } + + closesocket(listenSock); + ::freeaddrinfo(res); + ::freeaddrinfo(connectRes); + + SocketPtr first = std::make_shared<Socket>(static_cast<int>(acceptSock), SockAddr()); + SocketPtr second = std::make_shared<Socket>(static_cast<int>(connectSock), SockAddr()); + return SocketPair(first, second); +} +#else +// We can just use ::socketpair and wrap up the result in a Socket. +SocketPair socketPairImpl(const int type, const int protocol) { + // PF_LOCAL is the POSIX name for Unix domain sockets, while PF_UNIX + // is the name that BSD used. We use the BSD name because it is more + // widely supported (e.g. Solaris 10). + const int domain = PF_UNIX; + + int socks[2]; + const int result = ::socketpair(domain, type, protocol, socks); + if (result == 0) { + return SocketPair(std::make_shared<Socket>(socks[0], SockAddr()), + std::make_shared<Socket>(socks[1], SockAddr())); + } + return SocketPair(); +} +#endif +} // namespace + +// On UNIX, make a connected pair of PF_LOCAL (aka PF_UNIX) sockets via the native 'socketpair' +// call. The 'type' parameter should be one of SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET, etc. +// For Win32, we don't have a native socketpair function, so we hack up a connected PF_INET +// pair on a random port. +SocketPair socketPair(int type, int protocol) { + return socketPairImpl(type, protocol); +} + +} // namespace mongo diff --git a/src/mongo/util/net/sock_test_utils.h b/src/mongo/util/net/sock_test_utils.h new file mode 100644 index 00000000000..f168edf1292 --- /dev/null +++ b/src/mongo/util/net/sock_test_utils.h @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/util/net/sock.h" + +namespace mongo { +using SocketPtr = std::shared_ptr<Socket>; +using SocketPair = std::pair<SocketPtr, SocketPtr>; + +// Create a connected pair of sockets for testing purposes. +SocketPair socketPair(int type, int protocol = 0); + +} // namespace mongo diff --git a/src/mongo/util/net/sockaddr.cpp b/src/mongo/util/net/sockaddr.cpp index 1a2b058cbf3..5bf2998df6d 100644 --- a/src/mongo/util/net/sockaddr.cpp +++ b/src/mongo/util/net/sockaddr.cpp @@ -149,7 +149,7 @@ void SockAddr::initUnixDomainSocket(StringData path, int port) { uassert( 13079, "path to unix socket too long", path.size() < sizeof(as<sockaddr_un>().sun_path)); as<sockaddr_un>().sun_family = AF_UNIX; - path.copyTo(as<sockaddr_un>().sun_path, /* includeEndingNull =*/true); + str::copyAsCString(as<sockaddr_un>().sun_path, path); addressSize = sizeof(sockaddr_un); _isValid = true; } diff --git a/src/mongo/util/net/ssl/detail/impl/engine_apple.ipp b/src/mongo/util/net/ssl/detail/impl/engine_apple.ipp index 015a44fcee8..b863cb0e75a 100644 --- a/src/mongo/util/net/ssl/detail/impl/engine_apple.ipp +++ b/src/mongo/util/net/ssl/detail/impl/engine_apple.ipp @@ -158,10 +158,6 @@ bool engine::_initSSL(stream_base::handshake_type type, asio::error_code& ec) { } if (status == ::errSecSuccess) { - status = ::SSLSetPeerID(_ssl.get(), _ssl.get(), sizeof(native_handle_type)); - } - - if (status == ::errSecSuccess) { status = ::SSLSetIOFuncs(_ssl.get(), read_func, write_func); } diff --git a/src/mongo/util/net/ssl_manager_apple.cpp b/src/mongo/util/net/ssl_manager_apple.cpp index c13a4bb8eb8..dcde5542ca9 100644 --- a/src/mongo/util/net/ssl_manager_apple.cpp +++ b/src/mongo/util/net/ssl_manager_apple.cpp @@ -1156,7 +1156,6 @@ public: } uassertOSStatusOK(::SSLSetConnection(_ssl.get(), static_cast<void*>(this))); - uassertOSStatusOK(::SSLSetPeerID(_ssl.get(), _ssl.get(), sizeof(_ssl))); uassertOSStatusOK(::SSLSetIOFuncs(_ssl.get(), read_func, write_func)); uassertOSStatusOK(::SSLSetProtocolVersionMin(_ssl.get(), ctx->protoMin)); uassertOSStatusOK(::SSLSetProtocolVersionMax(_ssl.get(), ctx->protoMax)); @@ -1654,11 +1653,14 @@ Future<SSLPeerInfo> SSLManagerApple::parseAndValidatePeerCertificate( return swPeerSubjectName.getStatus(); } const auto peerSubjectName = std::move(swPeerSubjectName.getValue()); - LOGV2_DEBUG(23207, - 2, - "Accepted TLS connection from peer: {peerSubjectName}", - "Accepted TLS connection from peer", - "peerSubjectName"_attr = peerSubjectName); + // The cipher will be presented as a number. + ::SSLCipherSuite cipher; + uassertOSStatusOK(::SSLGetNegotiatedCipher(ssl, &cipher)); + + LOGV2_INFO(6723803, + "Accepted TLS connection from peer", + "peerSubjectName"_attr = peerSubjectName, + "cipher"_attr = cipher); // Server side. if (remoteHost.empty()) { diff --git a/src/mongo/util/net/ssl_manager_openssl.cpp b/src/mongo/util/net/ssl_manager_openssl.cpp index 43c646bc16c..b27676156d6 100644 --- a/src/mongo/util/net/ssl_manager_openssl.cpp +++ b/src/mongo/util/net/ssl_manager_openssl.cpp @@ -319,23 +319,6 @@ X509* X509_OBJECT_get0_X509(const X509_OBJECT* a) { return a->data.x509; } -UniqueStackOfX509 SSLgetVerifiedChain(SSL* s) { - auto* store = SSL_CTX_get_cert_store(SSL_get_SSL_CTX(s)); - auto* peerChain = SSL_get_peer_cert_chain(s); - UniqueX509 peer(SSL_get_peer_certificate(s)); - - UniqueX509StoreCtx ctx(X509_STORE_CTX_new()); - if (!X509_STORE_CTX_init(ctx.get(), store, peer.get(), peerChain)) { - return nullptr; - } - - if (X509_verify_cert(ctx.get()) <= 0) { - return nullptr; - } - - return UniqueStackOfX509(X509_STORE_CTX_get1_chain(ctx.get())); -} - const OCSP_CERTID* OCSP_SINGLERESP_get0_id(const OCSP_SINGLERESP* single) { return single->certId; } @@ -368,16 +351,24 @@ static ASN1OID tlsFeatureOID("1.3.6.1.5.5.7.1.24", "tlsfeature", "TLS Feature"); static int const NID_tlsfeature = OBJ_create(tlsFeatureOID.identifier.c_str(), tlsFeatureOID.shortDescription.c_str(), tlsFeatureOID.longDescription.c_str()); +#endif -#else UniqueStackOfX509 SSLgetVerifiedChain(SSL* s) { - auto chain = SSL_get0_verified_chain(s); + auto* store = SSL_CTX_get_cert_store(SSL_get_SSL_CTX(s)); + auto* peerChain = SSL_get_peer_cert_chain(s); + UniqueX509 peer(SSL_get_peer_certificate(s)); - return UniqueStackOfX509(X509_chain_up_ref(chain)); -} + UniqueX509StoreCtx ctx(X509_STORE_CTX_new()); + if (!X509_STORE_CTX_init(ctx.get(), store, peer.get(), peerChain)) { + return nullptr; + } -#endif + if (X509_verify_cert(ctx.get()) <= 0) { + return nullptr; + } + return UniqueStackOfX509(X509_STORE_CTX_get1_chain(ctx.get())); +} SSLX509Name convertX509ToSSLX509Name(X509_NAME* x509Name) { std::vector<std::vector<SSLX509Name::Entry>> entries; @@ -2996,7 +2987,7 @@ bool SSLManagerOpenSSL::_setupCRL(SSL_CTX* context, const std::string& crlFile) X509_STORE* store = SSL_CTX_get_cert_store(context); fassert(16583, store); - X509_STORE_set_flags(store, X509_V_FLAG_CRL_CHECK); + X509_STORE_set_flags(store, X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL); X509_LOOKUP* lookup = X509_STORE_add_lookup(store, X509_LOOKUP_file()); fassert(16584, lookup); @@ -3010,15 +3001,7 @@ bool SSLManagerOpenSSL::_setupCRL(SSL_CTX* context, const std::string& crlFile) return false; } - if (status == 1) { - LOGV2(4652601, "ssl imported 1 revoked certificate from the revocation list."); - } else { - LOGV2(4652602, - "ssl imported {numberCerts} revoked certificates from the revocation list", - "SSL imported revoked certificates from the revocation list", - "numberCerts"_attr = status); - } - + LOGV2(4652602, "SSL imported certificate revocation list(s)", "numberCRLs"_attr = status); return true; } @@ -3280,11 +3263,11 @@ Future<SSLPeerInfo> SSLManagerOpenSSL::parseAndValidatePeerCertificate( // TODO: check optional cipher restriction, using cert. auto peerSubject = getCertificateSubjectX509Name(peerCert.get()); - LOGV2_DEBUG(23229, - 2, - "Accepted TLS connection from peer: {peerSubject}", - "Accepted TLS connection from peer", - "peerSubject"_attr = peerSubject); + const auto cipher = SSL_get_current_cipher(conn); + LOGV2_INFO(6723801, + "Accepted TLS connection from peer", + "peerSubject"_attr = peerSubject, + "cipher"_attr = SSL_CIPHER_get_name(cipher)); StatusWith<stdx::unordered_set<RoleName>> swPeerCertificateRoles = _parsePeerRoles(peerCert.get()); diff --git a/src/mongo/util/net/ssl_manager_test.cpp b/src/mongo/util/net/ssl_manager_test.cpp index 1dd6585f7c1..80e83af242a 100644 --- a/src/mongo/util/net/ssl_manager_test.cpp +++ b/src/mongo/util/net/ssl_manager_test.cpp @@ -29,15 +29,20 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest +#include <asio.hpp> +#include <boost/filesystem.hpp> #include <fstream> #include "mongo/config.h" #include "mongo/platform/basic.h" +#include "mongo/bson/json.h" #include "mongo/transport/service_entry_point.h" #include "mongo/transport/transport_layer_asio.h" #include "mongo/transport/transport_layer_manager.h" +#include "mongo/util/net/sock_test_utils.h" #include "mongo/util/net/ssl/context.hpp" +#include "mongo/util/net/ssl/stream.hpp" #include "mongo/util/net/ssl_manager.h" #include "mongo/util/net/ssl_options.h" @@ -49,10 +54,36 @@ #include "mongo/util/net/ssl/context_openssl.hpp" #endif +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest + +namespace fs = boost::filesystem; namespace mongo { namespace { +#define TEST_CERTS_DIR "jstests/libs/" +// certs & CRLs rooted in ca.pem +constexpr const char* caFile = TEST_CERTS_DIR "ca.pem"; +constexpr const char* serverKeyFile = TEST_CERTS_DIR "server.pem"; +constexpr const char* clientKeyFile = TEST_CERTS_DIR "client.pem"; +constexpr const char* revokedClientKeyFile = TEST_CERTS_DIR "client_revoked.pem"; + +constexpr const char* intermediateACaFile = TEST_CERTS_DIR "intermediate-ca.pem"; +constexpr const char* intermediateALeafKeyFile = TEST_CERTS_DIR "server-intermediate-leaf.pem"; +constexpr const char* intermediateBCaFile = TEST_CERTS_DIR "intermediate-ca-B.pem"; +constexpr const char* intermediateBLeafKeyFile = TEST_CERTS_DIR "intermediate-ca-B-leaf.pem"; +constexpr const char* emptyCRL = TEST_CERTS_DIR "crl.pem"; +constexpr const char* expiredCRL = TEST_CERTS_DIR "crl_expired.pem"; +constexpr const char* clientRevokedCRL = TEST_CERTS_DIR "crl_client_revoked.pem"; +constexpr const char* intermediateBRevokedCRL = TEST_CERTS_DIR "crl_intermediate_ca_B_revoked.pem"; +constexpr const char* intermediateBCRL = TEST_CERTS_DIR "crl_from_intermediate_ca_B.pem"; + +// certs & CRLs rooted in trusted-ca.pem +constexpr const char* trustedCaFile = TEST_CERTS_DIR "trusted-ca.pem"; +constexpr const char* trustedServerKeyFile = TEST_CERTS_DIR "trusted-server.pem"; +constexpr const char* trustedClientKeyFile = TEST_CERTS_DIR "trusted-client.pem"; +constexpr const char* trustedEmptyCRL = TEST_CERTS_DIR "crl_from_trusted_ca.pem"; + // Test implementation needed by ASIO transport. class ServiceEntryPointUtil : public ServiceEntryPoint { public: @@ -103,7 +134,7 @@ public: } private: - mutable Mutex _mutex = MONGO_MAKE_LATCH("::_mutex"); + mutable Mutex _mutex; stdx::condition_variable _cv; std::vector<transport::SessionHandle> _sessions; transport::TransportLayer* _transport = nullptr; @@ -115,6 +146,76 @@ std::string loadFile(const std::string& name) { return str; } +// Reads the input stream until EOF or a valid PEM block is encountered. +// Skips private key PEM blocks if includePrivateKeys is true. +// Returns the parsed PEM block as a string (with newlines), or an empty +// string if none is found or a read error occurs. +std::string readOnePEMBlock(std::ifstream& inputStrm, bool includePrivateKeys) { + std::string line; + for (;;) { + std::stringstream output; + bool foundBegin = false; + bool foundEnd = false; + bool discard = false; + + while (!foundBegin && std::getline(inputStrm, line)) { + StringData lineSD(line); + foundBegin = (lineSD.startsWith("-----BEGIN ") && lineSD.endsWith("-----")); + } + if (!foundBegin) { + return ""; + } + + discard = (!includePrivateKeys && line.find("PRIVATE KEY") != std::string::npos); + output << line << std::endl; + + while (!foundEnd && std::getline(inputStrm, line)) { + StringData lineSD(line); + output << line << std::endl; + foundEnd = (lineSD.startsWith("-----END ") && lineSD.endsWith("-----")); + } + if (!foundEnd) { + return ""; + } + if (!discard) { + return output.str(); + } + } +} + +struct PEMFileSpec { + std::string path; + bool includePrivateKeys{false}; + void serialize(BSONObjBuilder* bob) const { + bob->append("path", path); + bob->append("includePrivateKeys", includePrivateKeys); + } +}; +// Given a list of PEM files, this concatenates the PEM blocks in those files +// (optionally filtering out private keys) and writes the result into a temporary +// file. Returns the path to the temp file. +std::string combinePEMFiles(const std::vector<PEMFileSpec>& pemSpecs) { + // make a temp file for the output + auto path = fs::temp_directory_path() / fs::unique_path("tmpfile_%%%%_%%%%_%%%%_%%%%.pem"); + std::ofstream outStream(path.string()); + invariant(outStream.is_open()); + + LOGV2( + 9476600, "Combining PEM files", "output"_attr = path.string(), "pemFiles"_attr = pemSpecs); + + // read & parse the PEM files; append PEM blocks to output + for (auto& pemSpec : pemSpecs) { + std::ifstream input(pemSpec.path); + std::string pemBlock; + do { + pemBlock = readOnePEMBlock(input, pemSpec.includePrivateKeys); + outStream << pemBlock; + } while (!pemBlock.empty()); + } + outStream.close(); + return path.string(); +} + TEST(SSLManager, matchHostname) { enum Expected : bool { match = true, mismatch = false }; const struct { @@ -734,6 +835,8 @@ TEST(SSLManager, TransientSSLParamsStressTestWithManager) { #endif // MONGO_CONFIG_SSL_PROVIDER == MONGO_CONFIG_SSL_PROVIDER_OPENSSL +#ifdef MONGO_CONFIG_SSL + static bool isSanWarningWritten(const std::vector<std::string>& logLines) { for (const auto& line : logLines) { if (std::string::npos != @@ -780,6 +883,288 @@ TEST(SSLManager, InitContextNoSanWarning) { ASSERT_FALSE(isSanWarningWritten(getCapturedTextFormatLogMessages())); } +class SSLTestFixture { +public: + SSLTestFixture(const SSLParams& ingressParams, + const SSLParams& egressParams, + bool ingressIsServer = true, + bool egressIsServer = true, + const boost::optional<TransientSSLParams>& transientSSLParams = boost::none) { + auto serviceContext = ServiceContext::make(); + setGlobalServiceContext(std::move(serviceContext)); + + // SSLManagerWindows uses this global boolean to decide whether to + // use unique key container names when setting up the crypto context. + // This must be true in order for the handshake to work. + isSSLServer = true; + + serverSSLManager = SSLManagerInterface::create(ingressParams, ingressIsServer); + clientSSLManager = + SSLManagerInterface::create(egressParams, transientSSLParams, egressIsServer); + + serverSSLContext = std::make_shared<asio::ssl::context>(asio::ssl::context::sslv23); + clientSSLContext = std::make_shared<asio::ssl::context>(asio::ssl::context::sslv23); + uassertStatusOK( + serverSSLManager->initSSLContext(serverSSLContext->native_handle(), + ingressParams, + SSLManagerInterface::ConnectionDirection::kIncoming)); + uassertStatusOK( + clientSSLManager->initSSLContext(clientSSLContext->native_handle(), + egressParams, + SSLManagerInterface::ConnectionDirection::kOutgoing)); + } + + void doHandshake() { + auto socks = socketPair(SOCK_STREAM); + + serverConn = std::make_shared<ConnectionContext>(socks.first->rawFD(), *serverSSLContext); + clientConn = std::make_shared<ConnectionContext>(socks.second->rawFD(), *clientSSLContext); + Status serverStatus = Status::OK(); + Status clientStatus = Status::OK(); + + auto serverThread = stdx::thread([this, &serverStatus]() { + try { + serverConn->sslSocket->handshake(asio::ssl::stream_base::server); + } catch (const DBException& ex) { + serverStatus = ex.toStatus().withContext("Server handshake failed"); + } + }); + + try { + clientConn->sslSocket->handshake(asio::ssl::stream_base::client); + } catch (const DBException& ex) { + clientStatus = ex.toStatus().withContext("Client handshake failed"); + } + serverThread.join(); + + // rethrow any handshake errors with context + uassertStatusOK(serverStatus); + uassertStatusOK(clientStatus); + } + + struct IngressEgressValidationResult { + StatusWith<SSLPeerInfo> ingress; + StatusWith<SSLPeerInfo> egress; + }; + IngressEgressValidationResult runIngressEgressValidation(); + + class ConnectionContext { + public: + ConnectionContext(int fd, asio::ssl::context& ctx) : io_context() { + asio::ip::tcp::socket socket(io_context, asio::ip::tcp::v4(), fd); + sslSocket = + std::make_unique<asio::ssl::stream<decltype(socket)>>(std::move(socket), ctx, ""); + } + asio::io_context io_context; + std::unique_ptr<asio::ssl::stream<asio::ip::tcp::socket>> sslSocket; + }; + + std::shared_ptr<SSLManagerInterface> clientSSLManager; + std::shared_ptr<SSLManagerInterface> serverSSLManager; + + std::shared_ptr<asio::ssl::context> clientSSLContext; + std::shared_ptr<asio::ssl::context> serverSSLContext; + + std::shared_ptr<ConnectionContext> clientConn; + std::shared_ptr<ConnectionContext> serverConn; +}; + +SSLTestFixture::IngressEgressValidationResult SSLTestFixture::runIngressEgressValidation() { + static const HostAndPort hostForLogging("hostforlogging"); + + // Caller must doHandshake beforehand + invariant(serverConn); + invariant(clientConn); + + IngressEgressValidationResult result{SSLPeerInfo{}, SSLPeerInfo{}}; + + // do ingress (server) first + try { + result.ingress = + serverSSLManager + ->parseAndValidatePeerCertificate(serverConn->sslSocket->native_handle(), + boost::none, + "", + hostForLogging, + nullptr) + .get(); + } catch (const DBException& ex) { + result.ingress = ex.toStatus(); + } + + // do egress (client) next + try { + result.egress = + clientSSLManager + ->parseAndValidatePeerCertificate(clientConn->sslSocket->native_handle(), + boost::none, + "localhost", + hostForLogging, + nullptr) + .get(); + } catch (const DBException& ex) { + result.egress = ex.toStatus(); + } + + return result; +} + +struct CertValidationTestCase { + std::string cafile; + std::string clusterCaFile; + bool pass; + bool allowInvalidCerts{false}; + + void serialize(BSONObjBuilder* bob) const { + bob->append("CAFile", cafile); + bob->append("clusterCAFile", clusterCaFile); + bob->append("expectPass", pass); + bob->append("allowInvalidCerts", allowInvalidCerts); + } +}; + +void checkValidationResults(SSLTestFixture::IngressEgressValidationResult& result, + bool expectIngressPass, + bool expectEgressPass, + ErrorCodes::Error expectIngressCode = ErrorCodes::SSLHandshakeFailed, + ErrorCodes::Error expectEgressCode = ErrorCodes::SSLHandshakeFailed) { + ASSERT_EQ(result.ingress.isOK(), expectIngressPass) + << "Ingress validation status: " << result.ingress.getStatus(); + ASSERT_EQ(result.egress.isOK(), expectEgressPass) + << "Egress validation status: " << result.egress.getStatus(); + if (!result.ingress.isOK()) { + ASSERT_EQ(result.ingress.getStatus().code(), expectIngressCode) + << "Ingress validation status: " << result.ingress.getStatus(); + } + if (!result.egress.isOK()) { + ASSERT_EQ(result.egress.getStatus().code(), expectEgressCode) + << "Egress validation status: " << result.egress.getStatus(); + } +} + +// Tests that validation fails if configured CRL for the issuer of the peer certificate being +// validated has expired. +// Caveats: +// - Apple: CRL unsupported; test disabled +// - Windows: validation fails, but with misleading error message +#if MONGO_CONFIG_SSL_PROVIDER != MONGO_CONFIG_SSL_PROVIDER_APPLE +TEST(SSLManager, expiredCRLTest) { + SSLParams clientParams; + clientParams.sslMode.store(::mongo::sslGlobalParams.SSLMode_requireSSL); + clientParams.sslAllowInvalidHostnames = true; + clientParams.sslCAFile = caFile; + clientParams.sslPEMKeyFile = clientKeyFile; + clientParams.sslCRLFile = expiredCRL; + + SSLParams serverParams; + serverParams.sslMode.store(::mongo::sslGlobalParams.SSLMode_requireSSL); + serverParams.sslAllowInvalidHostnames = true; + serverParams.sslCAFile = caFile; + serverParams.sslPEMKeyFile = serverKeyFile; + serverParams.sslCRLFile = expiredCRL; + + SSLTestFixture tf(serverParams, clientParams); + tf.doHandshake(); + auto result = tf.runIngressEgressValidation(); + checkValidationResults(result, false /*expectIngressPass*/, false /*expectEgressPass*/); + +#if MONGO_CONFIG_SSL_PROVIDER == MONGO_CONFIG_SSL_PROVIDER_WINDOWS + constexpr const char* cause = "revocation server was offline"; +#else + constexpr const char* cause = "expired"; +#endif + ASSERT_NE(result.ingress.getStatus().reason().find(cause), std::string::npos); + ASSERT_NE(result.egress.getStatus().reason().find(cause), std::string::npos); +} + +// Tests basic CRL revocation works on ingress if the client is configured with a revoked key. +// Caveats: +// - Apple: CRL unsupported; test disabled +TEST(SSLManager, basicCRLRevocationTests) { + struct TestCase { + std::string serverCRLFile; + bool serverPass; + void serialize(BSONObjBuilder* bob) const { + bob->append("serverCRLFile", serverCRLFile); + bob->append("serverPass", serverPass); + } + }; + + SSLParams clientParams; + clientParams.sslMode.store(::mongo::sslGlobalParams.SSLMode_requireSSL); + clientParams.sslAllowInvalidHostnames = true; + clientParams.sslCAFile = trustedCaFile; + clientParams.sslPEMKeyFile = revokedClientKeyFile; + + SSLParams serverParams; + serverParams.sslMode.store(::mongo::sslGlobalParams.SSLMode_requireSSL); + serverParams.sslAllowInvalidHostnames = true; + serverParams.sslCAFile = caFile; + serverParams.sslPEMKeyFile = trustedServerKeyFile; + + { + serverParams.sslCRLFile = emptyCRL; + LOGV2(9476702, "Running test case", "CRLFile"_attr = emptyCRL, "pass"_attr = true); + SSLTestFixture tf(serverParams, clientParams); + tf.doHandshake(); + auto result = tf.runIngressEgressValidation(); + checkValidationResults(result, true, true /*expectEgressPass*/); + } + { + serverParams.sslCRLFile = clientRevokedCRL; + LOGV2(9476703, "Running test case", "CRLFile"_attr = clientRevokedCRL, "pass"_attr = false); + SSLTestFixture tf(serverParams, clientParams); + tf.doHandshake(); + auto result = tf.runIngressEgressValidation(); + checkValidationResults(result, false, true /*expectEgressPass*/); + ASSERT_NE(result.ingress.getStatus().reason().find("revoked"), std::string::npos); + } +} + +// Tests whether validation passes if an intermediate CA issuer cert is revoked, but +// the end-entity cert is not. +// Caveats: +// - Apple: CRL unsupported; test disabled +// - Windows: multiple CRLs (root CRL + intermediate CRL) is not allowed +// TODO: backport SERVER-95583 +TEST(SSLManager, revocationWithCRLsIntermediateTests) { + // intermediate-ca-B.pem + intermediate-ca-B-leaf.pem bundle + const std::string intermediateBLeafWithIssuerCertKeyFile = combinePEMFiles( + {{intermediateBLeafKeyFile, true /*includePrivKey*/}, {intermediateBCaFile}}); + // crl_from_intermediate_ca_B.pem + crl_intermediate_ca_B_revoked.pem + const std::string crlsFromRootAndIntermediateB = + combinePEMFiles({{intermediateBRevokedCRL}, {intermediateBCRL}}); + + SSLParams clientParams; + clientParams.sslMode.store(::mongo::sslGlobalParams.SSLMode_requireSSL); + clientParams.sslAllowInvalidHostnames = true; + clientParams.sslCAFile = caFile; + clientParams.sslPEMKeyFile = clientKeyFile; + clientParams.sslCRLFile = crlsFromRootAndIntermediateB; + + SSLParams serverParams; + serverParams.sslMode.store(::mongo::sslGlobalParams.SSLMode_requireSSL); + serverParams.sslAllowInvalidHostnames = true; + serverParams.sslCAFile = caFile; + serverParams.sslPEMKeyFile = intermediateBLeafWithIssuerCertKeyFile; + +#if MONGO_CONFIG_SSL_PROVIDER == MONGO_CONFIG_SSL_PROVIDER_WINDOWS + ASSERT_THROWS_CODE_AND_WHAT( + SSLManagerInterface::create(clientParams, true), + DBException, + ErrorCodes::InvalidSSLConfiguration, + "CertAddCRLContextToStore Failed The object or property already exists."); +#else + SSLTestFixture tf(serverParams, clientParams); + tf.doHandshake(); + auto result = tf.runIngressEgressValidation(); + checkValidationResults(result, true, false); + ASSERT_NE(result.egress.getStatus().reason().find("revoked"), std::string::npos); +#endif +} + +#endif // MONGO_CONFIG_SSL_PROVIDER != MONGO_CONFIG_SSL_PROVIDER_APPLE +#endif // MONGO_CONFIG_SSL } // namespace } // namespace mongo diff --git a/src/mongo/util/net/ssl_manager_windows.cpp b/src/mongo/util/net/ssl_manager_windows.cpp index b88ae85e5b4..7169e1eef1f 100644 --- a/src/mongo/util/net/ssl_manager_windows.cpp +++ b/src/mongo/util/net/ssl_manager_windows.cpp @@ -2062,10 +2062,19 @@ Future<SSLPeerInfo> SSLManagerWindows::parseAndValidatePeerCertificate( return Future<SSLPeerInfo>::makeReady(SSLPeerInfo(sni)); } - LOGV2_DEBUG(23270, - 2, - "Accepted TLS connection from peer: {peerSubjectName}", - "peerSubjectName"_attr = peerSubjectName); + SecPkgContext_CipherInfo cipherInfo; + SECURITY_STATUS ssCipher = QueryContextAttributes(ssl, SECPKG_ATTR_CIPHER_INFO, &cipherInfo); + if (ssCipher != SEC_E_OK) { + return Status(ErrorCodes::SSLHandshakeFailed, + str::stream() + << "QueryContextAttributes for connection info failed with" << ssCipher); + } + const auto cipher = std::wstring(cipherInfo.szCipherSuite); + + LOGV2_INFO(6723802, + "Accepted TLS connection from peer", + "peerSubjectName"_attr = peerSubjectName, + "cipher"_attr = toUtf8String(cipher)); // If this is a server and client and server certificate are the same, log a warning. if (remoteHost.empty() && _sslConfiguration.serverSubjectName() == peerSubjectName) { diff --git a/src/mongo/util/processinfo_linux.cpp b/src/mongo/util/processinfo_linux.cpp index 3357687e6dd..7793011e901 100644 --- a/src/mongo/util/processinfo_linux.cpp +++ b/src/mongo/util/processinfo_linux.cpp @@ -427,7 +427,10 @@ public: /** * Get some details about the CPU */ - static void getCpuInfo(int& procCount, std::string& freq, std::string& features) { + static void getCpuInfo(int& procCount, + std::string& modelString, + std::string& freq, + std::string& features) { procCount = 0; @@ -439,6 +442,7 @@ public: {"features", [&](const std::string& value) { features = value; }}, #else {"processor", [&](const std::string& value) { procCount++; }}, + {"model name", [&](const std::string& value) { modelString = value; }}, {"cpu MHz", [&](const std::string& value) { freq = value; }}, {"flags", [&](const std::string& value) { features = value; }}, #endif @@ -664,6 +668,11 @@ void ProcessInfo::getExtraInfo(BSONObjBuilder& info) { appendNumber("voluntary_context_switches", ru.ru_nvcsw); appendNumber("involuntary_context_switches", ru.ru_nivcsw); + + LinuxProc p(_pid); + + // Append the number of thread in use + appendNumber("threads", p._nlwp); } /** @@ -710,13 +719,13 @@ unsigned long countNumaNodes() { void ProcessInfo::SystemInfo::collectSystemInfo() { utsname unameData; std::string distroName, distroVersion; - std::string cpuFreq, cpuFeatures; + std::string cpuString, cpuFreq, cpuFeatures; int cpuCount; int physicalCores; int cpuSockets; std::string verSig = LinuxSysHelper::readLineFromFile("/proc/version_signature"); - LinuxSysHelper::getCpuInfo(cpuCount, cpuFreq, cpuFeatures); + LinuxSysHelper::getCpuInfo(cpuCount, cpuString, cpuFreq, cpuFeatures); LinuxSysHelper::getNumPhysicalCores(physicalCores); cpuSockets = LinuxSysHelper::getNumCpuSockets(); LinuxSysHelper::getLinuxDistro(distroName, distroVersion); @@ -760,6 +769,7 @@ void ProcessInfo::SystemInfo::collectSystemInfo() { bExtra.append("versionSignature", verSig); bExtra.append("kernelVersion", unameData.release); + bExtra.append("cpuString", cpuString); bExtra.append("cpuFrequencyMHz", cpuFreq); bExtra.append("cpuFeatures", cpuFeatures); bExtra.append("pageSize", static_cast<long long>(pageSize)); diff --git a/src/mongo/util/processinfo_test.cpp b/src/mongo/util/processinfo_test.cpp index 050835f97ac..ab35a6b7374 100644 --- a/src/mongo/util/processinfo_test.cpp +++ b/src/mongo/util/processinfo_test.cpp @@ -33,13 +33,30 @@ #include <iostream> #include <vector> +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" #include "mongo/unittest/unittest.h" #include "mongo/util/processinfo.h" using boost::optional; -using mongo::ProcessInfo; -namespace mongo_test { +namespace mongo { + +namespace { +using StringMap = std::map<std::string, uint64_t>; + +StringMap toStringMap(BSONObj& obj) { + StringMap map; + + for (const auto& e : obj) { + map[e.fieldName()] = e.numberLong(); + } + + return map; +} + +#define ASSERT_KEY(_key) ASSERT_TRUE(stringMap.find(_key) != stringMap.end()); + TEST(ProcessInfo, SysInfoIsInitialized) { ProcessInfo processInfo; if (processInfo.supported()) { @@ -47,6 +64,20 @@ TEST(ProcessInfo, SysInfoIsInitialized) { } } +TEST(FTDCProcSysInfo, TestSysInfo) { + auto sysInfo = ProcessInfo(); + BSONObjBuilder builder; + sysInfo.appendSystemDetails(builder); + + BSONObj obj = builder.obj(); + auto stringMap = toStringMap(obj); + ASSERT_KEY("extra"); + + BSONObj extra = obj.getObjectField("extra"); + stringMap = toStringMap(extra); + ASSERT_KEY("cpuString"); +} + TEST(ProcessInfo, GetNumAvailableCores) { #if defined(__APPLE__) || defined(__linux__) || (defined(__sun) && defined(__SVR4)) || \ defined(_WIN32) @@ -59,4 +90,5 @@ TEST(ProcessInfo, GetNumAvailableCores) { TEST(ProcessInfo, GetNumCoresReturnsNonZeroNumberOfProcessors) { ASSERT_GREATER_THAN(ProcessInfo::getNumCores(), 0u); } -} // namespace mongo_test +} // namespace +} // namespace mongo diff --git a/src/mongo/util/processinfo_windows.cpp b/src/mongo/util/processinfo_windows.cpp index 51068027b51..5f3c0514949 100644 --- a/src/mongo/util/processinfo_windows.cpp +++ b/src/mongo/util/processinfo_windows.cpp @@ -39,6 +39,7 @@ #include "mongo/logv2/log.h" #include "mongo/util/processinfo.h" +#include "mongo/util/text.h" namespace mongo { @@ -248,6 +249,43 @@ bool getFileVersion(const char* filePath, DWORD& fileVersionMS, DWORD& fileVersi return true; } +std::string getCpuString() { + // get descriptive CPU string from registry + HKEY hKey; + LPCWSTR cpuKey = L"HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0"; + LPCWSTR valueName = L"ProcessorNameString"; + std::string cpuString; + + // Open the CPU key in the Windows Registry + if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, cpuKey, 0, KEY_READ, &hKey) == ERROR_SUCCESS) { + ScopeGuard guard([hKey] { RegCloseKey(hKey); }); + WCHAR cpuModel[128]; + DWORD bufferSize = sizeof(cpuModel); + + // Retrieve the value of ProcessorNameString + if (RegQueryValueEx(hKey, + valueName, + nullptr, + nullptr, + reinterpret_cast<LPBYTE>(cpuModel), + &bufferSize) == ERROR_SUCCESS) { + cpuString = toUtf8String(cpuModel); + } else { + auto ec = lastSystemError(); + LOGV2_WARNING(7663101, + "Failed to retrieve CPU model name from the registry", + "error"_attr = errorMessage(ec)); + } + + // Close the registry key + } else { + auto ec = lastSystemError(); + LOGV2_WARNING( + 7663102, "Failed to open CPU key in the registry", "error"_attr = errorMessage(ec)); + } + return cpuString; +} + void ProcessInfo::SystemInfo::collectSystemInfo() { BSONObjBuilder bExtra; std::stringstream verstr; @@ -267,6 +305,12 @@ void ProcessInfo::SystemInfo::collectSystemInfo() { pageSize = static_cast<unsigned long long>(ntsysinfo.dwPageSize); bExtra.append("pageSize", static_cast<long long>(pageSize)); + std::string cpuString = getCpuString(); + if (cpuString != nullptr) { + bExtra.append("cpuString", cpuString); + } + + // get memory info mse.dwLength = sizeof(mse); if (GlobalMemoryStatusEx(&mse)) { diff --git a/src/mongo/util/procparser_test.cpp b/src/mongo/util/procparser_test.cpp index 597c5dde272..259243264ef 100644 --- a/src/mongo/util/procparser_test.cpp +++ b/src/mongo/util/procparser_test.cpp @@ -41,6 +41,7 @@ #include "mongo/bson/bsonobjbuilder.h" #include "mongo/logv2/log.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/processinfo.h" namespace mongo { diff --git a/src/mongo/util/str.h b/src/mongo/util/str.h index 7c364ab9c8d..bebdc4c1d0f 100644 --- a/src/mongo/util/str.h +++ b/src/mongo/util/str.h @@ -46,6 +46,7 @@ #include "mongo/bson/util/builder.h" #include "mongo/platform/bits.h" #include "mongo/util/ctype.h" +#include "mongo/util/str_basic.h" // IWYU pragma: export namespace mongo { namespace str { diff --git a/src/mongo/util/str_basic.h b/src/mongo/util/str_basic.h new file mode 100644 index 00000000000..1c6fb6d352c --- /dev/null +++ b/src/mongo/util/str_basic.h @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +/** + * This header would be part of str.h, but is separated to break an include cycle with + * bson/util/builder.h + */ + +#include <cstring> + +#include "mongo/base/string_data.h" +#include "mongo/util/assert_util.h" + +namespace mongo::str { +/** + * Throws if sd contains any bytes equal to '\0' within its range. + * + * Note: When a StringData is constructed from a C string or std::string, the final '\0' byte is NOT + * considered in range and so will not cause this to throw. + */ +inline void uassertNoEmbeddedNulBytes(StringData sd) { + uassert(9527900, "illegal embedded NUL byte", sd.find('\0') == std::string::npos); +} + +/** + * Copies the contents of sd to dest and appends a NUL byte. + * + * Throws if sd already contains a NUL byte. + * Returns a pointer to the next byte to write to (equivalently, the byte after the appended NUL). + */ +inline char* copyAsCString(char* dest, StringData sd) { + uassertNoEmbeddedNulBytes(sd); + dest += sd.copy(dest, sd.size()); + *dest++ = '\0'; + return dest; +} +} // namespace mongo::str diff --git a/src/mongo/util/str_test.cpp b/src/mongo/util/str_test.cpp index 22b7d81a0e2..74223b76540 100644 --- a/src/mongo/util/str_test.cpp +++ b/src/mongo/util/str_test.cpp @@ -325,4 +325,46 @@ TEST(StringUtilsTest, GetCodePointLength) { } } +TEST(StringUtilsTest, UassertNoEmbeddedNulBytes) { + // These shouldn't throw. + uassertNoEmbeddedNulBytes({nullptr, 0}); + uassertNoEmbeddedNulBytes(""_sd); + uassertNoEmbeddedNulBytes("hello"_sd); + uassertNoEmbeddedNulBytes("hello\0"_sd.substr(0, 5)); + + // These should throw. + ASSERT_THROWS_CODE(uassertNoEmbeddedNulBytes("\0"_sd), DBException, 9527900); + ASSERT_THROWS_CODE(uassertNoEmbeddedNulBytes("\0hello"_sd), DBException, 9527900); + ASSERT_THROWS_CODE(uassertNoEmbeddedNulBytes("hello\0"_sd), DBException, 9527900); + ASSERT_THROWS_CODE(uassertNoEmbeddedNulBytes("hello\0world"_sd), DBException, 9527900); +} + +TEST(StringUtilsTest, CopyAsCString) { + char dest[100]; // big enough for anything we would reasonably add here. + + // Print address not contents on failures. + auto ptr = [](const char* p) { return static_cast<const void*>(p); }; + auto testValid = [&](StringData noNul, int line) { + // Make sure we write a nul byte. Without this, the test could pass if dest happened to have + // uninitialized zero bytes. + std::fill_n(dest, sizeof(dest), 0xff); + + ASSERT_EQ(ptr(copyAsCString(dest, noNul)), ptr(dest + noNul.size() + 1)) << "line:" << line; + ASSERT_EQ(dest[noNul.size()], '\0') << "line:" << line; + ASSERT_EQ(StringData(dest, noNul.size()), noNul) << "line:" << line; + }; + + // These shouldn't throw. + testValid({nullptr, 0}, __LINE__); + testValid(""_sd, __LINE__); + testValid("hello"_sd, __LINE__); + testValid("hello world"_sd.substr(0, 5), __LINE__); + + // These should throw. + ASSERT_THROWS_CODE(copyAsCString(dest, "\0"_sd), DBException, 9527900); + ASSERT_THROWS_CODE(copyAsCString(dest, "\0hello"_sd), DBException, 9527900); + ASSERT_THROWS_CODE(copyAsCString(dest, "hello\0"_sd), DBException, 9527900); + ASSERT_THROWS_CODE(copyAsCString(dest, "hello\0world"_sd), DBException, 9527900); +} + } // namespace mongo::str diff --git a/src/mongo/util/tick_source_bm.cpp b/src/mongo/util/tick_source_bm.cpp new file mode 100644 index 00000000000..ef6381b753d --- /dev/null +++ b/src/mongo/util/tick_source_bm.cpp @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/util/duration.h" +#include "mongo/util/system_tick_source.h" +#include "mongo/util/tick_source.h" +#include <benchmark/benchmark.h> + + +namespace mongo { + +static void BM_getTicks(benchmark::State& state) { + auto tickSource = SystemTickSource::get(); + for (auto _ : state) { + benchmark::DoNotOptimize(tickSource->getTicks()); + } + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_getTicks); + +} // namespace mongo diff --git a/src/mongo/util/uuid.h b/src/mongo/util/uuid.h index 078f658d07c..ce10692d1de 100644 --- a/src/mongo/util/uuid.h +++ b/src/mongo/util/uuid.h @@ -160,6 +160,11 @@ public: return _uuid >= rhs._uuid; } + template <typename H> + friend H AbslHashValue(H h, const UUID& uuid) { + return H::combine(std::move(h), uuid._uuid); + } + /** * Returns true only if the UUID is the RFC 4122 variant, v4 (random). */ diff --git a/src/mongo/watchdog/watchdog_test.cpp b/src/mongo/watchdog/watchdog_test.cpp index 8c3b5c7aab7..73ee6d6d3ab 100644 --- a/src/mongo/watchdog/watchdog_test.cpp +++ b/src/mongo/watchdog/watchdog_test.cpp @@ -493,7 +493,7 @@ TEST_F(WatchdogMonitorTest, PauseAndResume) { counterCheckPtr->setSignalOnCount(counterCheckCount); // Restart the monitor with a different interval. - monitor.setPeriod(Milliseconds(1007)); + monitor.setPeriod(Milliseconds(57)); counterCheckPtr->waitForCount(); // Wait for monitor to run at least once. |