diff options
| author | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-11 15:07:35 -0300 |
|---|---|---|
| committer | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-11 15:07:35 -0300 |
| commit | 4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch) | |
| tree | 1682a647d4463397c119183369ae6f750d5fdcff /src/mongo/bson | |
| parent | aa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff) | |
| parent | 8f0827553e09872941945a093b647a4211a9db7f (diff) | |
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'src/mongo/bson')
| -rw-r--r-- | src/mongo/bson/SConscript | 18 | ||||
| -rw-r--r-- | src/mongo/bson/bson_validate.cpp | 234 | ||||
| -rw-r--r-- | src/mongo/bson/bson_validate.h | 23 | ||||
| -rw-r--r-- | src/mongo/bson/bson_validate.idl | 61 | ||||
| -rw-r--r-- | src/mongo/bson/bson_validate_test.cpp | 280 | ||||
| -rw-r--r-- | src/mongo/bson/bsonelement.cpp | 42 | ||||
| -rw-r--r-- | src/mongo/bson/bsonelement.h | 36 | ||||
| -rw-r--r-- | src/mongo/bson/bsonelement_test.cpp | 69 | ||||
| -rw-r--r-- | src/mongo/bson/bsonobj.cpp | 70 | ||||
| -rw-r--r-- | src/mongo/bson/bsonobj.h | 11 | ||||
| -rw-r--r-- | src/mongo/bson/bsontypes.cpp | 3 | ||||
| -rw-r--r-- | src/mongo/bson/bsontypes.h | 5 | ||||
| -rw-r--r-- | src/mongo/bson/simple_bsonobj_comparator.h | 4 | ||||
| -rw-r--r-- | src/mongo/bson/util/bsoncolumn.cpp | 21 | ||||
| -rw-r--r-- | src/mongo/bson/util/bsoncolumn.h | 7 | ||||
| -rw-r--r-- | src/mongo/bson/util/bsoncolumn_test.cpp | 191 | ||||
| -rw-r--r-- | src/mongo/bson/util/bsoncolumn_util.h | 9 | ||||
| -rw-r--r-- | src/mongo/bson/util/builder.h | 10 | ||||
| -rw-r--r-- | src/mongo/bson/util/simple8b.cpp | 9 | ||||
| -rw-r--r-- | src/mongo/bson/util/simple8b_test.cpp | 34 |
20 files changed, 88 insertions, 1049 deletions
diff --git a/src/mongo/bson/SConscript b/src/mongo/bson/SConscript index 472aed01ad3..d4156225e85 100644 --- a/src/mongo/bson/SConscript +++ b/src/mongo/bson/SConscript @@ -28,8 +28,6 @@ env.CppUnitTest( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/bson/bson_validate', - '$BUILD_DIR/mongo/bson/util/bson_column', ], ) @@ -40,7 +38,6 @@ env.Benchmark( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/bson/bson_validate', ], ) @@ -52,21 +49,6 @@ env.CppLibfuzzerTest( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/bson/bson_validate', - ], -) - -env.Library( - target='bson_validate', - source=[ - 'bson_validate.cpp', - 'bson_validate.idl', - ], - LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/bson/util/bson_column', - '$BUILD_DIR/mongo/idl/idl_parser', - '$BUILD_DIR/mongo/idl/server_parameter', ], ) diff --git a/src/mongo/bson/bson_validate.cpp b/src/mongo/bson/bson_validate.cpp index da4b54ef194..08659efecd1 100644 --- a/src/mongo/bson/bson_validate.cpp +++ b/src/mongo/bson/bson_validate.cpp @@ -35,7 +35,6 @@ #include "mongo/bson/bson_depth.h" #include "mongo/bson/bson_validate.h" #include "mongo/bson/bsonelement.h" -#include "mongo/bson/util/bsoncolumn_util.h" #include "mongo/logv2/log.h" namespace mongo { @@ -80,25 +79,21 @@ static constexpr ValidationStyle kTypeInfoTable alignas(32)[32] = { MONGO_STATIC_ASSERT(sizeof(kTypeInfoTable) == 32); constexpr ErrorCodes::Error InvalidBSON = ErrorCodes::InvalidBSON; -constexpr ErrorCodes::Error NonConformantBSON = ErrorCodes::NonConformantBSON; - -template <bool precise> -Status _doValidateColumn(const char* originalBuffer, - uint64_t maxLength, - ValidationVersion validationVersion); template <bool precise> class ValidateBuffer { public: - ValidateBuffer(const char* data, uint64_t maxLength, ValidationVersion validationVersion) - : _data(data), _maxLength(maxLength), _validationVersion(validationVersion) { + ValidateBuffer(const char* data, uint64_t maxLength) : _data(data), _maxLength(maxLength) { if constexpr (precise) _frames.resize(BSONDepth::getMaxAllowableDepth() + 1); } Status validate() noexcept { try { - setupValidation(); + _currFrame = _frames.begin(); + _currElem = nullptr; + auto maxFrames = BSONDepth::getMaxAllowableDepth() + 1; // A flat BSON has one frame. + uassert(InvalidBSON, "Cannot enforce max nesting depth", _frames.size() <= maxFrames); uassert(InvalidBSON, "BSON data has to be at least 5 bytes", _maxLength >= 5); // Read the length as signed integer, to ensure we limit it to < 2GB. @@ -116,52 +111,9 @@ public: return Status::OK(); } - /* Assumes the root level is a single literal element (which may contain nested objects). - * Only validates up to the termination of that first literal, more data is permitted to - * remain in the buffer after that and is not validated. Throws exception on invalid data. - * Confirm field names for literals in BSONColumn have empty field names. - */ - int validateAndMeasureElem() { - setupValidation(); - uassert(InvalidBSON, - "BSON literal is not followed by fieldname", - _maxLength > 1); // must at least have a 0-terminator after control - // Confirm fieldName is just a null terminator - uassert(NonConformantBSON, - "BSON literal content does not have an empty fieldname", - _maxLength > 1 && _data[1] == 0); - - // Handle one element without using iterative loop, and without expecting - // multiple instances or an EOO. Only resume with the iterative loop if - // we have nested objects - _currElem = _data; - const char* ptr = _validateElem<false>(Cursor{_data + 2, _data + _maxLength}, *_data); - - if (_firstFrameUpdated) { - // We know that type was kObject/kArray/kCodeWScope - // Size is fieldname, type, and a stored int - int64_t size = - static_cast<int64_t>(ConstDataView(_data + 2).read<LittleEndian<int32_t>>()) + 2; - uassert(InvalidBSON, - "BSON literal content exceeds buffer size", - (size_t)size <= _maxLength); - _validateIterative(Cursor{ptr, _data + size}); - return size; - } else { - return ptr - _data; - } - } - private: struct Empty {}; - void inline setupValidation() { - _currFrame = _frames.begin(); - _currElem = nullptr; - auto maxFrames = BSONDepth::getMaxAllowableDepth() + 1; // A flat BSON has one frame. - uassert(InvalidBSON, "Cannot enforce max nesting depth", _frames.size() <= maxFrames); - } - /** * Extra information for each nesting level in the precise validation mode. */ @@ -177,8 +129,6 @@ private: typename std::conditional<precise, std::vector<Frame>, std::array<Frame, 32>>::type; struct Cursor { - /* Also requires remaining buf after the skip (both BSONColumn and BSONObj guarantee this - by having at minimum a trailing EOO) */ void skip(size_t len) { uassert(InvalidBSON, "BSON size is larger than buffer size", (ptr += len) < end); } @@ -213,10 +163,7 @@ private: uassert(ErrorCodes::Overflow, "BSONObj exceeds maximum nested object depth", ++_currFrame != _frames.end()); - return _updateFrame(cursor); - } - const char* _updateFrame(Cursor cursor) { auto obj = cursor.ptr; auto len = cursor.template read<int32_t>(); uassert(ErrorCodes::InvalidBSON, "Nested BSON object has to be at least 5 bytes", len >= 5); @@ -236,22 +183,12 @@ private: return true; } - const char* _validateSpecial(Cursor cursor, uint8_t type) { + static const char* _validateSpecial(Cursor cursor, uint8_t type) { switch (type) { - case BSONType::BinData: { - auto count = cursor.template read<uint32_t>(); - auto subtype = cursor.template read<uint8_t>(); - const char* columnStart = cursor.ptr; - cursor.skip(count); - if (subtype == BinDataType::Column && _validationVersion >= V2_Column) { - /* do not pass down cursor; we want to reset the nesting depth */ - uassert( - NonConformantBSON, - "Invalid BSON column", - _doValidateColumn<precise>(columnStart, count, _validationVersion).isOK()); - } + case BSONType::BinData: + cursor.skip(cursor.template read<uint32_t>()); // Like String, but... + cursor.skip(1); // ...add extra skip for the subtype byte to avoid overflow. break; - } case BSONType::Bool: if (auto value = cursor.template read<uint8_t>()) // If not 0, must be 1. uassert(InvalidBSON, "BSON bool is neither false nor true", value == 1); @@ -275,15 +212,10 @@ private: return cursor.ptr; } - template <bool nestedFrame> const char* _pushCodeWithScope(Cursor cursor) { - // Push a dummy frame to check the CodeWScope size. - if constexpr (nestedFrame) - cursor.ptr = _pushFrame(cursor); - else - cursor.ptr = _updateFrame(cursor); - cursor.skipString(); // Now skip the BSON UTF8 string containing the code. - _currElem = cursor.ptr - 1; // Use the terminating NUL as a dummy scope element. + cursor.ptr = _pushFrame(cursor); // Push a dummy frame to check the CodeWScope size. + cursor.skipString(); // Now skip the BSON UTF8 string containing the code. + _currElem = cursor.ptr - 1; // Use the terminating NUL as adummy scope element. return _pushFrame(cursor); } @@ -297,30 +229,21 @@ private: } } - template <bool nestedFrame> const char* _validateElem(Cursor cursor, uint8_t type) { if (MONGO_unlikely(type > JSTypeMax)) return _validateSpecial(cursor, type); auto style = kTypeInfoTable[type]; - if (MONGO_likely(style <= kSkip16)) { + if (MONGO_likely(style <= kSkip16)) cursor.skip(style * 4); - } else if (MONGO_likely(style == kString)) { + else if (MONGO_likely(style == kString)) cursor.skipString(); - } else if (MONGO_likely(style == kObjectOrArray)) { - if constexpr (nestedFrame) { - cursor.ptr = _pushFrame(cursor); - } else { - cursor.ptr = _updateFrame(cursor); - _firstFrameUpdated = true; - } - } else if (MONGO_unlikely(precise && type == CodeWScope)) { - cursor.ptr = _pushCodeWithScope<nestedFrame>(cursor); - if constexpr (!nestedFrame) - _firstFrameUpdated = true; - } else { + else if (MONGO_likely(style == kObjectOrArray)) + cursor.ptr = _pushFrame(cursor); + else if (MONGO_unlikely(precise && type == CodeWScope)) + cursor.ptr = _pushCodeWithScope(cursor); + else cursor.ptr = _validateSpecial(cursor, type); - } return cursor.ptr; } @@ -334,14 +257,14 @@ private: uint8_t type = *cursor.ptr; _currElem = cursor.ptr; cursor.ptr += len + 1; - cursor.ptr = _validateElem<true>(cursor, type); + cursor.ptr = _validateElem(cursor, type); if constexpr (precise) { // See if the _id field was just validated. If so, set the global scope element. if (_currFrame == _frames.begin() && StringData(_currElem + 1) == "_id"_sd) _currFrame->elem = BSONElement(_currElem); // This is fully validated now. } - dassert(cursor.ptr < cursor.end); + dassert(cursor.ptr <= cursor.end); } // Got the EOO byte: skip it and compare its location with the expected frame end. @@ -376,125 +299,16 @@ private: const char* _currElem = nullptr; // Element to validate: only the name is known to be good. typename Frames::iterator _currFrame; // Frame currently being validated. Frames _frames; // Has end pointers to check and the containing element for precise mode. - bool _firstFrameUpdated = false; // Has the first frame received nested while measuring an elem - ValidationVersion _validationVersion; -}; - -template <bool precise> -class ColumnValidator { -public: - static Status doValidateBSONColumn(const char* originalBuffer, - int maxLength, - ValidationVersion validationVersion) noexcept { - // run control pointer through to end of buffer - // run over literal data as directed by lengths from control - // check formatting of Simple8B blocks - // scan reference objects of interleaved mode starts - // confirm EOO terminations of interleaved modes - // content of interleaved objects does not need to be checked differently from - // standard Simple8B block and literal decodings - // confirm we end at end of buffer - const char* ptr = originalBuffer; - const char* end = originalBuffer + maxLength; - bool interleavedMode = false; - - try { - // Check this beforehand to ensure we cannot overflow the buffer with any strlen - uassert(NonConformantBSON, - "BSON column is missing EOO termination", - ptr < end && *(end - 1) == EOO); - - while (ptr < end) { - uint8_t control = *ptr; - if (control == EOO) { - ptr++; - if (interleavedMode) { - interleavedMode = false; - } else { - // should be the last control of the sequence - uassert(NonConformantBSON, - "BSONColumn EOO does not fully consume buffer", - ptr == end); - return Status::OK(); - } - } else if (bsoncolumn::isUncompressedLiteralControlByte(control)) { - ptr += ValidateBuffer<precise>(ptr, end - ptr, validationVersion) - .validateAndMeasureElem(); - } else if (bsoncolumn::isInterleavedStartControlByte(control)) { - // interleaved objects begin with a reference object, and then a series - // of diff blocks for followup objects, ending with an EOO. Nesting interleaved - // mode is not allowed. - uassert(NonConformantBSON, "Nested interleaved mode", !interleavedMode); - ptr++; - uassert(NonConformantBSON, - "Invalid reference object for interleaved mode", - validateBSON(ptr, end - ptr).isOK()); - // we now know due to validateBSON that it is safe to interpret *ptr - BSONObj reference(ptr); - ptr += reference.objsize(); - interleavedMode = true; - } else { - // Simple8b block sequence, just check for memory overflow of block count - uint8_t numBlocks = bsoncolumn::numSimple8bBlocksForControlByte(control); - int size = sizeof(uint64_t) * numBlocks; - uassert(NonConformantBSON, - "BSONColumn blocks exceed buffer size", - ptr + size + 1 <= end); - ptr += 1 + size; - } - } - } catch (const ExceptionForCat<ErrorCategory::ValidationError>& e) { - return Status(e.code(), str::stream() << e.what()); - } - - // We should not get here for a valid object, the final EOO should have returned OK - return Status(NonConformantBSON, "Missing terminating EOO"); - } }; - -template <bool precise> -Status _doValidateColumn(const char* originalBuffer, - uint64_t maxLength, - ValidationVersion validationVersion) { - if constexpr (precise) { - // First try validating using the fast but less precise version. That version will return - // a not-OK status for objects with CodeWScope or nesting exceeding 32 levels. These cases - // and actual failures will rerun the precise version that gives a detailed error context. - if (MONGO_likely(ColumnValidator<false>::doValidateBSONColumn( - originalBuffer, maxLength, validationVersion) - .isOK())) - return Status::OK(); - - return ColumnValidator<true>::doValidateBSONColumn( - originalBuffer, maxLength, validationVersion); - } else { - return ColumnValidator<false>::doValidateBSONColumn( - originalBuffer, maxLength, validationVersion); - } -} } // namespace -Status validateBSON(const char* originalBuffer, - uint64_t maxLength, - ValidationVersion validationVersion) noexcept { +Status validateBSON(const char* originalBuffer, uint64_t maxLength) noexcept { // First try validating using the fast but less precise version. That version will return // a not-OK status for objects with CodeWScope or nesting exceeding 32 levels. These cases and // actual failures will rerun the precise version that gives a detailed error context. - if (MONGO_likely( - ValidateBuffer<false>(originalBuffer, maxLength, validationVersion).validate().isOK())) + if (MONGO_likely(ValidateBuffer<false>(originalBuffer, maxLength).validate().isOK())) return Status::OK(); - return ValidateBuffer<true>(originalBuffer, maxLength, validationVersion).validate(); + return ValidateBuffer<true>(originalBuffer, maxLength).validate(); } - -Status validateBSON(const BSONObj& obj, ValidationVersion validationVersion) noexcept { - return validateBSON(obj.objdata(), obj.objsize(), validationVersion); -} - -Status validateBSONColumn(const char* originalBuffer, - int maxLength, - ValidationVersion validationVersion) noexcept { - return _doValidateColumn<true>(originalBuffer, maxLength, validationVersion); -} - } // namespace mongo diff --git a/src/mongo/bson/bson_validate.h b/src/mongo/bson/bson_validate.h index add1c18b55b..e169f852754 100644 --- a/src/mongo/bson/bson_validate.h +++ b/src/mongo/bson/bson_validate.h @@ -36,17 +36,6 @@ namespace mongo { -enum ValidationVersion { - /* Original validator */ - V1_Original = 1, - /* Adds validation for the content of Column-typed BinData */ - V2_Column = 2 -}; - -// When adding new versions of BSON validation, update both this and the range and the -// default for the server parameter in src/mongo/bson/bson_validate.idl -static constexpr ValidationVersion currentValidationVersion = V2_Column; - /** * Checks that the buf holds a BSON object as defined in http://bsonspec.org/spec.html. * Note that maxLength is the buffer size, NOT the BSON size. @@ -60,15 +49,5 @@ static constexpr ValidationVersion currentValidationVersion = V2_Column; * validity, code validity, correct length and formatting of binary subtypes, etc. * Length is only limited by the buffer's maxLength and the inherent 2GB - 1 format limitation. */ -Status validateBSON(const char* buf, - uint64_t maxLength, - ValidationVersion validationVersion = currentValidationVersion) noexcept; - -Status validateBSON(const BSONObj& obj, - ValidationVersion validationVersion = currentValidationVersion) noexcept; - -Status validateBSONColumn(const char* buf, - int maxLength, - ValidationVersion validationVersion = currentValidationVersion) noexcept; - +Status validateBSON(const char* buf, uint64_t maxLength) noexcept; } // namespace mongo diff --git a/src/mongo/bson/bson_validate.idl b/src/mongo/bson/bson_validate.idl deleted file mode 100644 index 9d4bb7b9fbe..00000000000 --- a/src/mongo/bson/bson_validate.idl +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (C) 2023-present MongoDB, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the Server Side Public License, version 1, -# as published by MongoDB, Inc. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Server Side Public License for more details. -# -# You should have received a copy of the Server Side Public License -# along with this program. If not, see -# <http://www.mongodb.com/licensing/server-side-public-license>. -# -# As a special exception, the copyright holders give permission to link the -# code of portions of this program with the OpenSSL library under certain -# conditions as described in each individual source file and distribute -# linked combinations including the program with the OpenSSL library. You -# must comply with the Server Side Public License in all respects for -# all of the code used other than as permitted herein. If you modify file(s) -# with this exception, you may extend this exception to your version of the -# file(s), but you are not obligated to do so. If you do not wish to do so, -# delete this exception statement from your version. If you delete this -# exception statement from all source files in the program, then also delete -# it in the license file. -# - -# BSON validate modes - -global: - cpp_namespace: "mongo" - -imports: - - "mongo/idl/basic_types.idl" - -enums: - BSONValidateMode: - description: "The validate mode that dbCheck will run for BSON consistency checks" - type: string - # kDefault: Only fast structural BSON consistency checks - # kExtended: Structural BSON consistency and extra fast checks on BSON specifications. - # kFull: Structural BSON consistency and extra comprehensive checks on BSON specifications. - values: - kDefault: "kDefault" - kExtended: "kExtended" - kFull: "kFull" - -# Server paramaters for validation. -# Range and default value should be kept in sync with src/mongo/bson/bson_validate.h - -server_parameters: - bsonTestValidationVersion: - description: "The version of the BSON validator that will be used to check correctness of data in test environments" - set_at: startup - cpp_vartype: int - cpp_varname: bsonTestValidationVersion - validator: - gte: 1 - lte: 2 - default: 2
\ No newline at end of file diff --git a/src/mongo/bson/bson_validate_test.cpp b/src/mongo/bson/bson_validate_test.cpp index c128da6e268..413efec49b7 100644 --- a/src/mongo/bson/bson_validate_test.cpp +++ b/src/mongo/bson/bson_validate_test.cpp @@ -34,9 +34,6 @@ #include "mongo/base/data_view.h" #include "mongo/bson/bson_depth.h" #include "mongo/bson/bson_validate.h" -#include "mongo/bson/util/bsoncolumn.h" -#include "mongo/bson/util/bsoncolumn_util.h" -#include "mongo/bson/util/bsoncolumnbuilder.h" #include "mongo/db/jsobj.h" #include "mongo/logv2/log.h" #include "mongo/platform/random.h" @@ -58,10 +55,10 @@ void appendInvalidStringElement(const char* fieldName, BufBuilder* bb) { TEST(BSONValidate, Basic) { BSONObj x; - ASSERT_TRUE(validateBSON(x).isOK()); + ASSERT_TRUE(x.valid()); x = BSON("x" << 1); - ASSERT_TRUE(validateBSON(x).isOK()); + ASSERT_TRUE(x.valid()); } TEST(BSONValidate, RandomData) { @@ -87,7 +84,7 @@ TEST(BSONValidate, RandomData) { ASSERT_EQUALS(size, o.objsize()); - if (validateBSON(o).isOK()) { + if (o.valid()) { numValid++; jsonSize += o.jsonString().size(); ASSERT_OK(validateBSON(o.objdata(), o.objsize())); @@ -138,7 +135,7 @@ TEST(BSONValidate, MuckingData1) { data[i] = 0xc8U; numToRun++; - if (validateBSON(mine).isOK()) { + if (mine.valid()) { numValid++; jsonSize += mine.jsonString().size(); ASSERT_OK(validateBSON(mine.objdata(), mine.objsize())); @@ -434,273 +431,4 @@ TEST(BSONValidateFast, MaxNestingDepth) { Status status = validateBSON(tooDeepNesting.objdata(), tooDeepNesting.objsize()); ASSERT_EQ(status.code(), ErrorCodes::Overflow); } - -TEST(BSONValidateFast, ErrorTooShort) { - BSONObj x; - x = BSON("foo" << 17 << "bar" - << "eliot"); - ASSERT_OK(validateBSON(x.objdata(), x.objsize())); - ASSERT_NOT_OK(validateBSON(x.objdata(), x.objsize() - 1)); - // Check if previous byte looks like EOO - char badCopy[16384]; - memcpy(badCopy, x.objdata(), x.objsize() - 1); - badCopy[x.objsize() - 2] = 0; - ASSERT_NOT_OK(validateBSON(badCopy, x.objsize() - 1)); -} - -class BSONValidateColumn : public unittest::Test { -public: - BSONElement objToElement(BSONObj val) { - BSONObjBuilder ob; - ob.append("0"_sd, val); - _elementMemory.emplace_front(ob.obj()); - return _elementMemory.front().firstElement(); - } - -private: - std::forward_list<BSONObj> _elementMemory; -}; - -TEST_F(BSONValidateColumn, BSONColumnInBSON) { - BSONColumnBuilder cb(""); - cb.append(BSON("a" - << "deadbeef") - .getField("a")); - cb.append(BSON("a" << 1).getField("a")); - cb.append(BSON("a" << 2).getField("a")); - cb.append(BSON("a" << 1).getField("a")); - BSONBinData columnData = cb.finalize(); - BSONObj obj = BSON("a" << columnData); - Status status = validateBSON(obj.objdata(), obj.objsize()); - ASSERT_OK(status); - - // Change one important byte. - ((char*)columnData.data)[0] = '0'; - obj = BSON("a" << columnData); - status = validateBSON(obj.objdata(), obj.objsize()); - ASSERT_EQ(status.code(), ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnMissingEOO) { - BSONColumnBuilder cb(""); - - cb.append(BSON("a" - << "deadbeef") - .getField("a")); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); - - // Remove final EOO - ASSERT_EQ(validateBSONColumn((char*)columnData.data, columnData.length - 1).code(), - ErrorCodes::InvalidBSON); - // Remove final EOO and 0 previous byte (check no overflow) - ((char*)columnData.data)[columnData.length - 2] = 0; - ASSERT_EQ(validateBSONColumn((char*)columnData.data, columnData.length - 1).code(), - ErrorCodes::InvalidBSON); -} - -TEST(BSONValidateColumn, BSONColumnFieldnameNotEmpty) { - BSONColumnBuilder cb(""); - cb.append(BSON("a" - << "deadbeef") - .getField("a")); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); - - char buf[1024]; - buf[0] = ((const char*)columnData.data)[0]; - buf[1] = 'f'; - buf[2] = 'o'; - buf[3] = 'o'; - memcpy(buf + 4, ((const char*)columnData.data) + 1, columnData.length - 1); - - ASSERT_EQ(validateBSONColumn(buf, columnData.length + 3).code(), ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnNoOverflowMissingAllEOOInColumn) { - BSONColumnBuilder cb(""); - cb.append(BSON("a" - << "deadbeef") - .getField("a")); - BSONBinData columnData = cb.finalize(); - for (int i = 0; i < columnData.length; ++i) - if (((char*)columnData.data)[i] == 0) - ((char*)columnData.data)[i] = 1; - BSONObj obj = BSON("a" << columnData); - ASSERT_EQ(validateBSON(obj.objdata(), obj.objsize()).code(), ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnTrailingGarbage) { - BSONColumnBuilder cb(""); - cb.append(BSON("a" - << "deadbeef") - .getField("a")); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); - char badData[4096]; - memcpy(badData, columnData.data, columnData.length); - badData[columnData.length] = 1; - - ASSERT_EQ(validateBSONColumn(badData, columnData.length + 1).code(), - ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnNoOverflowBadContent) { - BSONColumnBuilder cb(""); - cb.append(BSON("a" - << "deadbeef") - .getField("a")); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); - - // Remove all string null terminators, expect failure but not overflow - for (int i = 0; i < columnData.length; ++i) - if (((char*)columnData.data)[i] == 0) - ((char*)columnData.data)[i] = 1; - ASSERT_EQ(validateBSONColumn((char*)columnData.data, columnData.length).code(), - ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnNoOverflowMissingFieldname) { - BSONColumnBuilder cb(""); - cb.append(objToElement(BSON("a" - << "deadbeef"))); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); - - ASSERT_EQ(validateBSONColumn((char*)columnData.data, 6 /* start of "a" */).code(), - ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnNoOverflowBadFieldname) { - BSONColumnBuilder cb(""); - cb.append(objToElement(BSON("a" - << "deadbeef"))); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); - - for (int i = 6 /* start of "a" string */; i < columnData.length; ++i) - if (((char*)columnData.data)[i] == 0) - ((char*)columnData.data)[i] = 1; - ASSERT_EQ(validateBSONColumn((char*)columnData.data, columnData.length).code(), - ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnNoOverflowBadLiteral) { - BSONColumnBuilder cb(""); - cb.append(BSON("a" - << "deadbeef") - .getField("a")); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); - - // Remove all string null terminators after string start, expect failure but not overflow - for (int i = 6 /* start of "deadbeef" string */; i < columnData.length; ++i) - if (((char*)columnData.data)[i] == 0) - ((char*)columnData.data)[i] = 1; - ASSERT_EQ(validateBSONColumn((char*)columnData.data, columnData.length).code(), - ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnInterleavedObjectPasses) { - BSONColumnBuilder cb(""); - cb.append(BSON("a" - << "deadbeef") - .getField("a")); - BSONObj subObj1 = BSON("b" - << "inside"); - cb.append(objToElement(subObj1)); - BSONObj subObj2 = BSON("b" - << "outside"); - cb.append(objToElement(subObj2)); - BSONObj subObj3 = BSON("b" - << "gone"); - cb.append(objToElement(subObj3)); - cb.append(BSON("c" - << "foobar") - .getField("c")); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); -} - -TEST_F(BSONValidateColumn, BSONColumnInterleavedNestedObjectPasses) { - BSONColumnBuilder cb(""); - cb.append(BSON("a" - << "deadbeef") - .getField("a")); - BSONObj subObj1 = BSON("d" - << "inside"); - BSONObj subObj2 = BSON("c" << subObj1); - BSONObj subObj3 = BSON("b" << subObj2); - cb.append(objToElement(subObj3)); - cb.append(BSON("c" - << "foobar") - .getField("c")); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); -} - -TEST_F(BSONValidateColumn, BSONColumnInterleavedEmptyObjectPasses) { - BSONColumnBuilder cb(""); - BSONObj subObj1; - cb.append(objToElement(subObj1)); - cb.append(BSON("c" - << "foobar") - .getField("c")); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); -} - -TEST(BSONValidateColumn, BSONColumnInterleavedNestedInterleaved) { - BufBuilder buffer; - BSONObj ref = BSON("c" << 1); - - buffer.appendChar(bsoncolumn::kInterleavedStartControlByteLegacy); - buffer.appendBuf(ref.objdata(), ref.objsize()); - buffer.appendChar(bsoncolumn::kInterleavedStartControlByteLegacy); - buffer.appendBuf(ref.objdata(), ref.objsize()); - buffer.appendChar(0); - buffer.appendChar(0); - - ASSERT_EQ(validateBSONColumn(buffer.buf(), buffer.len()), ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnNoOverflowBlocksShort) { - BSONColumnBuilder cb(""); - for (int i = 0; i < 100; ++i) - cb.append(BSON("a" << i).getField("a")); - - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); - /* Remove EOO and one block */ - ASSERT_EQ(validateBSONColumn((char*)columnData.data, columnData.length - 1 - 8).code(), - ErrorCodes::NonConformantBSON); -} - -TEST_F(BSONValidateColumn, BSONColumnBadExtendedSelector) { - BSONColumnBuilder cb(""); - for (int i = 0; i < 100; ++i) - cb.append(BSON("a" << i).getField("a")); - - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); - /* Change extended selector on a 7 selector to 14 */ - uint64_t block = ConstDataView((char*)columnData.data + 31) /* first 7 selector */ - .read<LittleEndian<uint64_t>>(); - ASSERT_EQ(7, block & 15); // Check that we found a 7 selector - block = (14 << 4) /* 14 extended selector */ - + 7 /* original selector */ - + ((block >> 8) << 8); /* original blocks */ - memcpy((char*)columnData.data + 31, &block, sizeof(block)); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); -} - -TEST(BSONValidateColumn, BSONColumnWithCodeWScope) { - BSONObj obj = BSON("a" << BSONCodeWScope("code", BSON("c" << 1))); - BSONColumnBuilder cb(""); - cb.append(obj.getField("a")); - BSONBinData columnData = cb.finalize(); - ASSERT_OK(validateBSONColumn((char*)columnData.data, columnData.length)); -} - } // namespace diff --git a/src/mongo/bson/bsonelement.cpp b/src/mongo/bson/bsonelement.cpp index ccfb0fd54e5..430b259699b 100644 --- a/src/mongo/bson/bsonelement.cpp +++ b/src/mongo/bson/bsonelement.cpp @@ -46,7 +46,6 @@ #include "mongo/logv2/log.h" #include "mongo/platform/strnlen.h" #include "mongo/util/base64.h" -#include "mongo/util/decimal_counter.h" #include "mongo/util/duration.h" #include "mongo/util/hex.h" #include "mongo/util/scopeguard.h" @@ -445,7 +444,6 @@ int BSONElement::compareElements(const BSONElement& l, */ std::vector<BSONElement> BSONElement::Array() const { chk(mongo::Array); - std::vector<BSONElement> v; BSONObjIterator i(Obj()); while (i.more()) { @@ -466,23 +464,6 @@ std::vector<BSONElement> BSONElement::Array() const { return v; } -std::vector<BSONElement> BSONElement::ArrayVerifyIndexes() const { - chk(mongo::Array); - - std::vector<BSONElement> v; - DecimalCounter<std::uint32_t> counter(0); - for (auto element : Obj()) { - auto fieldName = element.fieldNameStringData(); - uassert(ErrorCodes::BadValue, - fmt::format( - "Invalid array index field name: \"{}\", expected \"{}\"", fieldName, counter), - fieldName == counter); - counter++; - v.push_back(element); - } - return v; -} - int BSONElement::woCompare(const BSONElement& elem, ComparisonRulesSet rules, const StringData::ComparatorInterface* comparator) const { @@ -702,7 +683,7 @@ MONGO_COMPILER_NOINLINE void msgAssertedBadType [[noreturn]] (const char* data) } } // namespace -int BSONElement::computeSize(int8_t type, const char* elem, int fieldNameSize, int bufSize) { +int BSONElement::computeSize(int8_t type, const char* elem, int fieldNameSize) { enum SizeStyle : uint8_t { kFixed, // Total size is a fixed amount + key length. kIntPlusFixed, // Like Fixed, but also add in the int32 immediately following the key. @@ -774,23 +755,10 @@ int BSONElement::computeSize(int8_t type, const char* elem, int fieldNameSize, i // RegEx is two c-strings back-to-back. const char* p = elem + fieldNameSize + 1; - if (bufSize == 0) { - size_t len1 = strlen(p); - p = p + len1 + 1; - size_t len2 = strlen(p); - return (len1 + 1 + len2 + 1) + fieldNameSize + 1; - } else { - int searchSize = bufSize - fieldNameSize - 1; - int len1 = strnlen(p, searchSize); - if (len1 == searchSize) - return -1; - p = p + len1 + 1; - searchSize -= len1 + 1; - int len2 = strnlen(p, searchSize); - if (len2 == searchSize) - return -1; - return (len1 + 1 + len2 + 1) + fieldNameSize + 1; - } + size_t len1 = strlen(p); + p = p + len1 + 1; + size_t len2 = strlen(p); + return (len1 + 1 + len2 + 1) + fieldNameSize + 1; } std::string BSONElement::toString(bool includeFieldName, bool full) const { diff --git a/src/mongo/bson/bsonelement.h b/src/mongo/bson/bsonelement.h index f319c2981fb..237be73a14b 100644 --- a/src/mongo/bson/bsonelement.h +++ b/src/mongo/bson/bsonelement.h @@ -143,14 +143,7 @@ public: bool Bool() const { return chk(mongo::Bool).boolean(); } - std::vector<BSONElement> Array() const; // see implementation for detailed comments - - /** - * Like Array() above, but, if the array keys are not in sequential order or are otherwise - * invalid, an exception is thrown. - */ - std::vector<BSONElement> ArrayVerifyIndexes() const; mongo::OID OID() const { return chk(jstOID).__oid(); } @@ -380,11 +373,6 @@ public: bool isNumber() const; /** - * True if element is a NaN double or decimal. - */ - bool isNaN() const; - - /** * Return double value for this field. MUST be NumberDouble type. */ double _numberDouble() const { @@ -918,12 +906,6 @@ public: static const long long kLargestSafeLongLongAsDouble; static const long long kSmallestSafeLongLongAsDouble; - /** - * Compute the size of the encoding of the BSON object. If bufSize is provided, it will do - * so in an overflow-safe manner. - */ - static int computeSize(int8_t type, const char* data, int fieldNameSize, int bufSize = 0); - private: /** * This is to enable structured bindings for BSONElement, it should not be used explicitly. @@ -981,6 +963,9 @@ private: } return *this; } + + // Only called from constructors. + static int computeSize(int8_t type, const char* data, int fieldNameSize); }; inline bool BSONElement::trueValue() const { @@ -1020,21 +1005,6 @@ inline bool BSONElement::isNumber() const { } } -inline bool BSONElement::isNaN() const { - switch (type()) { - case NumberDouble: { - double d = _numberDouble(); - return std::isnan(d); - } - case NumberDecimal: { - Decimal128 d = _numberDecimal(); - return d.isNaN(); - } - default: - return false; - } -} - inline Decimal128 BSONElement::numberDecimal() const { switch (type()) { case NumberDouble: diff --git a/src/mongo/bson/bsonelement_test.cpp b/src/mongo/bson/bsonelement_test.cpp index 0921823f92f..8a2a7c3fc52 100644 --- a/src/mongo/bson/bsonelement_test.cpp +++ b/src/mongo/bson/bsonelement_test.cpp @@ -283,25 +283,6 @@ TEST(BSONElement, SafeNumberDoubleNegativeBound) { (double)BSONElement::kSmallestSafeLongLongAsDouble); } -TEST(BSONElement, IsNaN) { - ASSERT(BSON("" << std::numeric_limits<double>::quiet_NaN()).firstElement().isNaN()); - ASSERT(BSON("" << -std::numeric_limits<double>::quiet_NaN()).firstElement().isNaN()); - ASSERT(BSON("" << Decimal128::kPositiveNaN).firstElement().isNaN()); - ASSERT(BSON("" << Decimal128::kNegativeNaN).firstElement().isNaN()); - - ASSERT_FALSE(BSON("" << std::numeric_limits<double>::infinity()).firstElement().isNaN()); - ASSERT_FALSE(BSON("" << -std::numeric_limits<double>::infinity()).firstElement().isNaN()); - ASSERT_FALSE(BSON("" << Decimal128::kPositiveInfinity).firstElement().isNaN()); - ASSERT_FALSE(BSON("" << Decimal128::kNegativeInfinity).firstElement().isNaN()); - ASSERT_FALSE(BSON("" << Decimal128{"9223372036854775808.5"}).firstElement().isNaN()); - ASSERT_FALSE(BSON("" << Decimal128{"-9223372036854775809.99"}).firstElement().isNaN()); - ASSERT_FALSE(BSON("" << 12345LL).firstElement().isNaN()); - ASSERT_FALSE(BSON("" - << "foo") - .firstElement() - .isNaN()); -} - TEST(BSONElementIntegerParseTest, ParseIntegerElementToNonNegativeLongRejectsNegative) { BSONObj query = BSON("" << -2LL); ASSERT_NOT_OK(query.firstElement().parseIntegerElementToNonNegativeLong()); @@ -446,55 +427,5 @@ TEST(BSONElementTryCoeceToLongLongTest, CoerceFails) { ASSERT_NOT_OK(result) << " for input document " << testCase.toString(); } } - -TEST(BSONElement, ArrayToVectorFunctionsBehaveCorrectlyWithValidArray) { - // Create a valid array by creating a BSONObj with contiguous array indexes that is then - // passed to the BSONArray ctor. - BSONObj updateArrAsObj = BSON("0" - << "foo" - << "1" - << "bar"); - BSONArray updateArr(updateArrAsObj); - BSONObj parentObj = BSON("arr" << updateArr); - auto arrElem = parentObj.getField("arr"); - - // Both 'Array()' and 'ArrayVerifyIndexes()' will not throw, and instead create vectors of - // size 2. - auto elementVector = arrElem.Array(); - ASSERT_EQ(elementVector.size(), 2); - ASSERT(elementVector[0].binaryEqual(updateArrAsObj.getField("0"))); - ASSERT(elementVector[1].binaryEqual(updateArrAsObj.getField("1"))); - - auto verifiedVector = arrElem.ArrayVerifyIndexes(); - ASSERT_EQ(verifiedVector.size(), 2); - - // The two vectors should have the same elements. - ASSERT(elementVector[0].binaryEqual(verifiedVector[0])); - ASSERT(elementVector[1].binaryEqual(verifiedVector[1])); -} - -TEST(BSONElement, ArrayVerifyIndexesThrowsOnInvalidArrayIndexes) { - // Create our invalid array by creating a BSONObj with non contiguous array indexes that is then - // passed to the BSONArray ctor. - BSONObj updateArrAsObj = BSON("0" - << "foo" - << "2" - << "bar"); - BSONArray updateArr(updateArrAsObj); - BSONObj parentObj = BSON("badArray" << updateArr); - auto arrElem = parentObj.getField("badArray"); - - // The regular 'Array()' will not throw, and instead create an EOO BSONElement at index 1. - auto elementVector = arrElem.Array(); - ASSERT_EQ(elementVector.size(), 3); - - ASSERT(elementVector[0].binaryEqual(updateArrAsObj.getField("0"))); - ASSERT(elementVector[1].binaryEqual(BSONElement())); - ASSERT(elementVector[2].binaryEqual(updateArrAsObj.getField("2"))); - - // 'ArrayVerifyIndexes()', on the other hand, will throw. - ASSERT_THROWS(arrElem.ArrayVerifyIndexes(), ExceptionFor<ErrorCodes::BadValue>); -} - } // namespace } // namespace mongo diff --git a/src/mongo/bson/bsonobj.cpp b/src/mongo/bson/bsonobj.cpp index f0c5946565c..7b5fa7dae04 100644 --- a/src/mongo/bson/bsonobj.cpp +++ b/src/mongo/bson/bsonobj.cpp @@ -138,71 +138,41 @@ BSONObj BSONObj::getOwned(const BSONObj& obj) { return obj.getOwned(); } -BSONObj BSONObj::redact(RedactLevel level, - std::function<std::string(const BSONElement&)> fieldNameRedactor) const { +BSONObj BSONObj::redact(bool onlyEncryptedFields) const { _validateUnownedSize(objsize()); // Helper to get an "internal function" to be able to do recursion struct redactor { - void appendRedactedElem(BSONObjBuilder& builder, - const StringData& fieldNameString, - bool appendMask) { + void appendRedactedElem(BSONObjBuilder& builder, const BSONElement& e, bool appendMask) { if (appendMask) { - builder.append(fieldNameString, "###"_sd); + builder.append(e.fieldNameStringData(), "###"_sd); } else { - builder.appendNull(fieldNameString); + builder.appendNull(e.fieldNameStringData()); } } void operator()(BSONObjBuilder& builder, const BSONObj& obj, bool appendMask, - RedactLevel level, - std::function<std::string(const BSONElement&)> fieldNameRedactor) { + bool onlyEncryptedFields) { for (BSONElement e : obj) { - StringData fieldNameString; - // Temporarily allocated string that must live long enough to be copied by builder. - std::string tempString; - if (!fieldNameRedactor) { - fieldNameString = e.fieldNameStringData(); - } else { - tempString = fieldNameRedactor(e); - fieldNameString = {tempString}; - } if (e.type() == Object) { - BSONObjBuilder subBuilder = builder.subobjStart(fieldNameString); - operator()(subBuilder, e.Obj(), appendMask, level, fieldNameRedactor); + BSONObjBuilder subBuilder = builder.subobjStart(e.fieldNameStringData()); + operator()(subBuilder, e.Obj(), appendMask, onlyEncryptedFields); subBuilder.done(); } else if (e.type() == Array) { - BSONObjBuilder subBuilder = builder.subarrayStart(fieldNameString); - operator()(subBuilder, e.Obj(), appendMask, level, fieldNameRedactor); + BSONObjBuilder subBuilder = builder.subarrayStart(e.fieldNameStringData()); + operator()(subBuilder, e.Obj(), appendMask, onlyEncryptedFields); subBuilder.done(); } else { - // SERVER-79068 Templatizing this could be a good opportunity for performance - // improvements. - switch (level) { - case RedactLevel::all: { - appendRedactedElem(builder, fieldNameString, appendMask); - break; - } - case RedactLevel::encryptedAndSensitive: { - if (e.type() == BinData && - (e.binDataType() == BinDataType::Encrypt || - e.binDataType() == BinDataType::Sensitive)) { - appendRedactedElem(builder, fieldNameString, appendMask); - } else { - builder.append(e); - } - break; - } - case RedactLevel::sensitiveOnly: { - if (e.type() == BinData && e.binDataType() == BinDataType::Sensitive) { - appendRedactedElem(builder, fieldNameString, appendMask); - } else { - builder.append(e); - } - break; + if (onlyEncryptedFields) { + if (e.type() == BinData && e.binDataType() == BinDataType::Encrypt) { + appendRedactedElem(builder, e, appendMask); + } else { + builder.append(e); } + } else { + appendRedactedElem(builder, e, appendMask); } } } @@ -211,7 +181,7 @@ BSONObj BSONObj::redact(RedactLevel level, try { BSONObjBuilder builder; - redactor()(builder, *this, /*appendMask=*/true, level, fieldNameRedactor); + redactor()(builder, *this, /*appendMask=*/true, onlyEncryptedFields); return builder.obj(); } catch (const ExceptionFor<ErrorCodes::BSONObjectTooLarge>&) { } @@ -221,7 +191,7 @@ BSONObj BSONObj::redact(RedactLevel level, // we use BSONType::jstNull, which ensures the redacted object will not be larger than the // original. BSONObjBuilder builder; - redactor()(builder, *this, /*appendMask=*/false, level, fieldNameRedactor); + redactor()(builder, *this, /*appendMask=*/false, onlyEncryptedFields); return builder.obj(); } @@ -328,6 +298,10 @@ BSONObj BSONObj::jsonStringBuffer(JsonStringFormat format, } } +bool BSONObj::valid() const { + return validateBSON(objdata(), objsize()).isOK(); +} + int BSONObj::woCompare(const BSONObj& r, const Ordering& o, ComparisonRulesSet rules, diff --git a/src/mongo/bson/bsonobj.h b/src/mongo/bson/bsonobj.h index 5793f9e5212..a5270adfd25 100644 --- a/src/mongo/bson/bsonobj.h +++ b/src/mongo/bson/bsonobj.h @@ -264,14 +264,10 @@ public: */ BSONObj copy() const; - enum class RedactLevel : int8_t { all, encryptedAndSensitive, sensitiveOnly }; - /** * @return a new full (and owned) redacted copy of the object. */ - BSONObj redact( - RedactLevel level = RedactLevel::all, - std::function<std::string(const BSONElement&)> fieldNameRedactor = nullptr) const; + BSONObj redact(bool onlyEncryptedFields = false) const; /** * Readable representation of a BSON object in an extended JSON-style notation. @@ -642,6 +638,11 @@ public: bool hasFieldNames() const; /** + * Returns true if this object is valid and returns false otherwise. + */ + bool valid() const; + + /** * add all elements of the object to the specified vector */ void elems(std::vector<BSONElement>&) const; diff --git a/src/mongo/bson/bsontypes.cpp b/src/mongo/bson/bsontypes.cpp index 1f9112f2dc4..8be3c5e1d04 100644 --- a/src/mongo/bson/bsontypes.cpp +++ b/src/mongo/bson/bsontypes.cpp @@ -199,8 +199,6 @@ const char* typeName(BinDataType type) { return "encrypt"; case Column: return "column"; - case Sensitive: - return "sensitive"; case bdtCustom: return "Custom"; default: @@ -219,7 +217,6 @@ bool isValidBinDataType(int type) { case Encrypt: case Column: case bdtCustom: - case Sensitive: return true; default: return false; diff --git a/src/mongo/bson/bsontypes.h b/src/mongo/bson/bsontypes.h index 5204f1c3f72..0dc2fe8a4ab 100644 --- a/src/mongo/bson/bsontypes.h +++ b/src/mongo/bson/bsontypes.h @@ -198,9 +198,8 @@ enum BinDataType { bdtUUID = 3, /* deprecated */ newUUID = 4, /* language-independent UUID format across all drivers */ MD5Type = 5, - Encrypt = 6, /* encryption placeholder or encrypted data */ - Column = 7, /* compressed column */ - Sensitive = 8, /* data that should be redacted and protected from unnecessary exposure */ + Encrypt = 6, /* encryption placeholder or encrypted data */ + Column = 7, /* compressed column */ bdtCustom = 128 }; diff --git a/src/mongo/bson/simple_bsonobj_comparator.h b/src/mongo/bson/simple_bsonobj_comparator.h index b073491747f..ed08cbdd939 100644 --- a/src/mongo/bson/simple_bsonobj_comparator.h +++ b/src/mongo/bson/simple_bsonobj_comparator.h @@ -95,10 +95,6 @@ public: }; }; -inline auto simpleHash(const BSONObj& obj) { - return SimpleBSONObjComparator::kInstance.hash(obj); -} - /** * A set of BSONObjs that performs comparisons with simple binary semantics. */ diff --git a/src/mongo/bson/util/bsoncolumn.cpp b/src/mongo/bson/util/bsoncolumn.cpp index df2d74b5b0f..aa81339f9aa 100644 --- a/src/mongo/bson/util/bsoncolumn.cpp +++ b/src/mongo/bson/util/bsoncolumn.cpp @@ -157,11 +157,11 @@ BSONColumn::ElementStorage::ContiguousBlock::~ContiguousBlock() { } } -std::pair<const char*, int> BSONColumn::ElementStorage::ContiguousBlock::done() { +const char* BSONColumn::ElementStorage::ContiguousBlock::done() { auto ptr = _storage.contiguous(); - int size = _storage._endContiguous(); + _storage._endContiguous(); _finished = true; - return std::make_pair(ptr, size); + return ptr; } char* BSONColumn::ElementStorage::allocate(int bytes) { @@ -211,9 +211,8 @@ void BSONColumn::ElementStorage::_beginContiguous() { _contiguousEnabled = true; } -int BSONColumn::ElementStorage::_endContiguous() { +void BSONColumn::ElementStorage::_endContiguous() { _contiguousEnabled = false; - return _pos - _contiguousPos; } BSONColumn::ElementStorage::Element BSONColumn::ElementStorage::allocate(BSONType type, @@ -523,12 +522,12 @@ void BSONColumn::Iterator::_incrementInterleaved() { } // Store built BSONObj in the decompressed list - auto [objdata, objsize] = contiguous.done(); - BSONElement obj; + const char* objdata = contiguous.done(); + BSONElement obj(objdata); - // If no data was added, use a EOO literal. As buffer size is 0 we cannot interpret it as BSON. - if (objsize > 0) { - obj = BSONElement(objdata); + // If no data was added, use a EOO literal instead of an empty object. + if (obj.objsize() == 0) { + obj = BSONElement(); } _column->_decompressed.emplace_back(obj); @@ -541,7 +540,7 @@ void BSONColumn::Iterator::_handleEOO() { } bool BSONColumn::Iterator::_isLiteral(char control) { - return (control & 0xE0) == 0 || control == (char)MinKey || control == (char)MaxKey; + return (control & 0xE0) == 0; } bool BSONColumn::Iterator::_isInterleavedStart(char control) { diff --git a/src/mongo/bson/util/bsoncolumn.h b/src/mongo/bson/util/bsoncolumn.h index 9bfee3e76cf..23d4b19c7b5 100644 --- a/src/mongo/bson/util/bsoncolumn.h +++ b/src/mongo/bson/util/bsoncolumn.h @@ -307,8 +307,7 @@ private: ContiguousBlock(ElementStorage& storage); ~ContiguousBlock(); - // Return pointer to contigous block and the block size - std::pair<const char*, int> done(); + const char* done(); private: ElementStorage& _storage; @@ -360,8 +359,8 @@ private: // Starts contiguous mode void _beginContiguous(); - // Ends contiguous mode, returns size of block - int _endContiguous(); + // Ends contiguous mode + void _endContiguous(); // Full memory blocks that are kept alive. std::vector<std::unique_ptr<char[]>> _blocks; diff --git a/src/mongo/bson/util/bsoncolumn_test.cpp b/src/mongo/bson/util/bsoncolumn_test.cpp index a2ef0b2feae..0fe621fbd30 100644 --- a/src/mongo/bson/util/bsoncolumn_test.cpp +++ b/src/mongo/bson/util/bsoncolumn_test.cpp @@ -4967,43 +4967,6 @@ TEST_F(BSONColumnTest, ObjectWithOnlyEmptyObjsDoesNotStartInterleavingFromAppend test(true, appendInterleavedStart); } -TEST_F(BSONColumnTest, InterleavedFullSkipAfterObjectSkip) { - BSONColumnBuilder cb("test"_sd, true); - - // This test makes sure we're not leaking the skip from the 'yyyyyy' field into the next - // measurement. 'yyyyyy' will be written into the buffer for the second item before we realize - // that it only contain skips. We must not attempt to interpret this memory when the next - // measurement is all skips. - std::vector<BSONElement> elems = { - createElementObj(BSON("x" << 1 << "yyyyyy" << BSON("z" << 2))), - createElementObj(BSON("x" << 1)), - BSONElement()}; - - for (auto elem : elems) { - if (!elem.eoo()) - cb.append(elem); - else - cb.skip(); - } - - BufBuilder expected; - appendInterleavedStart(expected, elems.front().Obj()); - appendSimple8bControl(expected, 0b1000, 0b0000); - appendSimple8bBlocks64(expected, - {kDeltaForBinaryEqualValues, - deltaInt32(elems[1].Obj()["x"_sd], elems[0].Obj()["x"_sd]), - boost::none}, - 1); - appendSimple8bControl(expected, 0b1000, 0b0000); - appendSimple8bBlocks64(expected, {kDeltaForBinaryEqualValues, boost::none, boost::none}, 1); - appendEOO(expected); - appendEOO(expected); - - auto binData = cb.finalize(); - verifyBinary(binData, expected); - verifyDecompression(binData, elems); -} - TEST_F(BSONColumnTest, NonZeroRLEInFirstBlockAfterSimple8bBlocks) { BSONColumnBuilder cb("test"_sd); @@ -5337,160 +5300,6 @@ TEST_F(BSONColumnTest, AppendMinKeyInSubObjAfterMerge) { cb.append(createElementObj(obj.obj())), DBException, ErrorCodes::InvalidBSONType); } -TEST_F(BSONColumnTest, DecompressMinKey) { - BufBuilder expected; - appendLiteral(expected, createElementMinKey()); - appendEOO(expected); - - BSONBinData binData; - binData.data = expected.buf(); - binData.length = expected.len(); - binData.type = Column; - - verifyDecompression(binData, {createElementMinKey()}); -} - -TEST_F(BSONColumnTest, DecompressMaxKey) { - BufBuilder expected; - appendLiteral(expected, createElementMaxKey()); - appendEOO(expected); - - BSONBinData binData; - binData.data = expected.buf(); - binData.length = expected.len(); - binData.type = Column; - - verifyDecompression(binData, {createElementMaxKey()}); -} - -TEST_F(BSONColumnTest, DecompressMinKeyInSubObj) { - BSONObjBuilder obj; - { - BSONObjBuilder builder = obj.subobjStart("root"); - builder.append(createElementMinKey()); - } - - BSONObj ref = obj.obj(); - - BufBuilder expected; - appendInterleavedStart(expected, ref); - appendSimple8bControl(expected, 0b1000, 0b0000); - appendSimple8bBlocks64(expected, {kDeltaForBinaryEqualValues}, 1); - appendEOO(expected); - appendEOO(expected); - - BSONBinData binData; - binData.data = expected.buf(); - binData.length = expected.len(); - binData.type = Column; - - verifyDecompression(binData, {createElementObj(ref)}); -} - -TEST_F(BSONColumnTest, DecompressMinKeyInSubObjAfterInterleaveStart) { - - BSONObjBuilder obj; - { - BSONObjBuilder builder = obj.subobjStart("root"); - builder.append(createElementMinKey()); - } - - std::vector<BSONElement> elems = {createElementObj(BSON("root" << BSON("0" << 1))), - createElementObj(obj.obj())}; - - BufBuilder expected; - appendInterleavedStart(expected, elems[0].Obj()); - appendSimple8bControl(expected, 0b1000, 0b0000); - appendSimple8bBlocks64(expected, {kDeltaForBinaryEqualValues}, 1); - appendLiteral(expected, createElementMinKey()); - appendEOO(expected); - appendEOO(expected); - - BSONBinData binData; - binData.data = expected.buf(); - binData.length = expected.len(); - binData.type = Column; - - verifyDecompression(binData, elems); -} - -TEST_F(BSONColumnTest, DecompressMinKeyInSubObjAfterInterleaveStartInAppendMode) { - BSONObjBuilder obj; - { - BSONObjBuilder builder = obj.subobjStart("root"); - builder.append(createElementMinKey()); - } - - std::vector<BSONElement> elems(7, createElementObj(BSON("root" << BSON("0" << 1)))); - elems.push_back(createElementObj(obj.obj())); - - BufBuilder expected; - appendInterleavedStart(expected, elems[0].Obj()); - appendSimple8bControl(expected, 0b1000, 0b0000); - appendSimple8bBlocks64(expected, - {kDeltaForBinaryEqualValues, - kDeltaForBinaryEqualValues, - kDeltaForBinaryEqualValues, - kDeltaForBinaryEqualValues, - kDeltaForBinaryEqualValues, - kDeltaForBinaryEqualValues, - kDeltaForBinaryEqualValues}, - 1); - appendLiteral(expected, createElementMinKey()); - appendEOO(expected); - appendEOO(expected); - - BSONBinData binData; - binData.data = expected.buf(); - binData.length = expected.len(); - binData.type = Column; - - verifyDecompression(binData, elems); -} - -TEST_F(BSONColumnTest, DecompressMinKeyInSubObjAfterMerge) { - BSONObjBuilder obj; - { - BSONObjBuilder builder = obj.subobjStart("root"); - builder.append("a", "asd"); - builder.append(createElementMinKey()); - } - - // Make sure we handle MinKey even if we would detect that "a" needs to be merged before - // observing the MinKey. - std::vector<BSONElement> elems = {createElementObj(BSON("root" << BSON("0" << 1))), - createElementObj(obj.obj())}; - - BufBuilder expected; - appendInterleavedStart(expected, - BSON("root" << BSON("a" - << "asd" - << "0" << 1))); - appendSimple8bControl(expected, 0b1000, 0b0000); - appendSimple8bBlocks64(expected, - { - boost::none, - kDeltaForBinaryEqualValues, - }, - 1); - appendSimple8bControl(expected, 0b1000, 0b0000); - appendSimple8bBlocks64(expected, - { - kDeltaForBinaryEqualValues, - }, - 1); - appendLiteral(expected, createElementMinKey()); - appendEOO(expected); - appendEOO(expected); - - BSONBinData binData; - binData.data = expected.buf(); - binData.length = expected.len(); - binData.type = Column; - - verifyDecompression(binData, elems); -} - // The large literal emits this on Visual Studio: Fatal error C1091: compiler limit: string exceeds // 65535 bytes in length #if !defined(_MSC_VER) || _MSC_VER >= 1929 diff --git a/src/mongo/bson/util/bsoncolumn_util.h b/src/mongo/bson/util/bsoncolumn_util.h index 5b3110a13c9..9d9eec07e8e 100644 --- a/src/mongo/bson/util/bsoncolumn_util.h +++ b/src/mongo/bson/util/bsoncolumn_util.h @@ -37,15 +37,6 @@ static constexpr char kInterleavedStartControlByteLegacy = (char)0xF0; static constexpr char kInterleavedStartControlByte = (char)0xF1; static constexpr char kInterleavedStartArrayRootControlByte = (char)0xF2; -inline bool isUncompressedLiteralControlByte(uint8_t control) { - return (control & 0xE0) == 0 || control == (uint8_t)MinKey || control == (uint8_t)MaxKey; -} - -inline bool isInterleavedStartControlByte(char control) { - return control == kInterleavedStartControlByteLegacy || - control == kInterleavedStartControlByte || control == kInterleavedStartArrayRootControlByte; -} - inline bool isLiteralControlByte(char control) { return (control & 0xE0) == 0; } diff --git a/src/mongo/bson/util/builder.h b/src/mongo/bson/util/builder.h index c3fc7facf26..5d7d2d5673c 100644 --- a/src/mongo/bson/util/builder.h +++ b/src/mongo/bson/util/builder.h @@ -325,7 +325,7 @@ public: @return point to region that was skipped. pointer may change later (on realloc), so for immediate use only */ - char* skip(size_t n) { + char* skip(int n) { return grow(n); } @@ -394,7 +394,7 @@ public: } void appendBuf(const void* src, size_t len) { if (len) - memcpy(grow(len), src, len); + memcpy(grow((int)len), src, len); } template <class T> @@ -403,7 +403,7 @@ public: } void appendStr(StringData str, bool includeEndingNull = true) { - const size_t len = str.size() + (includeEndingNull ? 1 : 0); + const int len = str.size() + (includeEndingNull ? 1 : 0); str.copyTo(grow(len), includeEndingNull); } @@ -423,8 +423,8 @@ public: } /* returns the pre-grow write position */ - char* grow(size_t by) { - if (MONGO_likely(by <= static_cast<size_t>(_end - _nextByte))) { + char* grow(int by) { + if (MONGO_likely(by <= _end - _nextByte)) { char* oldNextByte = _nextByte; _nextByte += by; return oldNextByte; diff --git a/src/mongo/bson/util/simple8b.cpp b/src/mongo/bson/util/simple8b.cpp index 264e999b8a2..2efa9484cd3 100644 --- a/src/mongo/bson/util/simple8b.cpp +++ b/src/mongo/bson/util/simple8b.cpp @@ -130,9 +130,9 @@ constexpr std::array<uint8_t, 15> kBaseSelectorToShiftSize = { // Transfer from a selector to a specific extension type // This is for selector 7 and 8 extensions where the selector value is passed along with // selector index. -constexpr std::array<std::array<uint8_t, 16>, 2> kSelectorToExtension = { - std::array<uint8_t, 16>{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0}, - std::array<uint8_t, 16>{0, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 0, 0}}; +constexpr std::array<std::array<uint8_t, 14>, 2> kSelectorToExtension = { + std::array<uint8_t, 14>{0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + std::array<uint8_t, 14>{0, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3}}; // Transfer from a extensionType and selectorIdx to the selector value to be held in the 4 lsb (base // selector) @@ -614,9 +614,6 @@ void Simple8bBuilder<T>::_handleRleTermination() { } --_rleCount; } - - // Reset which selectors are possible to use for next word - isSelectorPossible.fill(true); } template <typename T> diff --git a/src/mongo/bson/util/simple8b_test.cpp b/src/mongo/bson/util/simple8b_test.cpp index df8c849f73b..6ef1866f855 100644 --- a/src/mongo/bson/util/simple8b_test.cpp +++ b/src/mongo/bson/util/simple8b_test.cpp @@ -1407,37 +1407,3 @@ TEST(Simple8b, ValueTooLargeBitCountUsedForExtendedSelectors) { }); ASSERT_FALSE(builder.append(value)); } - -TEST(Simple8b, ResetRLEAfterLargeValue) { - uint8_t kRleMultiplier = 120; - uint8_t kBaseSelectorMask = 0x000000000000000F; - uint8_t kRleSelector = 15; - // Large value that can be only be stored in the extended selectors that encodes a bit shift - uint64_t large = 0xC000000000000000; - - BufBuilder buf; - Simple8bBuilder<uint64_t> b([&buf](uint64_t simple8bBlock) { - buf.appendNum(simple8bBlock); - return true; - }); - - // Write as many of these large values we need to ensure a non-RLE block is written followed by - // an RLE block. - for (int i = 0; i < kRleMultiplier + 7; ++i) { - ASSERT_TRUE(b.append(large)); - } - - // Add a large value that can only fit in the base selector which can encode up to 60 meaningful - // bits. When terminating RLE we should completely reset to allow this value to be appended. - ASSERT_TRUE(b.append(0x07FFFFFFFFFFFFFF)); - - b.flush(); - auto size = buf.len(); - auto data = buf.release(); - - // The second block should be an RLE block - ASSERT_GT(size, 16); - uint64_t secondBlock = - ConstDataView(data.get() + sizeof(uint64_t)).read<LittleEndian<uint64_t>>(); - ASSERT_TRUE((secondBlock & kBaseSelectorMask) == kRleSelector); -} |
