diff options
Diffstat (limited to 'src/mongo/db/query/query_shape')
20 files changed, 3630 insertions, 0 deletions
diff --git a/src/mongo/db/query/query_shape/README.md b/src/mongo/db/query/query_shape/README.md new file mode 100644 index 00000000000..b3c02d28e69 --- /dev/null +++ b/src/mongo/db/query/query_shape/README.md @@ -0,0 +1,69 @@ +# Query Shape +A query shape is a transformed version of a command with literal values replaced by a "canonical" +BSON Type placeholder. Hence, different instances of a command would be considered to have the same +query shape if they are identical once their literal values are abstracted. + +For example, these two queries would have the same shape: +```js +db.example.findOne({x: 24}); +db.example.findOne({x: 53}); +``` +While these queries would each have a distinct shape: +```js +db.example.findOne({x: 53, y: 1}); +db.example.findOne({x: 53}); +db.example.findOne({x: "string"}); +``` +While different literal _values_ result in the same shape (matching `x` for 23 vs 53), different +BSON _types_ of the literal are considered distinct shapes (matching `x` for 53 vs "string"). + +The concept of a query shape exists not just for the find command, but for many of the CRUD commands +and aggregate. It also includes most (but not all) components of these commands, not just the query +predicate (MatchExpresssion). In these ways, "query" is meant more generally. While some components +included in the query shape are shared across the different types of commands (e.g., the "hint" +field), some are unique. For example, a find command would include a `filter` while an aggregate +command would have a `pipeline`. + +You can see which components are considered part of the query shape or not for each specific shape +type in their respective "shape component" classes, whose purpose is to determine which components +are relevant and should be included for determining the shape for specific type of command. The +structure is as follows: +- [`CmdSpecificShapeComponents`](query_shape.h#L65) + - [`LetShapeComponent`](cmd_with_let_shape.h#L48) + - [`AggCmdShapeComponents`](agg_cmd_shape.h#L82) + - [`FindCmdShapeComponents`](find_cmd_shape.h#L48) + +See more information for the different shapes in their respective classes, structured as follows: +- [`Shape`](query_shape.h) + - [`CmdWithLetShape`](cmd_with_let_shape.h) + - [`AggCmdShape`](agg_cmd_shape.h) + - [`FindCmdShape`](find_cmd_shape.h) + +## Serialization Options +`SerializationOptions` describes the way we serialize literal values. + +There are 3 different serialization options: +- `kUnchanged`: literals are serialized unmodified + - `{x: 5, y: "hello"}` -> `{x: 5, y: "hello"}` +- `kToDebugTypeString`: human readable format, type string of the literal is serialized + - `{x: 5, y: "hello"}` -> `{x: "?number", y: "?string"}` +- `kToRepresentativeParseableValue`: literal serialized to one canonical value for given type, which + must be parseable + - `{x: 5, y: "hello"}` -> `{x: 1, y: "?"}` + - An example of a query which is serialized differently due to the parseable requirement is `{x: + {$regex: "^p.*"}}`. If we serialized the pattern as if it were a normal string we would end up + with `{x: {$regex: "?"}}` however `"?"` is not a valid regex pattern, so this would fail + parsing. Instead we will serialize it this way to maintain parseability, `{x: {$regex: + "\\?"}}`, since `"\\?"` is valid regex. + +See [serialization_options.h](serialization_options.h) for more details. + +When we compute the [query shape hash](query_shape.cpp#L99-107), we use the +`kToRepresentativeParseableValue`, since all literals of the same type will become the same value. +This allows us to group together queries that have the same structure but different literal values +into the same shape, since they will result in the same hash. The term we use to refer to this is +"shapify", as we simplify the queries into their query shape. + +When shapifying, we try to get as close as possible to the original user input, but there are some +stages like `$jsonSchema` and `$setWindowFields` that output "internal" stages that are already +transformed from user input. diff --git a/src/mongo/db/query/query_shape/SConscript b/src/mongo/db/query/query_shape/SConscript new file mode 100644 index 00000000000..d4bddba4934 --- /dev/null +++ b/src/mongo/db/query/query_shape/SConscript @@ -0,0 +1,42 @@ +# -*- mode: python -*- + +Import([ + "env", + "get_option", +]) + +env = env.Clone() + +env.Library( + target='query_shape', source=['query_shape.cpp', 'shape_helpers.cpp'], LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/exec/document_value/document_value', + '$BUILD_DIR/mongo/db/pipeline/field_path', + 'query_shape_common', + ], LIBDEPS_PRIVATE=[ + ]) + +env.Library( + target='query_shape_common', source=[ + 'query_shape.idl', + 'serialization_options.cpp', + ], LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/exec/document_value/document_value', + '$BUILD_DIR/mongo/db/pipeline/field_path', + ], LIBDEPS_PRIVATE=[ + ]) + +env.CppUnitTest( + target="db_query_query_shape_test", + source=[ + "query_shape_test.cpp", + "query_shape_test.idl", + ], + LIBDEPS=[ + "$BUILD_DIR/mongo/db/auth/authmocks", + "$BUILD_DIR/mongo/db/query/query_test_service_context", + "$BUILD_DIR/mongo/db/service_context_d_test_fixture", + "query_shape", + ], +) diff --git a/src/mongo/db/query/query_shape/agg_cmd_shape.cpp b/src/mongo/db/query/query_shape/agg_cmd_shape.cpp new file mode 100644 index 00000000000..e997150ecc6 --- /dev/null +++ b/src/mongo/db/query/query_shape/agg_cmd_shape.cpp @@ -0,0 +1,125 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/agg_cmd_shape.h" + +#include "mongo/db/query/query_shape/shape_helpers.h" + +namespace mongo::query_shape { + +AggCmdShapeComponents::AggCmdShapeComponents( + const AggregateCommandRequest& aggRequest, + stdx::unordered_set<NamespaceString> involvedNamespaces_, + std::vector<BSONObj> pipeline) + : allowDiskUse(aggRequest.getAllowDiskUse()), + involvedNamespaces(std::move(involvedNamespaces_)), + representativePipeline(std::move(pipeline)) {} + +AggCmdShapeComponents::AggCmdShapeComponents( + OptionalBool allowDiskUse, + stdx::unordered_set<NamespaceString> involvedNamespaces_, + std::vector<BSONObj> pipeline) + : allowDiskUse(allowDiskUse), + involvedNamespaces(std::move(involvedNamespaces_)), + representativePipeline(std::move(pipeline)) {} + +void AggCmdShapeComponents::HashValue(absl::HashState state) const { + state = absl::HashState::combine(std::move(state), allowDiskUse); + for (auto&& shapifiedStage : representativePipeline) { + state = absl::HashState::combine(std::move(state), simpleHash(shapifiedStage)); + } +} + +void AggCmdShape::appendLetCmdSpecificShapeComponents( + BSONObjBuilder& bob, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts) const { + tassert(7633000, + "We don't support serializing to the unmodified shape here, since we have already " + "shapified and stored the representative query - we've lost the original literals", + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged); + + if (opts == SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + // We have this copy stored already! + return _components.appendTo(bob); + } else { + // The cached pipeline shape doesn't match the requested options, so we have to + // re-parse the pipeline from the initial request. + expCtx->inMongos = _inMongos; + expCtx->addResolvedNamespaces(_components.involvedNamespaces); + auto reparsed = Pipeline::parse(_components.representativePipeline, expCtx); + auto serializedPipeline = reparsed->serializeToBson(opts); + AggCmdShapeComponents{ + _components.allowDiskUse, _components.involvedNamespaces, serializedPipeline} + .appendTo(bob); + } +} + +void AggCmdShapeComponents::appendTo(BSONObjBuilder& bob) const { + bob.append("command", "aggregate"); + + // pipeline + bob.append(AggregateCommandRequest::kPipelineFieldName, representativePipeline); + + // allowDiskUse + if (allowDiskUse.has_value()) { + bob.append(AggregateCommandRequest::kAllowDiskUseFieldName, bool(allowDiskUse)); + } +} + +// As part of the size, we must track the allocation of elements in the representative +// pipeline, as well as the elements in the unordered set of involved namespaces. +size_t AggCmdShapeComponents::size() const { + return sizeof(AggCmdShapeComponents) + shape_helpers::containerSize(representativePipeline) + + shape_helpers::containerSize(involvedNamespaces); +} + +AggCmdShape::AggCmdShape(const AggregateCommandRequest& aggregateCommand, + NamespaceString origNss, + stdx::unordered_set<NamespaceString> involvedNamespaces_, + const Pipeline& pipeline, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : CmdWithLetShape(aggregateCommand.getLet(), + expCtx, + _components, + std::move(origNss), + aggregateCommand.getCollation().value_or(BSONObj())), + _components(aggregateCommand, + std::move(involvedNamespaces_), + pipeline.serializeToBson( + SerializationOptions::kRepresentativeQueryShapeSerializeOptions)), + _inMongos(expCtx->inMongos) {} + +size_t AggCmdShape::extraSize() const { + // To account for possible padding, we calculate the extra space with the difference instead of + // using sizeof(bool); + return sizeof(AggCmdShape) - sizeof(CmdWithLetShape) - sizeof(AggCmdShapeComponents); +} + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/agg_cmd_shape.h b/src/mongo/db/query/query_shape/agg_cmd_shape.h new file mode 100644 index 00000000000..c0ef5a7b06f --- /dev/null +++ b/src/mongo/db/query/query_shape/agg_cmd_shape.h @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <boost/intrusive_ptr.hpp> + +#include "mongo/db/pipeline/aggregate_command_gen.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/pipeline/pipeline.h" +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" +#include "mongo/db/query/query_shape/query_shape.h" + +namespace mongo::query_shape { + +/** + * A struct representing the aggregate command's specific components that are to be considered part + * of the query shape. + * + * This struct stores the shapified version of the pipeline as a memory optimization. We'll need to + * store the BSON version in either case, since often the parsed version needs that BSON to survive + * as backing memory, so we store the representative pipeline shape so that we are able to parse the + * pipeline again if we need to compute a different shape. + */ +struct AggCmdShapeComponents : public query_shape::CmdSpecificShapeComponents { + AggCmdShapeComponents(const AggregateCommandRequest&, + stdx::unordered_set<NamespaceString> involvedNamespaces, + std::vector<BSONObj> shapifiedPipeline); + + AggCmdShapeComponents(OptionalBool allowDiskUse, + stdx::unordered_set<NamespaceString> involvedNamespaces, + std::vector<BSONObj> shapifiedPipeline); + + size_t size() const final; + + void appendTo(BSONObjBuilder&) const; + + void HashValue(absl::HashState state) const final; + + OptionalBool allowDiskUse; + + stdx::unordered_set<NamespaceString> involvedNamespaces; + + // The representative query shape of the pipeline. + std::vector<BSONObj> representativePipeline; +}; + +/** + * A class representing the query shape of an aggregate command. The components are listed above. + * This class knows how to utilize those components to serialize to BSON with any + * SerializationOptions. Mostly this involves correctly setting up an ExpressionContext to re-parse + * the request if needed. + */ +class AggCmdShape : public CmdWithLetShape { +public: + AggCmdShape(const AggregateCommandRequest&, + NamespaceString origNss, + stdx::unordered_set<NamespaceString> involvedNamespaces, + const Pipeline&, + const boost::intrusive_ptr<ExpressionContext>&); + + void appendLetCmdSpecificShapeComponents(BSONObjBuilder& bob, + const boost::intrusive_ptr<ExpressionContext>&, + const SerializationOptions&) const final; + size_t extraSize() const final override; + +private: + AggCmdShapeComponents _components; + // Flag to denote if the query was run on mongos. Needed to rebuild the "dummy" expression + // context for re-parsing. + bool _inMongos; +}; +static_assert(sizeof(AggCmdShape) <= + sizeof(CmdWithLetShape) + sizeof(AggCmdShapeComponents) + 8 /* bool and padding*/, + "If the class' members have changed, this assert and the extraSize() calculation may " + "need to be updated with a new value."); +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp b/src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp new file mode 100644 index 00000000000..c617391f4e7 --- /dev/null +++ b/src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp @@ -0,0 +1,266 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/bson/json.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_shape/agg_cmd_shape.h" +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" +#include "mongo/db/query/query_test_service_context.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_shape { + +namespace { +static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl"); + +class AggCmdShapeTest : public unittest::Test { +public: + void setUp() final { + _queryTestServiceContext = std::make_unique<QueryTestServiceContext>(); + _operationContext = _queryTestServiceContext->makeOperationContext(); + _expCtx = make_intrusive<ExpressionContextForTest>(); + } + + std::unique_ptr<AggregateCommandRequest> makeAggregateCommandRequest( + std::vector<StringData> stagesJson, + boost::optional<StringData> letJson = boost::none, + boost::optional<StringData> collationJson = boost::none) { + std::vector<BSONObj> pipeline; + for (auto&& stage : stagesJson) { + pipeline.push_back(fromjson(stage.rawData())); + } + + auto aggRequest = + std::make_unique<AggregateCommandRequest>(kDefaultTestNss, std::move(pipeline)); + if (letJson) { + aggRequest->setLet(fromjson(letJson->rawData())); + } + if (collationJson) { + aggRequest->setCollation(fromjson(collationJson->rawData())); + } + return aggRequest; + } + + std::unique_ptr<AggCmdShape> makeShapeFromPipeline( + std::vector<StringData> stagesJson, + boost::optional<StringData> letJson = boost::none, + boost::optional<StringData> collationJson = boost::none) { + + auto aggRequest = makeAggregateCommandRequest( + std::move(stagesJson), std::move(letJson), std::move(collationJson)); + + auto parsedPipeline = Pipeline::parse(aggRequest->getPipeline(), _expCtx); + return std::make_unique<AggCmdShape>(*aggRequest, + kDefaultTestNss, + stdx::unordered_set<NamespaceString>{kDefaultTestNss}, + *parsedPipeline, + _expCtx); + } + std::unique_ptr<AggCmdShapeComponents> makeShapeComponentsFromPipeline( + std::vector<StringData> stagesJson, OptionalBool allowDiskUse = {}) { + auto aggRequest = makeAggregateCommandRequest(std::move(stagesJson)); + + auto parsedPipeline = Pipeline::parse(aggRequest->getPipeline(), _expCtx); + return std::make_unique<AggCmdShapeComponents>( + *aggRequest, + stdx::unordered_set<NamespaceString>{kDefaultTestNss}, + parsedPipeline->serializeToBson( + SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + } + + std::unique_ptr<QueryTestServiceContext> _queryTestServiceContext; + + ServiceContext::UniqueOperationContext _operationContext; + boost::intrusive_ptr<ExpressionContext> _expCtx; +}; + +TEST_F(AggCmdShapeTest, BasicPipelineShape) { + auto shape = + makeShapeFromPipeline({R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "cmdNs": { + "db": "testDB", + "coll": "testColl" + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "$and": [ + { + "x": { + "$eq": "?number" + } + }, + { + "y": { + "$lte": "?number" + } + } + ] + } + }, + { + "$group": { + "_id": "$y", + "z": { + "$max": "$z" + }, + "w": { + "$avg": "$w" + } + } + } + ] + })", + shape->toBson(_operationContext.get(), + SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +TEST_F(AggCmdShapeTest, IncludesLet) { + auto shape = makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}, + R"({x: 4, y: "str"})"_sd); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "cmdNs": { + "db": "testDB", + "coll": "testColl" + }, + "let": { + "x": "?number", + "y": "?string" + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "x": { + "$eq": "?number" + } + } + }, + { + "$limit": "?number" + } + ] + })", + shape->toBson(_operationContext.get(), + SerializationOptions::kDebugQueryShapeSerializeOptions)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "cmdNs": { + "db": "testDB", + "coll": "testColl" + }, + "let": { + "x": { + "$const": 1 + }, + "y": { + "$const": "?" + } + }, + "command": "aggregate", + "pipeline": [ + { + "$match": { + "x": { + "$eq": 1 + } + } + }, + { + "$limit": 1 + } + ] + })", + shape->toBson(_operationContext.get(), + SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); +} + +TEST_F(AggCmdShapeTest, SizeOfAggCmdShapeComponents) { + auto aggComponents = makeShapeComponentsFromPipeline( + {R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}, + false /*allowDiskUse*/); + + // The sizes of any members of AggCmdShapeComponents are typically accounted for by + // sizeof(AggCmdShapeComponents). The important part of the test here is to ensure that any + // additional memory allocations are also included in the size() operation. In our case, + // we expect additional memory use from the representative pipeline and the involved + // namespaces set. + const auto pipelineSize = shape_helpers::containerSize(aggComponents->representativePipeline); + const auto involvedNamespacesSize = sizeof(kDefaultTestNss) + + kDefaultTestNss.size(); // kDefaultTestNss is the only value in the unordered set. + + ASSERT_EQ(aggComponents->size(), + sizeof(AggCmdShapeComponents) + pipelineSize + involvedNamespacesSize); +} + +TEST_F(AggCmdShapeTest, EquivalentAggCmdShapeComponentSizes) { + auto aggComponentsDiskUseFalse = makeShapeComponentsFromPipeline( + {R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}, + false /*allowDiskUse*/); + auto aggComponentsDiskUseTrue = makeShapeComponentsFromPipeline( + {R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}, + true /*allowDiskUse*/); + ASSERT_EQ(aggComponentsDiskUseFalse->size(), aggComponentsDiskUseTrue->size()); +} + +TEST_F(AggCmdShapeTest, DifferentAggCmdShapeComponentSizes) { + auto smallAggComponents = makeShapeComponentsFromPipeline({R"({$match: {x: 3, y: {$lte: 3}}})"}, + false /*allowDiskUse*/); + auto largeAggComponents = makeShapeComponentsFromPipeline( + {R"({$match: {x: 3, y: {$lte: 3}}})"_sd, + R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"}, + false /*allowDiskUse*/); + ASSERT_LT(smallAggComponents->size(), largeAggComponents->size()); +} + +TEST_F(AggCmdShapeTest, SizeOfAggCmdShapeWithAndWithoutLet) { + auto shapeWithoutLet = makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}); + auto shapeWithLet = makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}, + R"({x: 4, y: "str"})"_sd); + ASSERT_LT(shapeWithoutLet->size(), shapeWithLet->size()); +} + +TEST_F(AggCmdShapeTest, SizeOfAggCmdShapeWithAndWithoutCollation) { + auto shapeWithoutCollation = + makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}); + auto shapeWithCollation = makeShapeFromPipeline( + {R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}, boost::none, R"({locale: "en_US"})"_sd); + ASSERT_LT(shapeWithoutCollation->size(), shapeWithCollation->size()); +} +} // namespace +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/cmd_with_let_shape.cpp b/src/mongo/db/query/query_shape/cmd_with_let_shape.cpp new file mode 100644 index 00000000000..2bbb6dfeadc --- /dev/null +++ b/src/mongo/db/query/query_shape/cmd_with_let_shape.cpp @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" + +namespace mongo::query_shape { + +namespace { +BSONObj extractLetShape(BSONObj letSpec, + const SerializationOptions& opts, + const boost::intrusive_ptr<ExpressionContext>& expCtx) { + if (letSpec.isEmpty()) { + // Fast path for the common case. + return letSpec; + } + + BSONObjBuilder bob; + for (BSONElement elem : letSpec) { + auto expr = Expression::parseOperand(expCtx.get(), elem, expCtx->variablesParseState); + auto redactedValue = expr->serialize(opts); + // Note that this will throw on deeply nested let variables. + redactedValue.addToBsonObj(&bob, opts.serializeFieldPathFromString(elem.fieldName())); + } + return bob.obj(); +} + +auto representativeLetShape(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx) { + return let ? extractLetShape( + *let, SerializationOptions::kRepresentativeQueryShapeSerializeOptions, expCtx) + : BSONObj(); +} +} // namespace + +LetShapeComponent::LetShapeComponent(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const CmdSpecificShapeComponents& unownedInnerComponents_) + : shapifiedLet(representativeLetShape(let, expCtx)), + hasLet(bool(let)), + unownedInnerComponents(unownedInnerComponents_) {} + +void LetShapeComponent::HashValue(absl::HashState state) const { + state = absl::HashState::combine( + std::move(state), hasLet, simpleHash(shapifiedLet), unownedInnerComponents); +} + +size_t LetShapeComponent::size() const { + return sizeof(LetShapeComponent) + shapifiedLet.objsize() + unownedInnerComponents.size(); +} + +void LetShapeComponent::addLetBson(BSONObjBuilder& bob, + const SerializationOptions& opts, + const boost::intrusive_ptr<ExpressionContext>& expCtx) const { + if (hasLet) { + auto shapeToAppend = shapifiedLet; + if (opts != SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + // We have the representative query cached/stored here, but the caller is asking for a + // different format, so we must re-compute. + shapeToAppend = extractLetShape(shapifiedLet, opts, expCtx); + } + bob.append(FindCommandRequest::kLetFieldName, shapeToAppend); + } +} + +void CmdWithLetShape::appendCmdSpecificShapeComponents(BSONObjBuilder& bob, + OperationContext* opCtx, + const SerializationOptions& opts) const { + auto expCtx = + ExpressionContext::makeBlankExpressionContext(opCtx, nssOrUUID, _let.shapifiedLet); + _let.addLetBson(bob, opts, expCtx); + appendLetCmdSpecificShapeComponents(bob, expCtx, opts); +} + +CmdWithLetShape::CmdWithLetShape(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const CmdSpecificShapeComponents& unownedInnerComponents, + NamespaceStringOrUUID nssOrUUID, + BSONObj collation) + : Shape(nssOrUUID, collation), _let(let, expCtx, unownedInnerComponents) {} + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/cmd_with_let_shape.h b/src/mongo/db/query/query_shape/cmd_with_let_shape.h new file mode 100644 index 00000000000..a1c127b9999 --- /dev/null +++ b/src/mongo/db/query/query_shape/cmd_with_let_shape.h @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/bson/bsonobj.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/query/parsed_find_command.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/serialization_options.h" + +namespace mongo::query_shape { + +/** + * This struct is bit of a weird one. We want to use it as the shape's _entire_ "specific + * components" (rather than introduce more virtual functions to that interface). So, we track here + * the let component (as the name suggests) but we also keep an unowned reference to the specific + * components of CmdWithLetShape sub-classes. This class doesn't really do all that much with those + * components except track a reference to them and ensure their size is accounted for and their hash + * value is incorporated. + */ +struct LetShapeComponent : public CmdSpecificShapeComponents { + LetShapeComponent(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const CmdSpecificShapeComponents& unownedInnerComponents); + + /** + * Hashes to include the shapified let parameters and also the hash of 'unownedInnerComponents'. + */ + void HashValue(absl::HashState state) const final; + + /** + * Includes the size of the let parameters and the size of 'unownedInnerComponents.' + */ + size_t size() const final; + + /** + * Adds _only_ the let params. + */ + void addLetBson(BSONObjBuilder&, + const SerializationOptions&, + const boost::intrusive_ptr<ExpressionContext>&) const; + + BSONObj shapifiedLet; + bool hasLet; + // Tracked so that this can be hash combined correctly. + const CmdSpecificShapeComponents& unownedInnerComponents; +}; + +/** + * The 'let' command argument is semi-generic in that it is supported in a couple commands. However + * it is treated specially since it supports using expressions as the let constants. Using + * expressions induces a library dependency that we don't want in the Shape interface itself. So + * this class handles tracking and adding the 'let' component of the shape for sub-classes. + */ +class CmdWithLetShape : public Shape { +public: + CmdWithLetShape(boost::optional<BSONObj> let, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const CmdSpecificShapeComponents& unownedInnerComponents, + NamespaceStringOrUUID, + BSONObj collation_); + + const CmdSpecificShapeComponents& specificComponents() const final { + return _let; + } + +protected: + void appendCmdSpecificShapeComponents(BSONObjBuilder&, + OperationContext* opCtx, + const SerializationOptions& opts) const final; + virtual void appendLetCmdSpecificShapeComponents( + BSONObjBuilder&, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions&) const = 0; + + LetShapeComponent _let; +}; +static_assert(sizeof(CmdWithLetShape) == sizeof(Shape) + sizeof(LetShapeComponent), + "If the class' members have changed, this assert and the extraSize() calculation may " + "need to be updated with a new value."); + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp b/src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp new file mode 100644 index 00000000000..21812d97fc7 --- /dev/null +++ b/src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp @@ -0,0 +1,78 @@ +/** + * Copyright (C) 2024-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_shape { + +namespace { +static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl"); + +class CmdWithLetShapeTest : public unittest::Test {}; + + +struct DummyInnerComponent : public CmdSpecificShapeComponents { + DummyInnerComponent(){}; + void HashValue(absl::HashState state) const {} + size_t size() const final { + return sizeof(*this); + } +}; + +TEST_F(CmdWithLetShapeTest, SizeOfLetShapeComponent) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto let = fromjson(R"({x: 4, y: "str"})"); + auto innerComponents = std::make_unique<DummyInnerComponent>(); + auto components = std::make_unique<LetShapeComponent>(let, expCtx, *innerComponents); + + const auto minimumSize = sizeof(CmdSpecificShapeComponents) + sizeof(BSONObj) + sizeof(bool) + + sizeof(void*) /*CmdSpecificShapeComponents&*/ + + static_cast<size_t>(components->shapifiedLet.objsize()) + + components->unownedInnerComponents.size(); + + ASSERT_GTE(components->size(), minimumSize); + ASSERT_LTE(components->size(), minimumSize + 8 /*padding*/); +} + +TEST_F(CmdWithLetShapeTest, SizeOfComponentWithAndWithoutLet) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto let = fromjson(R"({x: 4, y: "str"})"); + auto innerComponents = std::make_unique<DummyInnerComponent>(); + auto componentsWithLet = std::make_unique<LetShapeComponent>(let, expCtx, *innerComponents); + auto componentsWithNoLet = + std::make_unique<LetShapeComponent>(boost::none, expCtx, *innerComponents); + + ASSERT_LT(componentsWithNoLet->size(), componentsWithLet->size()); +} + +} // namespace +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/find_cmd_shape.cpp b/src/mongo/db/query/query_shape/find_cmd_shape.cpp new file mode 100644 index 00000000000..2d018de2619 --- /dev/null +++ b/src/mongo/db/query/query_shape/find_cmd_shape.cpp @@ -0,0 +1,227 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/find_cmd_shape.h" + +#include "mongo/db/query/projection_ast_util.h" +#include "mongo/db/query/query_shape/shape_helpers.h" + +namespace mongo::query_shape { +namespace { + +BSONObj projectionShape(const boost::optional<projection_ast::Projection>& proj, + const SerializationOptions& opts = + SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + return proj ? projection_ast::serialize(*proj->root(), opts) : BSONObj(); +} + +BSONObj sortShape(const boost::optional<SortPattern>& sort, + const SerializationOptions& opts = + SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + return sort + ? sort->serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts) + .toBson() + : BSONObj(); +} + +void maybeAddWithName(const OptionalBool& optBool, BSONObjBuilder& bob, StringData name) { + if (optBool.has_value()) { + bob.append(name, bool(optBool)); + } +} + +void addRemainingFindCommandFields(const FindCmdShapeComponents& components, BSONObjBuilder& bob) { + maybeAddWithName(components.singleBatch, bob, FindCommandRequest::kSingleBatchFieldName); + maybeAddWithName(components.allowDiskUse, bob, FindCommandRequest::kAllowDiskUseFieldName); + maybeAddWithName(components.returnKey, bob, FindCommandRequest::kReturnKeyFieldName); + maybeAddWithName(components.showRecordId, bob, FindCommandRequest::kShowRecordIdFieldName); + maybeAddWithName(components.tailable, bob, FindCommandRequest::kTailableFieldName); + maybeAddWithName(components.awaitData, bob, FindCommandRequest::kAwaitDataFieldName); + maybeAddWithName(components.oplogReplay, bob, FindCommandRequest::kOplogReplayFieldName); +} + +} // namespace + +FindCmdShapeComponents::FindCmdShapeComponents( + const ParsedFindCommand& request, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts) + : filter(request.filter->serialize(opts)), + projection(projectionShape(request.proj, opts)), + sort(sortShape(request.sort, opts)), + min(shape_helpers::extractMinOrMaxShape(request.findCommandRequest->getMin(), opts)), + max(shape_helpers::extractMinOrMaxShape(request.findCommandRequest->getMax(), opts)), + singleBatch(request.findCommandRequest->getSingleBatch()), + allowDiskUse(request.findCommandRequest->getAllowDiskUse().has_value() + ? boost::optional<bool>(bool(request.findCommandRequest->getAllowDiskUse())) + : boost::none), + returnKey(request.findCommandRequest->getReturnKey()), + showRecordId(request.findCommandRequest->getShowRecordId()), + tailable(request.findCommandRequest->getTailable()), + awaitData(request.findCommandRequest->getAwaitData()), + oplogReplay(request.findCommandRequest->getOplogReplay()), + hasField(), + serializationOpts(opts) { + hasField.projection = request.proj.has_value(); + hasField.sort = request.sort.has_value(); + hasField.limit = request.findCommandRequest->getLimit().has_value(); + hasField.skip = request.findCommandRequest->getSkip().has_value(); +} + +void FindCmdShapeComponents::appendTo(BSONObjBuilder& bob) const { + + bob.append("command", "find"); + + std::unique_ptr<MatchExpression> filterExpr; + // Filter. + bob.append(FindCommandRequest::kFilterFieldName, filter); + + if (hasField.projection) { + bob.append(FindCommandRequest::kProjectionFieldName, projection); + } + + if (!max.isEmpty()) { + bob.append(FindCommandRequest::kMaxFieldName, max); + } + if (!min.isEmpty()) { + bob.append(FindCommandRequest::kMinFieldName, min); + } + + // Sort. + if (hasField.sort) { + bob.append(FindCommandRequest::kSortFieldName, sort); + } + + // The values here don't matter (assuming we're not using the 'kUnchanged' policy). + tassert(7973601, + "Serialization policy not supported - original values have been discarded", + serializationOpts.literalPolicy != LiteralSerializationPolicy::kUnchanged); + if (hasField.limit) { + serializationOpts.appendLiteral(&bob, FindCommandRequest::kLimitFieldName, 1ll); + } + if (hasField.skip) { + serializationOpts.appendLiteral(&bob, FindCommandRequest::kSkipFieldName, 1ll); + } + + // Add the fields that require no transformation. + addRemainingFindCommandFields(*this, bob); +} + +void FindCmdShapeComponents::HashValue(absl::HashState state) const { + absl::HashState::combine(std::move(state), + simpleHash(filter), + simpleHash(projection), + simpleHash(sort), + simpleHash(min), + simpleHash(max), + singleBatch, + allowDiskUse, + returnKey, + showRecordId, + tailable, + awaitData, + oplogReplay, + hasField); +} + +std::unique_ptr<FindCommandRequest> FindCmdShape::toFindCommandRequest() const { + auto fcr = std::make_unique<FindCommandRequest>(nssOrUUID); + + fcr->setFilter(components.filter); + if (components.hasField.projection) + fcr->setProjection(components.projection); + if (components.hasField.sort) + fcr->setSort(components.sort); + + fcr->setMin(components.min); + fcr->setMax(components.max); + + // Doesn't matter what value to use for limit and skip in the context of a shape. + if (components.hasField.limit) + fcr->setLimit(1ll); + if (components.hasField.skip) + fcr->setSkip(1ll); + + // All the booleans. + if (components.singleBatch.has_value()) + fcr->setSingleBatch(bool(components.singleBatch)); + if (components.allowDiskUse.has_value()) + fcr->setAllowDiskUse(bool(components.allowDiskUse)); + if (components.returnKey.has_value()) + fcr->setReturnKey(bool(components.returnKey)); + if (components.showRecordId.has_value()) + fcr->setShowRecordId(bool(components.showRecordId)); + if (components.tailable.has_value()) + fcr->setTailable(bool(components.tailable)); + if (components.awaitData.has_value()) + fcr->setAwaitData(bool(components.awaitData)); + if (components.oplogReplay.has_value()) + fcr->setOplogReplay(bool(components.oplogReplay)); + + // Common shape components. + if (_let.hasLet) + fcr->setLet(_let.shapifiedLet); + if (!collation.isEmpty()) + fcr->setCollation(collation); + + + return fcr; +} + +FindCmdShape::FindCmdShape(const ParsedFindCommand& findRequest, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : CmdWithLetShape(findRequest.findCommandRequest->getLet(), + expCtx, + components, + findRequest.findCommandRequest->getNamespaceOrUUID(), + findRequest.findCommandRequest->getCollation()), + components(findRequest, expCtx) {} + +void FindCmdShape::appendLetCmdSpecificShapeComponents( + BSONObjBuilder& bob, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts) const { + if (opts == SerializationOptions::kRepresentativeQueryShapeSerializeOptions) { + // Fast path: we already have this. + return components.appendTo(bob); + } else { + // Slow path: we need to re-parse from our representative shapes. + auto request = uassertStatusOKWithContext( + parsed_find_command::parse(expCtx, + toFindCommandRequest(), + ExtensionsCallbackNoop(), + MatchExpressionParser::kAllowAllSpecialFeatures), + "Could not re-parse a representative query shape"); + + // This constructor will shapify according to the options. + FindCmdShapeComponents{*request, expCtx, opts}.appendTo(bob); + } +} + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/find_cmd_shape.h b/src/mongo/db/query/query_shape/find_cmd_shape.h new file mode 100644 index 00000000000..49d70b8ec27 --- /dev/null +++ b/src/mongo/db/query/query_shape/find_cmd_shape.h @@ -0,0 +1,130 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/query/query_shape/cmd_with_let_shape.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/shape_helpers.h" + +namespace mongo::query_shape { + +/** + * This struct tracks the components of a find command which are important for the find query shape. + * It attempts to only track those which are _unique_ to a find command - common elements should go + * on some super class. + * + * Data elements which are shapified like 'filter' are stored in their shapified form. By default + * and in most cases this will be the representative query shape form so that it can be re-parsed, + * but as a convenience for serializing it is also supported to construct and serialize this with + * other options. + */ +struct FindCmdShapeComponents : public CmdSpecificShapeComponents { + + FindCmdShapeComponents(const ParsedFindCommand& request, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts = + SerializationOptions::kRepresentativeQueryShapeSerializeOptions); + + /** + * Appends using the SerializationOptions given in the constructor. + */ + void appendTo(BSONObjBuilder&) const; + + size_t size() const final { + return sizeof(FindCmdShapeComponents) + filter.objsize() + projection.objsize() + + sort.objsize() + min.objsize() + max.objsize(); + } + + BSONObj filter; + BSONObj projection; + BSONObj sort; + BSONObj min; + BSONObj max; + + OptionalBool singleBatch; + OptionalBool allowDiskUse; + OptionalBool returnKey; + OptionalBool showRecordId; + OptionalBool tailable; + OptionalBool awaitData; + OptionalBool oplogReplay; + + // This anonymous struct represents the presence of the member variables as C++ bit fields. + // In doing so, each of these boolean values takes up 1 bit instead of 1 byte. + struct HasField { + HasField() : projection(false), sort(false), limit(false), skip(false) {} + bool projection : 1; + bool sort : 1; + bool limit : 1; + bool skip : 1; + } hasField; + + // We save a copy of the options used when constructed so we know how to properly append things + // like limit and skip - either a 1 or "?number". We could have the caller pass the options + // again during 'appendTo()', but this introduces a risk that the options provided are different + // than the ones we used to compute 'filter' and the other components. + SerializationOptions serializationOpts; + + void HashValue(absl::HashState state) const final; +}; + +class FindCmdShape : public CmdWithLetShape { +public: + FindCmdShape(const ParsedFindCommand& findRequest, + const boost::intrusive_ptr<ExpressionContext>& expCtx); + + /** + * Assembles a parseable FindCommandRequest representing this shape - some of the pieces are + * stored right here in the shape, others are in parent classes. + */ + std::unique_ptr<FindCommandRequest> toFindCommandRequest() const; + + FindCmdShapeComponents components; + +protected: + void appendLetCmdSpecificShapeComponents(BSONObjBuilder& bob, + const boost::intrusive_ptr<ExpressionContext>& expCtx, + const SerializationOptions& opts) const final; +}; + +template <typename H> +H AbslHashValue(H h, const FindCmdShapeComponents::HasField& hasField) { + return H::combine( + std::move(h), hasField.projection, hasField.sort, hasField.limit, hasField.skip); +} + +// This assertion is still active on the maintained master branch. On the v6.0 branch, we disable it +// since it is not passing on all toolchains/platforms - notably x86 macOS. The intent of the +// assertion is to prevent accidental additions of data members, which should not happen on this +// branch without first happening on the master branch and passing that assertion. +// static_assert(sizeof(FindCmdShape) == sizeof(CmdWithLetShape) + sizeof(FindCmdShapeComponents), +// "If the class' members have changed, this assert and the extraSize() calculation +// may " "need to be updated with a new value."); +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/find_cmd_shape_test.cpp b/src/mongo/db/query/query_shape/find_cmd_shape_test.cpp new file mode 100644 index 00000000000..0d839a5d3d2 --- /dev/null +++ b/src/mongo/db/query/query_shape/find_cmd_shape_test.cpp @@ -0,0 +1,238 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_shape/find_cmd_shape.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_shape { + +namespace { +/** + * TODO this was stolen from another test. Time for a library? + * Simplistic redaction strategy for testing which appends the field name to the prefix "REDACT_". + */ +std::string applyHmacForTest(StringData sd) { + return "REDACT_" + sd.toString(); +} + +static const NamespaceStringOrUUID kDefaultTestNss = + NamespaceStringOrUUID{NamespaceString("testDB.testColl")}; + +struct RequestOptions { + OptionalBool singleBatch = {}; + OptionalBool allowDiskUse = {}; + OptionalBool returnKey = {}; + OptionalBool showRecordId = {}; + OptionalBool tailable = {}; + OptionalBool awaitData = {}; + OptionalBool limit = {}; + OptionalBool skip = {}; +}; +class FindCmdShapeTest : public ServiceContextTest { +public: + void setUp() final { + _expCtx = make_intrusive<ExpressionContextForTest>(); + } + + std::unique_ptr<FindCmdShape> makeShapeFromSort(StringData sortJson) { + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setSort(fromjson(sortJson.rawData())); + auto&& parsedRequest = + uassertStatusOK(::mongo::parsed_find_command::parse(_expCtx, std::move(fcr))); + return std::make_unique<FindCmdShape>(*parsedRequest, _expCtx); + } + + BSONObj sortShape(StringData sortJson) { + auto shape = makeShapeFromSort(sortJson); + return shape->components.sort; + } + + /** + * Returns the shape of the input sort, or boost::none if the input shape was a natural sort + * which got converted into a hint. + */ + boost::optional<BSONObj> maybeRedactedSortShape(StringData sortJson) { + auto shape = makeShapeFromSort(sortJson); + SerializationOptions opts = SerializationOptions::kDebugQueryShapeSerializeOptions; + opts.transformIdentifiers = true; + opts.transformIdentifiersCallback = applyHmacForTest; + auto shapeBson = shape->toBson(_expCtx->opCtx, opts); + if (auto sortElem = shapeBson["sort"]; !sortElem.eoo()) { + return sortElem.Obj().getOwned(); + } + return boost::none; + } + + BSONObj redactedSortShape(StringData sortJson) { + return *maybeRedactedSortShape(sortJson); + } + + boost::intrusive_ptr<ExpressionContext> _expCtx; + + std::unique_ptr<FindCmdShapeComponents> makeShapeComponentsFromFilter( + BSONObj filter, const RequestOptions& requestOptions = {}) { + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + fcr->setSingleBatch(requestOptions.singleBatch); + fcr->setAllowDiskUse(requestOptions.allowDiskUse); + fcr->setReturnKey(requestOptions.returnKey); + fcr->setAllowDiskUse(requestOptions.showRecordId); + fcr->setTailable(requestOptions.tailable); + fcr->setAwaitData(requestOptions.awaitData); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)})); + return std::make_unique<FindCmdShapeComponents>(*parsedFind, _expCtx); + } + + std::unique_ptr<FindCmdShape> makeShapeFromFilter(const BSONObj& filter) { + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)})); + return std::make_unique<FindCmdShape>(*parsedFind, _expCtx); + } +}; + +TEST_F(FindCmdShapeTest, NormalSortPattern) { + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"a.b.c":1,"foo":-1})", + sortShape(R"({"a.b.c": 1, "foo": -1})")); +} + +TEST_F(FindCmdShapeTest, NaturalSortPattern) { + // $natural sorts are interpreted as a hint. Hints are not part of the shape (but should show up + // in the query stats key). + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({})", + sortShape(R"({$natural: 1})")); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({})", + sortShape(R"({$natural: -1})")); +} + +TEST_F(FindCmdShapeTest, NaturalSortPatternWithMeta) { + ASSERT_THROWS_CODE( + sortShape(R"({$natural: 1, x: {$meta: "textScore"}})"), DBException, ErrorCodes::BadValue); +} + +TEST_F(FindCmdShapeTest, MetaPatternWithoutNatural) { + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"normal":1,"$computed1":{"$meta":"textScore"}})", + sortShape(R"({normal: 1, x: {$meta: "textScore"}})")); +} + +// Here we have one test to ensure that the redaction policy is accepted and applied in the +// query_shape utility, but there are more extensive redaction tests in sort_pattern_test.cpp +TEST_F(FindCmdShapeTest, RespectsRedactionPolicy) { + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"REDACT_normal":1,"REDACT_y":1})", + redactedSortShape(R"({normal: 1, y: 1})")); + + // No need to redact $natural. Again, this will be interpreted as a hint, but this test is + // interesting to ensure the $-prefix of $natural doesn't confuse us. + ASSERT(!maybeRedactedSortShape(R"({$natural: 1})")); +} + +TEST_F(FindCmdShapeTest, SizeOfShapeComponents) { + auto query = BSON("query" << 1 << "xEquals" << 42); + auto findCmdComponent = makeShapeComponentsFromFilter(query.getOwned()); + const auto querySize = findCmdComponent->filter.objsize(); + + const auto minimumSize = sizeof(FindCmdShapeComponents) + querySize; + ASSERT_GT(findCmdComponent->size(), minimumSize); + ASSERT_LTE(findCmdComponent->size(), + minimumSize + static_cast<size_t>(4 * BSONObj().objsize())); +} + +TEST_F(FindCmdShapeTest, EquivalentShapeComponentsSizes) { + auto query = BSON("query" << 1 << "xEquals" << 42); + // Tailable can not be set together with 'singleBatch' option. + auto mostlyTrueComponent = makeShapeComponentsFromFilter(query.getOwned(), + {/* singleBatch = */ false, + /* allowDiskUse = */ true, + /* returnKey = */ true, + /* showRecordId = */ true, + /* tailable = */ true, + /* awaitData = */ true, + /* limit = */ true, + /* skip = */ true}); + + auto mostlyFalseComponent = makeShapeComponentsFromFilter(query.getOwned(), + {/* singleBatch = */ false, + /* allowDiskUse = */ false, + /* returnKey = */ false, + /* showRecordId = */ false, + /* tailable = */ true, + /* awaitData = */ false, + /* limit = */ false, + /* skip = */ false}); + + ASSERT_EQ(mostlyTrueComponent->size(), mostlyFalseComponent->size()); +} + +TEST_F(FindCmdShapeTest, DifferentShapeComponentsSizes) { + auto smallQuery = BSON("query" << BSONObj()); + auto smallFindCmdComponent = makeShapeComponentsFromFilter(smallQuery.getOwned()); + + auto largeQuery = BSON("query" << 1 << "xEquals" << 42); + auto largeFindCmdComponent = makeShapeComponentsFromFilter(largeQuery.getOwned()); + + ASSERT_LT(smallQuery.objsize(), largeQuery.objsize()); + ASSERT_LT(smallFindCmdComponent->size(), largeFindCmdComponent->size()); +} + +TEST_F(FindCmdShapeTest, SizeOfShapeWithAndWithoutLet) { + auto filter = BSON("query" << 1 << "xEquals" << 42); + auto shapeWithoutLet = makeShapeFromFilter(filter.getOwned()); + + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + fcr->setLet(fromjson(R"({x: 4})")); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)})); + auto shapeWithLet = std::make_unique<FindCmdShape>(*parsedFind, _expCtx); + + ASSERT_LT(shapeWithoutLet->size(), shapeWithLet->size()); +} + +TEST_F(FindCmdShapeTest, SizeOfShapeWithAndWithoutCollation) { + auto filter = BSON("query" << 1 << "xEquals" << 42); + auto shapeWithoutCollation = makeShapeFromFilter(filter.getOwned()); + + auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss); + fcr->setFilter(filter.getOwned()); + fcr->setCollation(fromjson(R"({locale: "en_US"})")); + auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)})); + auto shapeWithCollation = std::make_unique<FindCmdShape>(*parsedFind, _expCtx); + + ASSERT_LT(shapeWithoutCollation->size(), shapeWithCollation->size()); +} + +} // namespace + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/query_shape.cpp b/src/mongo/db/query/query_shape/query_shape.cpp new file mode 100644 index 00000000000..2fa0520120e --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape.cpp @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/query_shape.h" + +#include "mongo/base/status.h" +#include "mongo/crypto/sha256_block.h" +#include "mongo/db/query/find_command_gen.h" +#include "mongo/db/query/projection_ast_util.h" +#include "mongo/db/query/query_request_helper.h" +#include "mongo/db/query/query_shape/agg_cmd_shape.h" +#include "mongo/db/query/query_shape/find_cmd_shape.h" +#include "mongo/db/query/query_shape/query_shape_gen.h" +#include "mongo/db/query/query_shape/shape_helpers.h" +#include "mongo/db/query/sort_pattern.h" + +namespace mongo::query_shape { + +namespace { +void appendCmdNs(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts) { + BSONObjBuilder nsObj = bob.subobjStart("cmdNs"); + shape_helpers::appendNamespaceShape(nsObj, nss, opts); + nsObj.doneFast(); +} +} // namespace + +Shape::Shape(NamespaceStringOrUUID nssOrUUID_, BSONObj collation_) + : nssOrUUID(nssOrUUID_), collation(std::move(collation_)) {} + + +BSONObj Shape::toBson(OperationContext* opCtx, const SerializationOptions& opts) const { + BSONObjBuilder bob; + appendCmdNsOrUUID(bob, opts); + if (!collation.isEmpty()) { + // Collation is never shapified. We use find command's collation name definition, but it + // should be the same for all requests. + bob.append(FindCommandRequest::kCollationFieldName, collation); + } + appendCmdSpecificShapeComponents(bob, opCtx, opts); + return bob.obj(); +} + +size_t Shape::size() const { + return sizeof(Shape) + shape_helpers::optionalObjSize(collation) + specificComponents().size() + + extraSize(); +} + +QueryShapeHash Shape::sha256Hash(OperationContext* opCtx) const { + // The Query Shape Hash should use the representative query shape. + auto serialized = + toBson(opCtx, SerializationOptions::kRepresentativeQueryShapeSerializeOptions); + return SHA256Block::computeHash((const uint8_t*)serialized.sharedBuffer().get(), + serialized.objsize()); +} + +void Shape::appendCmdNsOrUUID(BSONObjBuilder& bob, const SerializationOptions& opts) const { + if (nssOrUUID.nss()) { + appendCmdNs(bob, *nssOrUUID.nss(), opts); + } else { + BSONObjBuilder cmdNs = bob.subobjStart("cmdNs"); + cmdNs.append("uuid", opts.serializeIdentifier(nssOrUUID.uuid()->toString())); + cmdNs.append("db", opts.serializeIdentifier(nssOrUUID.db())); + cmdNs.doneFast(); + } +} + +void Shape::appendCmdNs(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts) const { + BSONObjBuilder nsObj = bob.subobjStart("cmdNs"); + shape_helpers::appendNamespaceShape(nsObj, nss, opts); + nsObj.doneFast(); +} + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/query_shape.h b/src/mongo/db/query/query_shape/query_shape.h new file mode 100644 index 00000000000..dc83cbab127 --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape.h @@ -0,0 +1,165 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/matcher/expression.h" +#include "mongo/db/pipeline/aggregate_command_gen.h" +#include "mongo/db/query/find_command_gen.h" +#include "mongo/db/query/query_request_helper.h" +#include "mongo/db/query/query_shape/shape_helpers.h" + +namespace mongo::query_shape { + +/** + * Each type of "query" command likely has different fields/options that are considered important + * for the shape. For example, a find command has a skip and a limit, and an aggregate command has a + * pipeline. This interface is used to allow different sub-commands to diverge in this way but still + * ensure we can appropriately hash them to compare their shapes, and properly account for their + * size. + * + * This struct is split out as a separate inheritence hierarchy from 'Shape' to make it easier to + * ensure each piece is hashed without sub-classes needing to enumerate the parent class's member + * variables. + */ +struct CmdSpecificShapeComponents { + virtual ~CmdSpecificShapeComponents() {} + + /** + * Sub-classes should implement this in a way which includes all shape-relevant state. If two + * shapes should compare equal, they should result in the same hash value. For example for the + * find command - we would include the _shapified_ filter and projection here, but we will not + * include the comment - which is not part of the shape. + */ + virtual void HashValue(absl::HashState state) const = 0; + + /** + * It is important for shape components to accurately report their size, and to make a + * reasonable effort to maintain a minimal size. We use the query shape in memory-constrained + * data structures, so a bigger shape means we can have fewer different shapes stored (for + * example in the query stats store). + * + * We cannot just use sizeof() because there are some variable size data members (like BSON + * objects) which depend on the particular instance. + */ + virtual size_t size() const = 0; + + // Some template boilerplate to allow sub-classes to overload the hash implementation. + template <typename H> + friend H AbslHashValue(H state, const CmdSpecificShapeComponents& value) { + value.HashValue(absl::HashState::Create(&state)); + return std::move(state); + } +}; + +using QueryShapeHash = SHA256Block; + +/** + * A query "shape" is a version of a command with literal values abstracted so that two instances of + * the command may compare/hash equal even if they use slightly different literal values. This + * concept exists not just the find command, but planned for many of the CRUD commands + aggregate. + * It also includes most (but not all) components of these commands, not just the query predicate + * (MatchExpresssion). In these ways, "query" is meant more generally. + * + * A "Query Shape" can vary depending on the command (e.g. find, aggregate, or distinct). This + * abstract struct is the API we must implement for each command which we want to have a "shape" + * concept. + * + * In order to properly account for the size of a query shape, the CmdSpecificShapeComponents should + * include all meaningful memory consumption, and be sure to report it in 'size()'. Subclasses of + * 'Shape' are not expected to have any meaningful memory usage outside of that struct. + */ +class Shape { +public: + virtual ~Shape() {} + + /** + * Sub-classes are expected to implement this as a mechanism for plugging in their command + * specific shape components. + */ + virtual const CmdSpecificShapeComponents& specificComponents() const = 0; + + /** + * Note this may involve re-parsing command BSON and so is not necessarily cheap. + */ + BSONObj toBson(OperationContext*, const SerializationOptions&) const; + + /** + * The Query Shape Hash is defined to be the SHA256 Hash of the representatice query shape. This + * helper computes that. + */ + QueryShapeHash sha256Hash(OperationContext*) const; + + /** + * The size of a query shape is important, since we store these in space-constrained + * environments like the query stats store. + */ + size_t size() const; + + /** + * This should be overriden by a child class if it has members whose sizes are not included in + * specificComponents().size(). + */ + virtual size_t extraSize() const { + return 0; + } + template <typename H> + friend H AbslHashValue(H h, const Shape& shape) { + h = H::combine(std::move(h), shape.nssOrUUID, shape.specificComponents()); + if (!shape.collation.isEmpty()) + h = H::combine(std::move(h), simpleHash(shape.collation)); + return h; + } + + + // Not shapified but it is an identifier so it may be transformed. + NamespaceStringOrUUID nssOrUUID; + + // Never shapified. If it's empty, leave it off. + BSONObj collation; + +protected: + Shape(NamespaceStringOrUUID, BSONObj collation_); + + /** + * Along with the hash implementation, this is the main way that shapes are 'shapified' - + * sub-classes should implement this to add the shapified versions of their literals to an + * object. Depending on 'opts', this may be eligible to be used for output in $queryStats or as + * the object to compute the QueryShapeHash. + */ + virtual void appendCmdSpecificShapeComponents(BSONObjBuilder&, + OperationContext*, + const SerializationOptions& opts) const = 0; + +private: + void appendCmdNsOrUUID(BSONObjBuilder&, const SerializationOptions&) const; + void appendCmdNs(BSONObjBuilder&, const NamespaceString&, const SerializationOptions&) const; +}; + +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/query_shape.idl b/src/mongo/db/query/query_shape/query_shape.idl new file mode 100644 index 00000000000..77e71756467 --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape.idl @@ -0,0 +1,50 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. + +global: + cpp_namespace: "mongo::query_shape" + +imports: + - "mongo/idl/basic_types.idl" + + +structs: + CommandNamespace: + description: "Representation of the cmdNs sub-object of the query shape." + fields: + db: + type: string + coll: + type: string + optional: true + uuid: + type: string + optional: true + tenantId: + type: string + optional: true +
\ No newline at end of file diff --git a/src/mongo/db/query/query_shape/query_shape_test.cpp b/src/mongo/db/query/query_shape/query_shape_test.cpp new file mode 100644 index 00000000000..d6185b5c5cb --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape_test.cpp @@ -0,0 +1,767 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/db/matcher/expression_geo.h" +#include "mongo/db/matcher/extensions_callback_real.h" +#include "mongo/db/matcher/parsed_match_expression_for_test.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/query_shape_test_gen.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/db/query/query_shape/shape_helpers.h" +#include "mongo/db/service_context_test_fixture.h" +#include "mongo/unittest/bson_test_util.h" +#include "mongo/unittest/unittest.h" + +namespace mongo::query_shape { + +namespace { +BSONObj predicateShape(const MatchExpression* expr) { + return expr->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions); +} +BSONObj predicateShape(std::string filterJson) { + return predicateShape(ParsedMatchExpressionForTest(filterJson).get()); +} + +BSONObj predicateShapeRedacted(const MatchExpression* expr) { + return expr->serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST); +} +BSONObj predicateShapeRedacted(std::string filterJson) { + return predicateShapeRedacted(ParsedMatchExpressionForTest(filterJson).get()); +} + +// TODO SERVER-87736 There is no 'auto' here, make that more clear. +#define ASSERT_SHAPE_EQ_AUTO(expected, actual) \ + ASSERT_BSONOBJ_EQ_AUTO(expected, predicateShape(actual)) + +#define ASSERT_REDACTED_SHAPE_EQ_AUTO(expected, actual) \ + ASSERT_BSONOBJ_EQ_AUTO(expected, predicateShapeRedacted(actual)) + + +TEST(QueryPredicateShape, Equals) { + ASSERT_SHAPE_EQ_AUTO( // Implicit equals + R"({"a":{"$eq":"?number"}})", + "{a: 5}"); + ASSERT_SHAPE_EQ_AUTO( // Explicit equals + R"({"a":{"$eq":"?number"}})", + "{a: {$eq: 5}}"); + ASSERT_SHAPE_EQ_AUTO( // implicit $and + R"({"$and":[{"a":{"$eq":"?number"}},{"b":{"$eq":"?number"}}]})", + "{a: 5, b: 6}"); + ASSERT_REDACTED_SHAPE_EQ_AUTO( // Implicit equals + R"({"HASH<a>":{"$eq":"?number"}})", + "{a: 5}"); + ASSERT_REDACTED_SHAPE_EQ_AUTO( // Explicit equals + R"({"HASH<a>":{"$eq":"?number"}})", + "{a: {$eq: 5}}"); + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"$and":[{"HASH<a>":{"$eq":"?number"}},{"HASH<b>":{"$eq":"?number"}}]})", + "{a: 5, b: 6}"); + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"HASH<foo>.HASH<$bar>":{"$eq":"?number"}})", + R"({"foo.$bar":0})"); +} + +TEST(QueryPredicateShape, ArraySubTypes) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + "{a: {$eq: '[]'}}", + "{a: []}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + "{a: {$eq: '?array<?number>'}}", + "{a: [2]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?number>"}})", + "{a: [2, 3]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?object>"}})", + "{a: [{}]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?object>"}})", + "{a: [{}, {}]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?array>"}})", + "{a: [[], [], []]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<?array>"}})", + "{a: [[2, 3], ['string'], []]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<>"}})", + "{a: [{}, 2]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<>"}})", + "{a: [[], 2]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<>"}})", + "{a: [[{}, 'string'], 2]}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$eq":"?array<>"}})", + "{a: [[{}, 'string'], 2]}"); +} + +TEST(QueryPredicateShape, Comparisons) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$lt": "?number" + } + }, + { + "b": { + "$gt": "?number" + } + }, + { + "c": { + "$gte": "?number" + } + }, + { + "c": { + "$lte": "?number" + } + } + ] + })", + "{a: {$lt: 5}, b: {$gt: 6}, c: {$gte: 3, $lte: 10}}"); +} + +namespace { +void assertShapeIs(std::string filterJson, BSONObj expectedShape) { + ASSERT_BSONOBJ_EQ(expectedShape, predicateShape(filterJson)); +} + +void assertRedactedShapeIs(std::string filterJson, BSONObj expectedShape) { + ASSERT_BSONOBJ_EQ(expectedShape, predicateShapeRedacted(filterJson)); +} +} // namespace + +TEST(QueryPredicateShape, Regex) { + // Note/warning: 'fromjson' will parse $regex into a /regex/, so these tests can't use + // auto-updating BSON assertions. + assertShapeIs("{a: /a+/}", + BSON("a" << BSON("$regex" + << "?string"))); + assertShapeIs("{a: /a+/i}", + BSON("a" << BSON("$regex" + << "?string" + << "$options" + << "?string"))); + assertRedactedShapeIs("{a: /a+/}", + BSON("HASH<a>" << BSON("$regex" + << "?string"))); + assertRedactedShapeIs("{a: /a+/}", + BSON("HASH<a>" << BSON("$regex" + << "?string"))); +} + +TEST(QueryPredicateShape, Mod) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$mod":["?number","?number"]}})", + "{a: {$mod: [2, 0]}}"); +} + +TEST(QueryPredicateShape, Exists) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$exists":"?bool"}})", + "{a: {$exists: true}}"); +} + +TEST(QueryPredicateShape, In) { + // Any number of children in any order is always the same shape + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$in":"?array<?number>"}})", + "{a: {$in: [1]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$in":"?array<>"}})", + "{a: {$in: [1, 4, 'str', /regex/]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$in":"?array<>"}})", + "{a: {$in: ['str', /regex/, 1, 4]}}"); +} + +TEST(QueryPredicateShape, BitTestOperators) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAllSet":"?array<?number>"}})", + "{a: {$bitsAllSet: [1, 5]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAllSet":"?array<?number>"}})", + "{a: {$bitsAllSet: 50}}"); + + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAnySet":"?array<?number>"}})", + "{a: {$bitsAnySet: [1, 5]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAnySet":"?array<?number>"}})", + "{a: {$bitsAnySet: 50}}"); + + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAllClear":"?array<?number>"}})", + "{a: {$bitsAllClear: [1, 5]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAllClear":"?array<?number>"}})", + "{a: {$bitsAllClear: 50}}"); + + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAnyClear":"?array<?number>"}})", + "{a: {$bitsAnyClear: [1, 5]}}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$bitsAnyClear":"?array<?number>"}})", + "{a: {$bitsAnyClear: 50}}"); +} + +TEST(QueryPredicateShape, AlwaysBoolean) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"$alwaysTrue":"?number"})", + "{$alwaysTrue: 1}"); + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"$alwaysFalse":"?number"})", + "{$alwaysFalse: 1}"); +} + +TEST(QueryPredicateShape, And) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$lt": "?number" + } + }, + { + "b": { + "$gte": "?number" + } + }, + { + "c": { + "$lte": "?number" + } + } + ] + })", + "{$and: [{a: {$lt: 5}}, {b: {$gte: 3}}, {c: {$lte: 10}}]}"); +} + +TEST(QueryPredicateShape, Or) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({ + "$or": [ + { + "a": { + "$eq": "?number" + } + }, + { + "b": { + "$in": "?array<?number>" + } + }, + { + "c": { + "$gt": "?number" + } + } + ] + })", + "{$or: [{a: 5}, {b: {$in: [1,2,3]}}, {c: {$gt: 10}}]}"); +} + +TEST(QueryPredicateShape, ElemMatch) { + // ElemMatchObjectMatchExpression + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({ + "a": { + "$elemMatch": { + "$and": [ + { + "b": { + "$eq": "?number" + } + }, + { + "c": { + "$exists": "?bool" + } + } + ] + } + } + })", + "{a: {$elemMatch: {b: 5, c: {$exists: true}}}}"); + + // ElemMatchValueMatchExpression + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"a":{"$elemMatch":{"$gt":"?number","$lt":"?number"}}})", + "{a: {$elemMatch: {$gt: 5, $lt: 10}}}"); + + // Nested + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({ + "HASH<a>": { + "$elemMatch": { + "$elemMatch": { + "$gt": "?number", + "$lt": "?number" + } + } + } + })", + "{a: {$elemMatch: {$elemMatch: {$gt: 5, $lt: 10}}}}"); +} + +TEST(QueryPredicateShape, InternalBucketGeoWithinMatchExpression) { + auto query = + "{ $_internalBucketGeoWithin: {withinRegion: {$centerSphere: [[0, 0], 10]}, field: " + "\"a\"} " + "}"; + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({ + "$_internalBucketGeoWithin": { + "withinRegion": { + "$centerSphere": "?array<>" + }, + "field": "HASH<a>" + } + })", + query); +} + +TEST(QueryPredicateShape, NorMatchExpression) { + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"$nor":[{"HASH<a>":{"$lt":"?number"}},{"HASH<b>":{"$gt":"?number"}}]})", + "{ $nor: [ { a: {$lt: 5} }, { b: {$gt: 4} } ] }"); +} + +TEST(QueryPredicateShape, NotMatchExpression) { + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"HASH<price>":{"$not":{"$gt":"?number"}}})", + "{ price: { $not: { $gt: 1.99 } } }"); + // Test the special case where NotMatchExpression::serialize() reduces to $alwaysFalse. + auto emptyAnd = std::make_unique<AndMatchExpression>(); + const MatchExpression& notExpr = NotMatchExpression(std::move(emptyAnd)); + auto serialized = + notExpr.serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$alwaysFalse":"?number"})", + serialized); +} + +TEST(QueryPredicateShape, SizeMatchExpression) { + ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT + R"({"HASH<price>":{"$size":"?number"}})", + "{ price: { $size: 2 } }"); +} + +TEST(QueryPredicateShape, TextMatchExpression) { + TextMatchExpressionBase::TextParams params = {"coffee"}; + auto expr = ExtensionsCallbackNoop().createText(params); + auto literalAndFieldRedactOpts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$text": { + "$search": "?string", + "$language": "?string", + "$caseSensitive": "?bool", + "$diacriticSensitive": "?bool" + } + })", + expr->serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST)); +} + +TEST(QueryPredicateShape, TwoDPtInAnnulusExpression) { + const MatchExpression& expr = TwoDPtInAnnulusExpression({}, {}); + auto literalAndFieldRedactOpts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$TwoDPtInAnnulusExpression":true})", + expr.serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST)); +} + +TEST(QueryPredicateShape, WhereMatchExpression) { + ASSERT_SHAPE_EQ_AUTO( // NOLINT + R"({"$where":"?javascript"})", + "{$where: \"some_code()\"}"); +} + +BSONObj queryShapeForOptimizedExprExpression(std::string exprPredicateJson) { + ParsedMatchExpressionForTest expr(exprPredicateJson); + // We need to optimize an $expr expression in order to generate an $_internalExprEq. It's + // not clear we'd want to do optimization before computing the query shape, but we should + // support the computation on any MatchExpression, and this is the easiest way we can create + // this type of MatchExpression node. + auto optimized = MatchExpression::optimize(expr.release()); + return predicateShape(optimized.get()); +} + +TEST(QueryPredicateShape, OptimizedExprPredicates) { + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprEq": "?number" + } + }, + { + "$expr": { + "$eq": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$eq: ['$a', 2]}}")); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprLt": "?number" + } + }, + { + "$expr": { + "$lt": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$lt: ['$a', 2]}}")); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprLte": "?number" + } + }, + { + "$expr": { + "$lte": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$lte: ['$a', 2]}}")); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprGt": "?number" + } + }, + { + "$expr": { + "$gt": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$gt: ['$a', 2]}}")); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "a": { + "$_internalExprGte": "?number" + } + }, + { + "$expr": { + "$gte": [ + "$a", + "?number" + ] + } + } + ] + })", + queryShapeForOptimizedExprExpression("{$expr: {$gte: ['$a', 2]}}")); +} + +TEST(QueryShapeIDL, ShapifyIDLStruct) { + SerializationOptions options; + options.transformIdentifiers = true; + options.transformIdentifiersCallback = [](StringData s) -> std::string { + return str::stream() << "HASH<" << s << ">"; + }; + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + + auto nested = NestedStruct("value", + ExampleEnumEnum::Value1, + "hello", + {1, 2, 3, 4}, + "field.path", + {"field.path.1", "fieldpath2"}, + NamespaceString{"db", "coll"}, + NamespaceString{"db", "coll"}, + 177, + true); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "stringField": "value", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": [ + 1, + 2, + 3, + 4 + ], + "fieldpath": "field.path", + "fieldpathList": [ + "field.path.1", + "fieldpath2" + ], + "nss": "db.coll", + "plainNss": "db.coll", + "safeInt64Field": 177, + "boolField": true + })", + nested.toBSON()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "stringField": "?string", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": "?array<?number>", + "fieldpath": "HASH<field>.HASH<path>", + "fieldpathList": [ + "HASH<field>.HASH<path>.HASH<1>", + "HASH<fieldpath2>" + ], + "nss": "HASH<db.coll>", + "plainNss": "db.coll", + "safeInt64Field": "?number", + "boolField": "?bool" + })", + nested.toBSON(options)); + + + auto parent = ParentStruct(nested, nested); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "nested_shape": { + "stringField": "value", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": [ + 1, + 2, + 3, + 4 + ], + "fieldpath": "field.path", + "fieldpathList": [ + "field.path.1", + "fieldpath2" + ], + "nss": "db.coll", + "plainNss": "db.coll", + "safeInt64Field": 177, + "boolField": true + }, + "nested_no_shape": { + "stringField": "value", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": [ + 1, + 2, + 3, + 4 + ], + "fieldpath": "field.path", + "fieldpathList": [ + "field.path.1", + "fieldpath2" + ], + "nss": "db.coll", + "plainNss": "db.coll", + "safeInt64Field": 177, + "boolField": true + } + })", + parent.toBSON()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "nested_shape": { + "stringField": "?string", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": "?array<?number>", + "fieldpath": "HASH<field>.HASH<path>", + "fieldpathList": [ + "HASH<field>.HASH<path>.HASH<1>", + "HASH<fieldpath2>" + ], + "nss": "HASH<db.coll>", + "plainNss": "db.coll", + "safeInt64Field": "?number", + "boolField": "?bool" + }, + "nested_no_shape": { + "stringField": "value", + "enumField": "EnumValue1", + "stringIntVariantEnum": "hello", + "arrayOfInts": [ + 1, + 2, + 3, + 4 + ], + "fieldpath": "field.path", + "fieldpathList": [ + "field.path.1", + "fieldpath2" + ], + "nss": "db.coll", + "plainNss": "db.coll", + "safeInt64Field": 177, + "boolField": true + } + })", + parent.toBSON(options)); +} + +} // namespace + +namespace { + +static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl"); + +struct DummyShapeSpecificComponents : public query_shape::CmdSpecificShapeComponents { + DummyShapeSpecificComponents(){}; + void HashValue(absl::HashState state) const {} + size_t size() const final { + return sizeof(DummyShapeSpecificComponents); + } +}; + +class DummyShape : public Shape { +public: + DummyShape(NamespaceStringOrUUID nssOrUUID, + BSONObj collation, + DummyShapeSpecificComponents dummyComponents) + : Shape(nssOrUUID, collation) { + components = dummyComponents; + } + + const CmdSpecificShapeComponents& specificComponents() const final { + return components; + } + + void appendCmdSpecificShapeComponents(BSONObjBuilder&, + OperationContext*, + const SerializationOptions& opts) const final {} + DummyShapeSpecificComponents components; +}; + +class DummyShapeWithExtraSize : public Shape { +public: + DummyShapeWithExtraSize(NamespaceStringOrUUID nssOrUUID, + BSONObj collation, + DummyShapeSpecificComponents dummyComponents) + : Shape(nssOrUUID, collation) { + components = dummyComponents; + } + + const CmdSpecificShapeComponents& specificComponents() const final { + return components; + } + + // Random number for testing purposes. + size_t extraSize() const final override { + return 125; + } + void appendCmdSpecificShapeComponents(BSONObjBuilder&, + OperationContext*, + const SerializationOptions& opts) const final {} + + DummyShapeSpecificComponents components; +}; + +class UniversalShapeTest : public ServiceContextTest {}; + +TEST_F(UniversalShapeTest, SizeOfSpecificComponents) { + auto innerComponents = std::make_unique<DummyShapeSpecificComponents>(); + ASSERT_EQ(innerComponents->size(), sizeof(CmdSpecificShapeComponents)); + ASSERT_EQ(innerComponents->size(), sizeof(void*) /*vtable ptr*/); +} + +TEST_F(UniversalShapeTest, SizeOfShape) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + // Make shape for testing. + auto collation = BSONObj{}; + auto innerComponents = std::make_unique<DummyShapeSpecificComponents>(); + auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents); + + ASSERT_EQ(innerComponents->size(), shape->specificComponents().size()); + ASSERT_EQ(shape->size(), + sizeof(NamespaceStringOrUUID) + sizeof(BSONObj) + sizeof(void*) /*vtable ptr*/ + + shape->specificComponents().size() + static_cast<size_t>(collation.objsize())); +} + +TEST_F(UniversalShapeTest, SizeOfShapeWithExtraSize) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + // Make shape for testing. + auto collation = BSONObj{}; + auto innerComponents = std::make_unique<DummyShapeSpecificComponents>(); + auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents); + auto shapeWithExtraSize = + std::make_unique<DummyShapeWithExtraSize>(kDefaultTestNss, collation, *innerComponents); + + ASSERT_EQ(shapeWithExtraSize->size(), shape->size() + shapeWithExtraSize->extraSize()); +} +} // namespace +} // namespace mongo::query_shape diff --git a/src/mongo/db/query/query_shape/query_shape_test.idl b/src/mongo/db/query/query_shape/query_shape_test.idl new file mode 100644 index 00000000000..06efb7ed1ef --- /dev/null +++ b/src/mongo/db/query/query_shape/query_shape_test.idl @@ -0,0 +1,91 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/idl/basic_types.idl" + +enums: + ExampleEnum: + description: "" + type: string + values: + Value1: "EnumValue1" + Value2: "EnumValue2" + +structs: + NestedStruct: + query_shape_component: true + strict: true + description: "" + fields: + stringField: + query_shape: literal + type: string + enumField: + query_shape: parameter + type: ExampleEnum + stringIntVariantEnum: + query_shape: parameter + type: + variant: [string, int] + arrayOfInts: + query_shape: literal + type: array<int> + fieldpath: + query_shape: anonymize + type: string + fieldpathList: + query_shape: anonymize + type: array<string> + nss: + query_shape: custom + type: namespacestring + plainNss: + query_shape: parameter + type: namespacestring + safeInt64Field: + query_shape: literal + type: safeInt64 + boolField: + query_shape: literal + type: bool + + ParentStruct: + query_shape_component: true + strict: true + description: "" + fields: + nested_shape: + query_shape: literal + type: NestedStruct + nested_no_shape: + query_shape: parameter + type: NestedStruct diff --git a/src/mongo/db/query/query_shape/serialization_options.cpp b/src/mongo/db/query/query_shape/serialization_options.cpp new file mode 100644 index 00000000000..e6008f8579b --- /dev/null +++ b/src/mongo/db/query/query_shape/serialization_options.cpp @@ -0,0 +1,515 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "serialization_options.h" +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + +#include "mongo/db/query/query_shape/serialization_options.h" + +#include <boost/optional.hpp> +#include <string> + +#include "mongo/base/string_data.h" +#include "mongo/bson/timestamp.h" +#include "mongo/db/exec/document_value/document.h" +#include "mongo/db/exec/document_value/value.h" +#include "mongo/db/query/explain_options.h" +#include "mongo/logv2/log.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + +namespace { + +// We'll pre-declare all of these strings so that we can avoid the allocations when we reference +// them later. +static constexpr StringData kUndefinedTypeString = "?undefined"_sd; +static constexpr StringData kStringTypeString = "?string"_sd; +static constexpr StringData kNumberTypeString = "?number"_sd; +static constexpr StringData kMinKeyTypeString = "?minKey"_sd; +static constexpr StringData kObjectTypeString = "?object"_sd; +static constexpr StringData kArrayTypeString = "?array"_sd; +static constexpr StringData kBinDataTypeString = "?binData"_sd; +static constexpr StringData kObjectIdTypeString = "?objectId"_sd; +static constexpr StringData kBoolTypeString = "?bool"_sd; +static constexpr StringData kDateTypeString = "?date"_sd; +static constexpr StringData kNullTypeString = "?null"_sd; +static constexpr StringData kRegexTypeString = "?regex"_sd; +static constexpr StringData kDbPointerTypeString = "?dbPointer"_sd; +static constexpr StringData kJavascriptTypeString = "?javascript"_sd; +static constexpr StringData kJavascriptWithScopeTypeString = "?javascriptWithScope"_sd; +static constexpr StringData kTimestampTypeString = "?timestamp"_sd; +static constexpr StringData kMaxKeyTypeString = "?maxKey"_sd; + +static const StringMap<StringData> kArrayTypeStringConstants{ + {kUndefinedTypeString.rawData(), "?array<?undefined>"_sd}, + {kStringTypeString.rawData(), "?array<?string>"_sd}, + {kNumberTypeString.rawData(), "?array<?number>"_sd}, + {kMinKeyTypeString.rawData(), "?array<?minKey>"_sd}, + {kObjectTypeString.rawData(), "?array<?object>"_sd}, + {kArrayTypeString.rawData(), "?array<?array>"_sd}, + {kBinDataTypeString.rawData(), "?array<?binData>"_sd}, + {kObjectIdTypeString.rawData(), "?array<?objectId>"_sd}, + {kBoolTypeString.rawData(), "?array<?bool>"_sd}, + {kDateTypeString.rawData(), "?array<?date>"_sd}, + {kNullTypeString.rawData(), "?array<?null>"_sd}, + {kRegexTypeString.rawData(), "?array<?regex>"_sd}, + {kDbPointerTypeString.rawData(), "?array<?dbPointer>"_sd}, + {kJavascriptTypeString.rawData(), "?array<?javascript>"_sd}, + {kJavascriptWithScopeTypeString.rawData(), "?array<?javascriptWithScope>"_sd}, + {kTimestampTypeString.rawData(), "?array<?timestamp>"_sd}, + {kMaxKeyTypeString.rawData(), "?array<?maxKey>"_sd}, +}; + +static constexpr auto kRepresentativeString = "?"_sd; +static constexpr auto kRepresentativeNumber = 1; +static const auto kRepresentativeObject = BSON("?" + << "?"); +static const auto kRepresentativeArray = BSONArray(); +static constexpr auto kRepresentativeBinData = BSONBinData(); +static const auto kRepresentativeObjectId = OID::max(); +static constexpr auto kRepresentativeBool = true; +static const auto kRepresentativeDate = Date_t::fromMillisSinceEpoch(0); +static const auto kRepresentativeRegex = BSONRegEx("/\?/"); +static const auto kRepresentativeDbPointer = BSONDBRef("?.?", OID::max()); +static const auto kRepresentativeJavascript = BSONCode("return ?;"); +static const auto kRepresentativeJavascriptWithScope = BSONCodeWScope("return ?;", BSONObj()); +static const auto kRepresentativeTimestamp = Timestamp::min(); + +/** + * A default redaction strategy that generates easy to check results for testing purposes. + */ +std::string applyHmacForTest(StringData s) { + // Avoid ending in a parenthesis since the results will occur in a raw string where the )" + // sequence will accidentally terminate the string. + return str::stream() << "HASH<" << s << ">"; +} + +/** + * Computes a debug string meant to represent "any value of type t", where "t" is the type of the + * provided argument. For example "?number" for any number (int, double, etc.). + */ +StringData debugTypeString(BSONType t) { + // This is tightly coupled with 'canonicalizeBSONType' and therefore also with + // sorting/comparison semantics. + switch (t) { + case EOO: + case Undefined: + return kUndefinedTypeString; + case Symbol: + case String: + return kStringTypeString; + case NumberInt: + case NumberLong: + case NumberDouble: + case NumberDecimal: + return kNumberTypeString; + case MinKey: + return kMinKeyTypeString; + case Object: + return kObjectTypeString; + case Array: + // This case should only happen if we have an array within an array. + return kArrayTypeString; + case BinData: + return kBinDataTypeString; + case jstOID: + return kObjectIdTypeString; + case Bool: + return kBoolTypeString; + case Date: + return kDateTypeString; + case jstNULL: + return kNullTypeString; + case RegEx: + return kRegexTypeString; + case DBRef: + return kDbPointerTypeString; + case Code: + return kJavascriptTypeString; + case CodeWScope: + return kJavascriptWithScopeTypeString; + case bsonTimestamp: + return kTimestampTypeString; + case MaxKey: + return kMaxKeyTypeString; + default: + MONGO_UNREACHABLE_TASSERT(7539806); + } +} + +/** + * Returns an arbitrary value of the same type as the one given. For any number, this will be the + * number 1. For any boolean this will be true. + * TODO if you need a different value to make sure it will parse, you should not use this API. + */ +ImplicitValue defaultLiteralOfType(BSONType t) { + // This is tightly coupled with 'canonicalizeBSONType' and therefore also with + // sorting/comparison semantics. + switch (t) { + case EOO: + case Undefined: + return BSONUndefined; + case Symbol: + case String: + return kRepresentativeString; + case NumberInt: + case NumberLong: + case NumberDouble: + case NumberDecimal: + return kRepresentativeNumber; + case MinKey: + return MINKEY; + case Object: + return kRepresentativeObject; + case Array: + // This case should only happen if we have an array within an array. + return kRepresentativeArray; + case BinData: + return kRepresentativeBinData; + case jstOID: + return kRepresentativeObjectId; + case Bool: + return kRepresentativeBool; + case Date: + return kRepresentativeDate; + case jstNULL: + return BSONNULL; + case RegEx: + return kRepresentativeRegex; + case DBRef: + return kRepresentativeDbPointer; + case Code: + return kRepresentativeJavascript; + case CodeWScope: + return kRepresentativeJavascriptWithScope; + case bsonTimestamp: + return kRepresentativeTimestamp; + case MaxKey: + return MAXKEY; + default: + MONGO_UNREACHABLE_TASSERT(7539803); + } +} + +/** + * A struct representing the sub-type information for an array. + */ +struct ArraySubtypeInfo { + /** + * Whether the values of an array are all the same BSON type or not (mixed). + */ + enum class NTypes { kEmpty, kOneType, kMixed }; + ArraySubtypeInfo(NTypes nTypes_) : nTypes(nTypes_) {} + ArraySubtypeInfo(BSONType oneType) : nTypes(NTypes::kOneType), singleType(oneType) {} + + NTypes nTypes; + boost::optional<BSONType> singleType = boost::none; +}; + +template <typename ValueType> +using GetTypeFn = std::function<BSONType(ValueType)>; + +static GetTypeFn<BSONElement> getBSONElementType = [](const BSONElement& e) { return e.type(); }; +static GetTypeFn<Value> getValueType = [](const Value& v) { return v.getType(); }; + +/** + * Scans 'arrayOfValues' to see if all values are of the same type or not. Returns this info in a + * struct - see the struct definition for how it is represented. + * + * Templated algorithm to handle both iterators of BSONElements or iterators of Values. + * 'getTypeCallback' is provided to abstract away the different '.type()' vs '.getType()' APIs. + */ +template <typename ArrayType, typename ValueType> +ArraySubtypeInfo determineArraySubType(const ArrayType& arrayOfValues, + GetTypeFn<ValueType> getTypeCallback) { + boost::optional<BSONType> firstType = boost::none; + for (auto&& v : arrayOfValues) { + if (!firstType) { + firstType.emplace(getTypeCallback(v)); + } else if (*firstType != getTypeCallback(v)) { + return {ArraySubtypeInfo::NTypes::kMixed}; + } + } + return firstType ? ArraySubtypeInfo{*firstType} + : ArraySubtypeInfo{ArraySubtypeInfo::NTypes::kEmpty}; +} + +ArraySubtypeInfo determineArraySubType(const BSONObj& arrayAsObj) { + return determineArraySubType<BSONObj, BSONElement>(arrayAsObj, getBSONElementType); +} +ArraySubtypeInfo determineArraySubType(const std::vector<Value>& values) { + return determineArraySubType<std::vector<Value>, Value>(values, getValueType); +} + +template <typename ValueType> +StringData debugTypeString( + const ValueType& v, + GetTypeFn<ValueType> getTypeCallback, + std::function<ArraySubtypeInfo(ValueType)> determineArraySubTypeCallback) { + if (getTypeCallback(v) == BSONType::Array) { + // Iterating the array as .Obj(), as if it were a BSONObj (with field names '0', '1', etc.) + // is faster than converting the whole thing to an array which would force a copy. + auto typeInfo = determineArraySubTypeCallback(v); + switch (typeInfo.nTypes) { + case ArraySubtypeInfo::NTypes::kEmpty: + return "[]"_sd; + case ArraySubtypeInfo::NTypes::kOneType: + return kArrayTypeStringConstants.at(debugTypeString(*typeInfo.singleType)); + case ArraySubtypeInfo::NTypes::kMixed: + return "?array<>"; + default: + MONGO_UNREACHABLE_TASSERT(7539801); + } + } + return debugTypeString(getTypeCallback(v)); +} + +template <typename ValueType> +ImplicitValue defaultLiteralOfType( + const ValueType& v, + GetTypeFn<ValueType> getTypeCallback, + std::function<ArraySubtypeInfo(ValueType)> determineArraySubTypeCallback) { + if (getTypeCallback(v) == BSONType::Array) { + auto typeInfo = determineArraySubTypeCallback(v); + switch (typeInfo.nTypes) { + case ArraySubtypeInfo::NTypes::kEmpty: + return BSONArray(); + case ArraySubtypeInfo::NTypes::kOneType: + return std::vector<Value>{defaultLiteralOfType(*typeInfo.singleType)}; + case ArraySubtypeInfo::NTypes::kMixed: + // We don't care which types, we'll use a number and a string as the canonical + // mixed type array regardless. This is to ensure we don't get 2^N possibilities + // for mixed type scenarios - we wish to collapse all "mixed type" arrays to one + // canonical mix. The choice of int and string is mostly arbitrary - hopefully + // somewhat comprehensible at a glance. + return std::vector<Value>{Value(2), Value("or more types"_sd)}; + default: + MONGO_UNREACHABLE_TASSERT(7539805); + } + } + return defaultLiteralOfType(getTypeCallback(v)); +} + +ArraySubtypeInfo getSubTypeFromBSONElemArray(BSONElement arrayElem) { + // Iterating the array as .Obj(), as if it were a BSONObj (with field names '0', '1', etc.) + // is faster than converting the whole thing to an array which would force a copy. + return determineArraySubType(arrayElem.Obj()); +} +ArraySubtypeInfo getSubTypeFromValueArray(const Value& arrayVal) { + return determineArraySubType(arrayVal.getArray()); +} + +void appendDefaultOfNonArrayType(BSONObjBuilder* bob, StringData name, const BSONElement& e) { + switch (e.type()) { + case EOO: + case Undefined: + bob->appendUndefined(name); + return; + case Symbol: + case String: + bob->append(name, kRepresentativeString); + return; + case NumberInt: + case NumberLong: + case NumberDouble: + case NumberDecimal: + bob->append(name, kRepresentativeNumber); + return; + case MinKey: + bob->appendMinKey(name); + return; + case Object: + bob->append(name, kRepresentativeObject); + return; + case Array: + // This case is more complicated and callers should use a more generic helper. + MONGO_UNREACHABLE_TASSERT(8094100); + case BinData: + bob->append(name, kRepresentativeBinData); + return; + case jstOID: + bob->append(name, kRepresentativeObjectId); + return; + case Bool: + bob->append(name, kRepresentativeBool); + return; + case Date: + bob->append(name, kRepresentativeDate); + return; + case jstNULL: + bob->appendNull(name); + return; + case RegEx: + bob->append(name, kRepresentativeRegex); + return; + case DBRef: + bob->append(name, kRepresentativeDbPointer); + return; + case Code: + bob->append(name, kRepresentativeJavascript); + return; + case CodeWScope: + bob->append(name, kRepresentativeJavascriptWithScope); + return; + case bsonTimestamp: + bob->append(name, kRepresentativeTimestamp); + return; + case MaxKey: + bob->appendMaxKey(name); + return; + default: + MONGO_UNREACHABLE_TASSERT(8094101); + }; +} +} // namespace + +const SerializationOptions SerializationOptions::kRepresentativeQueryShapeSerializeOptions = + SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + +const SerializationOptions SerializationOptions::kDebugQueryShapeSerializeOptions = + SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString}; + +SerializationOptions::SerializationOptions(LiteralSerializationPolicy policy) + : literalPolicy(policy) {} +SerializationOptions::SerializationOptions( + boost::optional<ExplainOptions::Verbosity> explainVerbosity) + : verbosity(explainVerbosity) {} + +SerializationOptions::SerializationOptions(LiteralSerializationPolicy policy, + bool transformIdentifiers, + TokenizeIdentifierFunc transformIdentifiersCallbackFn) + : literalPolicy(policy), + transformIdentifiers(transformIdentifiers), + transformIdentifiersCallback(transformIdentifiersCallbackFn) {} + +const SerializationOptions SerializationOptions::kMarkIdentifiers_FOR_TEST{ + LiteralSerializationPolicy::kUnchanged, true, applyHmacForTest}; + +const SerializationOptions SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST{ + LiteralSerializationPolicy::kToDebugTypeString, true, applyHmacForTest}; + +// Overloads for BSONElem and Value. +StringData debugTypeString(BSONElement e) { + return debugTypeString<BSONElement>(e, getBSONElementType, getSubTypeFromBSONElemArray); +} +StringData debugTypeString(const Value& v) { + return debugTypeString<Value>(v, getValueType, getSubTypeFromValueArray); +} + +// Overloads for BSONElem and Value. +ImplicitValue defaultLiteralOfType(const Value& v) { + return defaultLiteralOfType<Value>(v, getValueType, getSubTypeFromValueArray); +} +ImplicitValue defaultLiteralOfType(BSONElement e) { + return defaultLiteralOfType<BSONElement>(e, getBSONElementType, getSubTypeFromBSONElemArray); +} + +void SerializationOptions::appendLiteral(BSONObjBuilder* bob, const BSONElement& e) const { + appendLiteral(bob, e.fieldNameStringData(), e); +} +void SerializationOptions::appendLiteral(BSONObjBuilder* bob, + StringData name, + const BSONElement& e) const { + // The first two cases are particularly performance sensitive. We could answer everything here + // with the code inside the 'kToDebugTypeString' branch, but there are some relatively easy ways + // to accomplish the first two policy cases (in the common cases), so we'll special case those + // in order to avoid constructing a temporary Value. + switch (literalPolicy) { + case LiteralSerializationPolicy::kUnchanged: + bob->appendAs(e, name); + return; + case LiteralSerializationPolicy::kToRepresentativeParseableValue: { + if (e.type() != BSONType::Array) { + appendDefaultOfNonArrayType(bob, name, e); + return; + } + // If it's an array we'll default to the slow but general codepath below. + [[fallthrough]]; + } + case LiteralSerializationPolicy::kToDebugTypeString: { + // Performance isn't as sensitive here. + return serializeLiteral(e).addToBsonObj(bob, name); + } + default: + MONGO_UNREACHABLE_TASSERT(8094102); + } +} + +void SerializationOptions::appendLiteral(BSONObjBuilder* bob, + StringData fieldName, + const ImplicitValue& v, + const boost::optional<Value>& representativeValue) const { + serializeLiteral(v, representativeValue).addToBsonObj(bob, fieldName); +} + +Value SerializationOptions::serializeLiteral( + const BSONElement& e, const boost::optional<Value>& representativeValue) const { + switch (literalPolicy) { + case LiteralSerializationPolicy::kUnchanged: + return Value(e); + case LiteralSerializationPolicy::kToDebugTypeString: + return Value(debugTypeString(e)); + case LiteralSerializationPolicy::kToRepresentativeParseableValue: + return representativeValue.value_or(defaultLiteralOfType(e)); + default: + MONGO_UNREACHABLE_TASSERT(7539802); + } +} + +Value SerializationOptions::serializeLiteral( + const ImplicitValue& v, const boost::optional<Value>& representativeValue) const { + switch (literalPolicy) { + case LiteralSerializationPolicy::kUnchanged: + return v; + case LiteralSerializationPolicy::kToDebugTypeString: + return Value(debugTypeString(v)); + case LiteralSerializationPolicy::kToRepresentativeParseableValue: + return representativeValue.value_or(defaultLiteralOfType(v)); + default: + MONGO_UNREACHABLE_TASSERT(7539804); + } +} + +std::string SerializationOptions::serializeFieldPathFromString(StringData path) const { + if (transformIdentifiers) { + try { + return serializeFieldPath(FieldPath(path, false)); + } catch (DBException& ex) { + LOGV2_DEBUG(7549808, + 1, + "Failed to convert a path string to a FieldPath", + "pathString"_attr = path, + "failure"_attr = ex.toStatus()); + return serializeFieldPath("invalidFieldPathPlaceholder"); + } + } + return path.toString(); +} +} // namespace mongo diff --git a/src/mongo/db/query/query_shape/serialization_options.h b/src/mongo/db/query/query_shape/serialization_options.h new file mode 100644 index 00000000000..226da7689d3 --- /dev/null +++ b/src/mongo/db/query/query_shape/serialization_options.h @@ -0,0 +1,236 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/exec/document_value/document.h" +#include "mongo/db/exec/document_value/value.h" +#include "mongo/db/pipeline/field_path.h" +#include "mongo/db/query/explain_options.h" +#include "mongo/util/assert_util.h" +#include <boost/optional.hpp> +#include <string> + +namespace mongo { +namespace { +// Should never be called, throw to ensure we catch this in tests. +std::string defaultHmacStrategy(StringData s) { + MONGO_UNREACHABLE_TASSERT(7332410); +} +} // namespace + +/** + * A policy enum for how to serialize literal values. + */ +enum class LiteralSerializationPolicy { + // The default way to serialize. Just serialize whatever literals were given if they are still + // available, or whatever you parsed them to. This is expected to be able to parse again, since + // it worked the first time. + kUnchanged, + // Serialize any literal value as "?number" or similar. For example "?bool" for any boolean. Use + // 'debugTypeString()' helper. + kToDebugTypeString, + // Serialize any literal value to one canonical value of the given type, with the constraint + // that the chosen representative value should be parseable in this context. There are some + // default implementations that will usually work (e.g. using the number 1 almost always works + // for numbers), but serializers should be careful to think about and test this if their parsers + // reject certain values. + kToRepresentativeParseableValue, +}; + +/** + * A struct with options for how you want to serialize a match or aggregation expression. + */ +struct SerializationOptions { + using TokenizeIdentifierFunc = std::function<std::string(StringData)>; + + // The default serialization options for a query shape. No need to redact identifiers for the + // this purpose. We may do that on the $queryStats read path. + static const SerializationOptions kRepresentativeQueryShapeSerializeOptions; + static const SerializationOptions kDebugQueryShapeSerializeOptions; + static const SerializationOptions kMarkIdentifiers_FOR_TEST; + static const SerializationOptions kDebugShapeAndMarkIdentifiers_FOR_TEST; + + SerializationOptions() = default; + SerializationOptions(LiteralSerializationPolicy policy); + SerializationOptions(boost::optional<ExplainOptions::Verbosity> explain); + SerializationOptions(LiteralSerializationPolicy policy, + bool transformIdentifiers, + TokenizeIdentifierFunc transformIdentifiersCallbackFn); + + /** + * Checks if this SerializationOptions represents the same options as another + * SerializationOptions. Note it cannot compare whether the two 'transformIdentifiersCallback's + * are the same - the language purposefully leaves the comparison operator undefined. + */ + bool operator==(const SerializationOptions& other) const { + return this->transformIdentifiers == other.transformIdentifiers && + // You cannot well determine std::function equivalence in C++, so this is the best we'll + // do. + (this->transformIdentifiersCallback == nullptr) == + (other.transformIdentifiersCallback == nullptr) && + this->literalPolicy == other.literalPolicy && this->verbosity == other.verbosity; + } + bool operator!=(const SerializationOptions& other) const { + return !(*this == other); + } + + // Helper function for removing identifiable information (like collection/db names). + // Note: serializeFieldPath/serializeFieldPathFromString should be used for field + // names. + std::string serializeIdentifier(StringData str) const { + if (transformIdentifiers) { + return transformIdentifiersCallback(str); + } + return str.toString(); + } + + std::string serializeFieldPath(FieldPath path) const { + if (transformIdentifiers) { + std::stringstream hmaced; + for (size_t i = 0; i < path.getPathLength(); ++i) { + if (i > 0) { + hmaced << "."; + } + hmaced << transformIdentifiersCallback(path.getFieldName(i)); + } + return hmaced.str(); + } + return path.fullPath(); + } + + std::string serializeFieldPathWithPrefix(FieldPath path) const { + return "$" + serializeFieldPath(path); + } + + std::string serializeFieldPathFromString(StringData path) const; + + std::vector<std::string> serializeFieldPathFromString( + const std::vector<std::string>& paths) const { + std::vector<std::string> result; + result.reserve(paths.size()); + for (auto& p : paths) { + result.push_back(serializeFieldPathFromString(p)); + } + return result; + } + + // Helper functions for applying hmac to BSONObj. Does not take into account anything to do with + // MQL semantics, removes all field names and literals in the passed in obj. + void addHmacedArrayToBuilder(BSONArrayBuilder* bab, std::vector<BSONElement> array) const { + for (const auto& elem : array) { + if (elem.type() == BSONType::Object) { + BSONObjBuilder subObj(bab->subobjStart()); + addHmacedObjToBuilder(&subObj, elem.Obj()); + subObj.done(); + } else if (elem.type() == BSONType::Array) { + BSONArrayBuilder subArr(bab->subarrayStart()); + addHmacedArrayToBuilder(&subArr, elem.Array()); + subArr.done(); + } else { + *bab << serializeLiteral(elem); + } + } + } + + void addHmacedObjToBuilder(BSONObjBuilder* bob, BSONObj objToHmac) const { + for (const auto& elem : objToHmac) { + auto fieldName = serializeFieldPath(elem.fieldName()); + if (elem.type() == BSONType::Object) { + BSONObjBuilder subObj(bob->subobjStart(fieldName)); + addHmacedObjToBuilder(&subObj, elem.Obj()); + subObj.done(); + } else if (elem.type() == BSONType::Array) { + BSONArrayBuilder subArr(bob->subarrayStart(fieldName)); + addHmacedArrayToBuilder(&subArr, elem.Array()); + subArr.done(); + } else { + appendLiteral(bob, fieldName, elem); + } + } + } + + /** + * Helper method to call 'serializeLiteral()' on 'e' and append the resulting value to 'bob' + * using the same name as 'e'. + */ + void appendLiteral(BSONObjBuilder* bob, const BSONElement& e) const; + void appendLiteral(BSONObjBuilder* bob, StringData name, const BSONElement& e) const; + /** + * Helper method to call 'serializeLiteral()' on 'v' and append the result to 'bob' using field + * name 'fieldName'. + */ + void appendLiteral(BSONObjBuilder* bob, + StringData fieldName, + const ImplicitValue& v, + const boost::optional<Value>& representativeValue = boost::none) const; + + /** + * Depending on the configured 'literalPolicy', serializeLiteral will return the appropriate + * value for adding literals to serialization output: + * - If 'literalPolicy' is 'kUnchanged', returns the input value unmodified. + * - If it is 'kToDebugTypeString', computes and returns the type string as a string Value. + * - If it is 'kToRepresentativeValue', it returns an arbitrary value of the same type as the + * one given. For any number, this will be the number 1. For any boolean this will be true. + * If the 'representativeValue' parameter if it is not none, returns it (regardless of type). + * + * Example usage: BSON("myArg" << options.serializeLiteral(_myArg)); + */ + Value serializeLiteral(const BSONElement& e, + const boost::optional<Value>& representativeValue = boost::none) const; + Value serializeLiteral(const ImplicitValue& v, + const boost::optional<Value>& representativeValue = boost::none) const; + + // 'literalPolicy' is an independent option to serialize in a general format with the aim of + // similar "shaped" queries serializing to the same object. For example, if set to + // 'kToDebugTypeString', then the serialization of {a: {$gt: 2}} should result in {a: {$gt: + // '?number'}}, as will the serialization of {a: {$gt: 3}}. + // + // "Literal" here is meant to stand in contrast to expression arguments, as in the $gt + // expressions in {$and: [{a: {$gt: 3}}, {b: {$gt: 4}}]}. There the only literals are 3 and 4, + // so the serialization expected for 'kToDebugTypeString' would be {$and: [{a: {$gt: + // '?number'}}, {b: {$lt: '?number'}}]}. + LiteralSerializationPolicy literalPolicy = LiteralSerializationPolicy::kUnchanged; + + // If true the caller must set transformIdentifiersCallback. 'transformIdentifiers' if set along + // with a strategy the redaction strategy will be called on any personal identifiable + // information (e.g., field paths/names, collection names) encountered before serializing them. + bool transformIdentifiers = false; + std::function<std::string(StringData)> transformIdentifiersCallback = defaultHmacStrategy; + + // For aggregation indicate whether we should use the more verbose serialization format. + boost::optional<ExplainOptions::Verbosity> verbosity = boost::none; + + // If set to true, serializes each stage and expression as needed for query analysis. + bool serializeForQueryAnalysis = false; +}; + +} // namespace mongo diff --git a/src/mongo/db/query/query_shape/shape_helpers.cpp b/src/mongo/db/query/query_shape/shape_helpers.cpp new file mode 100644 index 00000000000..8eea475ab78 --- /dev/null +++ b/src/mongo/db/query/query_shape/shape_helpers.cpp @@ -0,0 +1,108 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/query/query_shape/shape_helpers.h" + +#include "mongo/db/query/query_shape/query_shape_gen.h" + +namespace mongo::shape_helpers { + +static constexpr StringData hintSpecialField = "$hint"_sd; +// A "Flat" object is one with only top-level fields. We won't descend recursively to shapify any +// sub-objects. +BSONObj shapifyFlatObj(BSONObj obj, const SerializationOptions& opts, bool valuesAreLiterals) { + if (obj.isEmpty()) { + // fast-path for the common case. + return obj; + } + + BSONObjBuilder bob; + for (BSONElement elem : obj) { + if (hintSpecialField.compare(elem.fieldNameStringData()) == 0) { + if (elem.type() == BSONType::String) { + bob.append(hintSpecialField, opts.serializeFieldPathFromString(elem.String())); + } else if (elem.type() == BSONType::Object) { + opts.appendLiteral(&bob, hintSpecialField, elem.Obj()); + } else { + // SERVER-85500: $hint syntax will not be validated if the collection does not + // exist, so we should accept a value that is neither string nor object here. + opts.appendLiteral(&bob, hintSpecialField, elem); + } + continue; + } + + // $natural doesn't need to be redacted. + if (elem.fieldNameStringData().compare(query_request_helper::kNaturalSortField) == 0) { + bob.append(elem); + continue; + } + + if (valuesAreLiterals) { + opts.appendLiteral(&bob, opts.serializeFieldPathFromString(elem.fieldName()), elem); + } else { + bob.appendAs(elem, opts.serializeFieldPathFromString(elem.fieldName())); + } + } + return bob.obj(); +} + +BSONObj extractHintShape(BSONObj hintObj, const SerializationOptions& opts) { + return shapifyFlatObj(hintObj, opts, /* valuesAreLiterals = */ false); +} + +BSONObj extractMinOrMaxShape(BSONObj obj, const SerializationOptions& opts) { + return shapifyFlatObj(obj, opts, /* valuesAreLiterals = */ true); +} + +void appendNamespaceShape(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts) { + bob.append("db", opts.serializeIdentifier(nss.db())); + bob.append("coll", opts.serializeIdentifier(nss.coll())); +} + +NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt) { + tassert(7632900, "cmdNs must be an object.", cmdNsElt.type() == BSONType::Object); + auto cmdNs = query_shape::CommandNamespace::parse("cmdNs"_sd, cmdNsElt.embeddedObject()); + + if (cmdNs.getColl().has_value()) { + tassert(7632903, + "Exactly one of 'uuid' and 'coll' can be defined.", + !cmdNs.getUuid().has_value()); + return NamespaceString(cmdNs.getDb(), cmdNs.getColl().value()); + } else { + tassert(7632904, + "Exactly one of 'uuid' and 'coll' can be defined.", + !cmdNs.getColl().has_value()); + UUID uuid = uassertStatusOK(UUID::parse(cmdNs.getUuid().value().toString())); + return NamespaceStringOrUUID(cmdNs.getDb().toString(), uuid); + } +} + +} // namespace mongo::shape_helpers diff --git a/src/mongo/db/query/query_shape/shape_helpers.h b/src/mongo/db/query/query_shape/shape_helpers.h new file mode 100644 index 00000000000..4d0fadb4a47 --- /dev/null +++ b/src/mongo/db/query/query_shape/shape_helpers.h @@ -0,0 +1,101 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/bson/simple_bsonobj_comparator.h" +#include "mongo/db/query/query_shape/query_shape.h" +#include "mongo/db/query/query_shape/serialization_options.h" + +namespace mongo::shape_helpers { + +int64_t inline optionalObjSize(boost::optional<BSONObj> optionalObj) { + if (!optionalObj) + return 0; + return optionalObj->objsize(); +} + +template <typename T> +int64_t optionalSize(boost::optional<T> optionalVal) { + if (!optionalVal) + return 0; + return optionalVal->size(); +} + +template <typename T> +std::function<size_t(size_t, const T&)> sizeAccumulatorFunc() { + MONGO_UNREACHABLE; // Don't know how to compute the size of this template type. +}; + +template <> +inline std::function<size_t(size_t, const BSONObj&)> sizeAccumulatorFunc<BSONObj>() { + return [](size_t total, const BSONObj& obj) { + return total + sizeof(BSONObj) + static_cast<size_t>(obj.objsize()); + }; +} + +template <> +inline std::function<size_t(size_t, const NamespaceString&)> +sizeAccumulatorFunc<NamespaceString>() { + return [](size_t total, const NamespaceString& nss) { + // For each element, we have to track the size of the + // nss as well as the size allocated by the nss. It would be + // ideal to be able to ask the underlying namespace string for + // its capacity, but it's not something we have access to. + // Further, namespace strings appear to shrink to fit (i.e + // resize to correct size), so it may not be necessary. Should + // we also try to consider short string optimization? At the + // very least, the current approach gives us a good upper bound + // memory usage (assuming shrink to fit). + return total + sizeof(nss) + nss.size(); + }; +} + +template <typename Container> +size_t containerSize(const Container& container) { + return std::accumulate(container.begin(), + container.end(), + 0, + sizeAccumulatorFunc<typename Container::value_type>()); +} + +/** + * Serializes the given 'hintObj' in accordance with the options. Assumes the hint is correct and + * contains field names. It is possible that this hint doesn't actually represent an index, but we + * can't detect that here. + */ +BSONObj extractHintShape(BSONObj hintObj, const SerializationOptions& opts); +BSONObj extractMinOrMaxShape(BSONObj obj, const SerializationOptions& opts); + +NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt); +void appendNamespaceShape(BSONObjBuilder& bob, + const NamespaceString& nss, + const SerializationOptions& opts); + +} // namespace mongo::shape_helpers |