diff options
Diffstat (limited to 'src/mongo/db/pipeline')
256 files changed, 7582 insertions, 1811 deletions
diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index f5ad3d7cdb7..f3b54ed67a3 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -63,12 +63,10 @@ env.Library( ) env.Library( - target='aggregation_request_helper', - source=[ - 'aggregation_request_helper.cpp', + target='aggregation_request_helper', source=[ 'aggregate_command.idl', - ], - LIBDEPS=[ + 'aggregation_request_helper.cpp', + ], LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/exec/document_value/document_value', '$BUILD_DIR/mongo/db/namespace_string', @@ -80,8 +78,7 @@ env.Library( '$BUILD_DIR/mongo/db/write_concern_options', '$BUILD_DIR/mongo/idl/idl_parser', 'document_sources_idl', - ] -) + ]) env.Library( target='variable_validation', @@ -95,6 +92,7 @@ env.Library( env.Library( target='expression_context', source=[ + '$BUILD_DIR/mongo/db/query/query_shape/cmd_with_let_shape.cpp', 'expression.cpp', 'expression_context.cpp', 'expression_function.cpp', @@ -115,6 +113,7 @@ env.Library( '$BUILD_DIR/mongo/db/query/collation/collator_factory_interface', '$BUILD_DIR/mongo/db/query/datetime/date_time_support', '$BUILD_DIR/mongo/db/query/query_knobs', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape_common', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/stats/counters', @@ -274,6 +273,8 @@ pipelineEnv.Library( 'abt/field_map_builder.cpp', 'abt/match_expression_visitor.cpp', 'abt/utils.cpp', + '$BUILD_DIR/mongo/db/query/query_shape/agg_cmd_shape.cpp', + '$BUILD_DIR/mongo/db/query/query_stats/agg_key.cpp', 'document_source.cpp', 'document_source_add_fields.cpp', 'document_source_bucket.cpp', @@ -298,6 +299,7 @@ pipelineEnv.Library( 'document_source_internal_inhibit_optimization.cpp', 'document_source_internal_shard_filter.cpp', 'document_source_internal_shred_documents.cpp', + 'document_source_internal_shardserver_info.cpp', 'document_source_internal_split_pipeline.cpp', 'document_source_internal_unpack_bucket.cpp', 'document_source_limit.cpp', @@ -312,6 +314,7 @@ pipelineEnv.Library( 'document_source_out.cpp', 'document_source_plan_cache_stats.cpp', 'document_source_project.cpp', + 'document_source_query_stats.cpp', 'document_source_queue.cpp', 'document_source_redact.cpp', 'document_source_replace_root.cpp', @@ -350,7 +353,6 @@ pipelineEnv.Library( '$BUILD_DIR/mongo/client/clientdriver_minimal', '$BUILD_DIR/mongo/db/auth/auth', '$BUILD_DIR/mongo/db/bson/dotted_path_support', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/curop_failpoint_helpers', '$BUILD_DIR/mongo/db/exec/bucket_unpacker', '$BUILD_DIR/mongo/db/exec/document_value/document_value', @@ -483,11 +485,14 @@ env.Library( 'document_source_merge_modes.idl', 'document_source_merge_spec.cpp', 'document_source_parsing_validators.cpp', + 'document_source_query_stats.idl', + 'document_source_query_stats_validators.cpp', 'document_source_replace_root.idl', 'document_source_set_window_fields.idl', 'document_source_union_with.idl', 'document_source_set_variable_from_subpipeline.idl', 'exchange_spec.idl', + 'resume_token.idl', 'resume_token.cpp', 'storage_stats_spec.idl', 'value.idl', @@ -495,6 +500,8 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/exec/document_value/document_value', + '$BUILD_DIR/mongo/db/query/query_shape/query_shape', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats_parse', '$BUILD_DIR/mongo/db/storage/key_string', '$BUILD_DIR/mongo/idl/idl_parser', '$BUILD_DIR/mongo/s/common_s', @@ -580,9 +587,11 @@ env.CppUnitTest( 'document_source_change_stream_add_post_image_test.cpp', 'document_source_change_stream_test.cpp', 'document_source_check_resume_token_test.cpp', + 'document_source_coll_stats_test.cpp', 'document_source_count_test.cpp', 'document_source_current_op_test.cpp', 'document_source_densify_test.cpp', + 'document_source_documents_test.cpp', 'document_source_exchange_test.cpp', 'document_source_facet_test.cpp', 'document_source_find_and_modify_image_lookup_test.cpp', @@ -611,8 +620,11 @@ env.CppUnitTest( 'document_source_skip_test.cpp', 'document_source_sort_by_count_test.cpp', 'document_source_sort_test.cpp', + 'document_source_query_stats_test.cpp', 'document_source_union_with_test.cpp', + 'document_source_internal_all_collection_stats_test.cpp', 'document_source_internal_compute_geo_near_distance_test.cpp', + 'document_source_internal_convert_bucket_index_stats_test.cpp', 'document_source_internal_unpack_bucket_test/extract_or_build_project_to_internalize_test.cpp', 'document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp', 'document_source_internal_unpack_bucket_test/extract_project_for_pushdown_test.cpp', @@ -633,7 +645,9 @@ env.CppUnitTest( 'expression_date_test.cpp', 'expression_field_path_test.cpp', 'expression_find_internal_test.cpp', + 'expression_function_test.cpp', 'expression_javascript_test.cpp' if get_option('js-engine') != 'none' else [], + 'expression_let_test.cpp', 'expression_nary_test.cpp', 'expression_object_test.cpp', 'expression_or_test.cpp', diff --git a/src/mongo/db/pipeline/abt/abt_document_source_visitor.cpp b/src/mongo/db/pipeline/abt/abt_document_source_visitor.cpp index 1d136a098ce..a2bf9a28656 100644 --- a/src/mongo/db/pipeline/abt/abt_document_source_visitor.cpp +++ b/src/mongo/db/pipeline/abt/abt_document_source_visitor.cpp @@ -59,6 +59,7 @@ #include "mongo/db/pipeline/document_source_operation_metrics.h" #include "mongo/db/pipeline/document_source_out.h" #include "mongo/db/pipeline/document_source_plan_cache_stats.h" +#include "mongo/db/pipeline/document_source_query_stats.h" #include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/document_source_redact.h" #include "mongo/db/pipeline/document_source_replace_root.h" @@ -606,6 +607,10 @@ public: unsupportedStage(source); } + void visit(const DocumentSourceQueryStats* source) override { + unsupportedStage(source); + } + void visit(const DocumentSourceUnionWith* source) override { auto entry = _ctx.getNode(); ProjectionName unionProjName = entry._rootProjection; diff --git a/src/mongo/db/pipeline/accumulator.h b/src/mongo/db/pipeline/accumulator.h index 114ccc260ce..0fbc3c60baf 100644 --- a/src/mongo/db/pipeline/accumulator.h +++ b/src/mongo/db/pipeline/accumulator.h @@ -42,6 +42,7 @@ #include "mongo/db/exec/document_value/value_comparator.h" #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/expression_context.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/stdx/unordered_set.h" #include "mongo/util/summation.h" @@ -134,12 +135,12 @@ public: */ virtual Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options = {}) const { ExpressionConstant const* ec = dynamic_cast<ExpressionConstant const*>(initializer.get()); invariant(ec); invariant(ec->getValue().nullish()); - return DOC(getOpName() << argument->serialize(explain)); + return DOC(getOpName() << argument->serialize(options)); } virtual AccumulatorDocumentsNeeded documentsNeeded() const { diff --git a/src/mongo/db/pipeline/accumulator_js_reduce.cpp b/src/mongo/db/pipeline/accumulator_js_reduce.cpp index 78058d5913d..bd8c5ccf882 100644 --- a/src/mongo/db/pipeline/accumulator_js_reduce.cpp +++ b/src/mongo/db/pipeline/accumulator_js_reduce.cpp @@ -196,8 +196,9 @@ void AccumulatorInternalJsReduce::reset() { // Returns this accumulator serialized as a Value along with the reduce function. Document AccumulatorInternalJsReduce::serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { - return DOC(kName << DOC("data" << argument->serialize(explain) << "eval" << _funcSource)); + const SerializationOptions& options) const { + return DOC(kName << DOC("data" << argument->serialize(options) << "eval" + << options.serializeLiteral(_funcSource))); } REGISTER_ACCUMULATOR(accumulator, AccumulatorJs::parse); @@ -235,15 +236,16 @@ std::string parseFunction(StringData fieldName, Document AccumulatorJs::serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options) const { MutableDocument args; - args.addField("init", Value(_init)); - args.addField("initArgs", Value(initializer->serialize(explain))); - args.addField("accumulate", Value(_accumulate)); - args.addField("accumulateArgs", Value(argument->serialize(explain))); - args.addField("merge", Value(_merge)); + + args.addField("init", options.serializeLiteral(_init)); + args.addField("initArgs", initializer->serialize(options)); + args.addField("accumulate", options.serializeLiteral(_accumulate)); + args.addField("accumulateArgs", argument->serialize(options)); + args.addField("merge", options.serializeLiteral(_merge)); if (_finalize) { - args.addField("finalize", Value(*_finalize)); + args.addField("finalize", options.serializeLiteral(*_finalize)); } args.addField("lang", Value("js"_sd)); return DOC(kName << args.freeze()); diff --git a/src/mongo/db/pipeline/accumulator_js_reduce.h b/src/mongo/db/pipeline/accumulator_js_reduce.h index af025132ee0..9e3a6e63d40 100644 --- a/src/mongo/db/pipeline/accumulator_js_reduce.h +++ b/src/mongo/db/pipeline/accumulator_js_reduce.h @@ -66,7 +66,7 @@ public: virtual Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const override; + const SerializationOptions& options = {}) const override; private: static std::string parseReduceFunction(BSONElement func); @@ -101,10 +101,9 @@ public: void processInternal(const Value& input, bool merging) final; void reduceMemoryConsumptionIfAble() final; - Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const final; + const SerializationOptions& options = {}) const final; void startNewGroup(Value const& input) final; private: diff --git a/src/mongo/db/pipeline/accumulator_multi.cpp b/src/mongo/db/pipeline/accumulator_multi.cpp index d488bd8789c..26cfa570d91 100644 --- a/src/mongo/db/pipeline/accumulator_multi.cpp +++ b/src/mongo/db/pipeline/accumulator_multi.cpp @@ -178,9 +178,9 @@ const char* AccumulatorMinMaxN::getOpName() const { Document AccumulatorMinMaxN::serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options) const { MutableDocument args; - AccumulatorN::serializeHelper(initializer, argument, explain, args); + AccumulatorN::serializeHelper(initializer, argument, options, args); return DOC(getOpName() << args.freeze()); } @@ -230,10 +230,10 @@ void AccumulatorN::updateAndCheckMemUsage(size_t memAdded) { void AccumulatorN::serializeHelper(const boost::intrusive_ptr<Expression>& initializer, const boost::intrusive_ptr<Expression>& argument, - bool explain, + const SerializationOptions& options, MutableDocument& md) { - md.addField(kFieldNameN, Value(initializer->serialize(explain))); - md.addField(kFieldNameInput, Value(argument->serialize(explain))); + md.addField(kFieldNameN, Value(initializer->serialize(options))); + md.addField(kFieldNameInput, Value(argument->serialize(options))); } template <MinMaxSense s> @@ -385,9 +385,9 @@ const char* AccumulatorFirstLastN::getOpName() const { Document AccumulatorFirstLastN::serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options) const { MutableDocument args; - AccumulatorN::serializeHelper(initializer, argument, explain, args); + AccumulatorN::serializeHelper(initializer, argument, options, args); return DOC(getOpName() << args.freeze()); } @@ -531,32 +531,46 @@ template <TopBottomSense sense, bool single> Document AccumulatorTopBottomN<sense, single>::serialize( boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const { + const SerializationOptions& options) const { MutableDocument args; if constexpr (!single) { - args.addField(kFieldNameN, Value(initializer->serialize(explain))); + args.addField(kFieldNameN, Value(initializer->serialize(options))); } - auto serializedArg = argument->serialize(explain); - - // If 'argument' contains a field named 'output', this means that we are serializing the - // accumulator's original output expression under the field name 'output'. Otherwise, we are - // serializing a custom argument under the field name 'output'. For instance, a merging $group - // will provide an argument that merges multiple partial groups. - if (auto output = serializedArg[kFieldNameOutput]; !output.missing()) { - args.addField(kFieldNameOutput, Value(output)); + + // If 'argument' is either an ExpressionObject or an ExpressionConstant of object type, then + // we are serializing the original expression under the 'output' field of the object. Otherwise, + // we're serializing a custom expression for merging group. + if (auto argObj = dynamic_cast<ExpressionObject*>(argument.get())) { + bool foundOutputField = false; + for (auto& child : argObj->getChildExpressions()) { + if (child.first == kFieldNameOutput) { + auto output = child.second->serialize(options); + args.addField(kFieldNameOutput, output); + foundOutputField = true; + break; + } + } + tassert(7773700, "'output' field should be present.", foundOutputField); + } else if (auto argConst = dynamic_cast<ExpressionConstant*>(argument.get())) { + auto output = argConst->getValue().getDocument()[kFieldNameOutput]; + tassert(7773701, "'output' field should be present.", !output.missing()); + args.addField(kFieldNameOutput, output); } else { + auto serializedArg = argument->serialize(options); args.addField(kFieldNameOutput, serializedArg); } + args.addField(kFieldNameSortBy, Value(_sortPattern.serialize( - SortPattern::SortKeySerialization::kForPipelineSerialization))); + SortPattern::SortKeySerialization::kForPipelineSerialization, options))); return DOC(getOpName() << args.freeze()); } template <TopBottomSense sense> std::pair<SortPattern, BSONArray> parseAccumulatorTopBottomNSortBy(ExpressionContext* const expCtx, BSONObj sortBy) { + SortPattern sortPattern(sortBy, expCtx); BSONArrayBuilder sortFieldsExpBab; BSONObjIterator sortByBoi(sortBy); @@ -568,7 +582,7 @@ std::pair<SortPattern, BSONArray> parseAccumulatorTopBottomNSortBy(ExpressionCon // since the evaluated argument wouldn't have the same metadata as the original // document. Instead we use [{$meta: "textScore"}] as the sortFields expression so the // sortFields array contains the data we need for sorting. - const auto serialized = part.expression->serialize(false); + const auto serialized = part.expression->serialize(); sortFieldsExpBab.append(serialized.getDocument().toBson()); } else { sortFieldsExpBab.append((StringBuilder() << "$" << fieldName).str()); @@ -581,10 +595,8 @@ template <TopBottomSense sense, bool single> AccumulationExpression AccumulatorTopBottomN<sense, single>::parseTopBottomN( ExpressionContext* const expCtx, BSONElement elem, VariablesParseState vps) { auto name = AccumulatorTopBottomN<sense, single>::getName(); - const auto [n, output, sortBy] = accumulatorNParseArgs<single>(expCtx, elem, name.rawData(), true, vps); - auto [sortPattern, sortFieldsExp] = parseAccumulatorTopBottomNSortBy<sense>(expCtx, *sortBy); expCtx->sbeGroupCompatible = false; diff --git a/src/mongo/db/pipeline/accumulator_multi.h b/src/mongo/db/pipeline/accumulator_multi.h index 13e7971ee0c..32ced5b8c4e 100644 --- a/src/mongo/db/pipeline/accumulator_multi.h +++ b/src/mongo/db/pipeline/accumulator_multi.h @@ -34,6 +34,7 @@ #include "mongo/db/exec/sort_key_comparator.h" #include "mongo/db/index/sort_key_generator.h" #include "mongo/db/pipeline/accumulation_statement.h" +#include "mongo/db/query/query_shape/serialization_options.h" namespace mongo { @@ -85,7 +86,7 @@ public: */ static void serializeHelper(const boost::intrusive_ptr<Expression>& initializer, const boost::intrusive_ptr<Expression>& argument, - bool explain, + const SerializationOptions& options, MutableDocument& md); protected: @@ -136,7 +137,7 @@ public: Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const final; + const SerializationOptions& options = {}) const final; void reset() final; @@ -207,7 +208,7 @@ public: Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const final; + const SerializationOptions& options = {}) const final; void reset() final; @@ -322,7 +323,7 @@ public: Document serialize(boost::intrusive_ptr<Expression> initializer, boost::intrusive_ptr<Expression> argument, - bool explain) const final; + const SerializationOptions& options = {}) const final; void reset() final; diff --git a/src/mongo/db/pipeline/accumulator_test.cpp b/src/mongo/db/pipeline/accumulator_test.cpp index b4665b6739c..3f3a4ba9385 100644 --- a/src/mongo/db/pipeline/accumulator_test.cpp +++ b/src/mongo/db/pipeline/accumulator_test.cpp @@ -39,10 +39,12 @@ #include "mongo/db/pipeline/accumulation_statement.h" #include "mongo/db/pipeline/accumulator.h" #include "mongo/db/pipeline/accumulator_for_window_functions.h" +#include "mongo/db/pipeline/accumulator_js_reduce.h" #include "mongo/db/pipeline/accumulator_multi.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/collation/collator_interface_mock.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/dbtests/dbtests.h" #include "mongo/idl/server_parameter_test_util.h" #include "mongo/logv2/log.h" @@ -1869,6 +1871,199 @@ TEST(Accumulators, CovarianceWithRandomVariables) { assertCovariance<AccumulatorCovarianceSamp>(&expCtx, randomVariables, boost::none); } +Value parseAndSerializeAccumExpr( + const BSONObj& obj, + std::function<boost::intrusive_ptr<Expression>( + ExpressionContext* expCtx, BSONElement, const VariablesParseState&)> func) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto expCtx = make_intrusive<ExpressionContextForTest>(); + auto expr = func(expCtx.get(), obj.firstElement(), expCtx->variablesParseState); + return expr->serialize(options); +} + +Document parseAndSerializeAccum( + const BSONElement elem, + std::function<AccumulationExpression( + ExpressionContext* const expCtx, BSONElement, VariablesParseState)> func) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto expCtx = make_intrusive<ExpressionContextForTest>(); + VariablesParseState vps = expCtx->variablesParseState; + + auto expr = func(expCtx.get(), elem, vps); + auto accum = expr.factory(); + return accum->serialize(expr.initializer, expr.argument, options); +} + +Document parseAndSerializeAccumRepresentative( + const BSONElement elem, + std::function<AccumulationExpression( + ExpressionContext* const expCtx, BSONElement, VariablesParseState)> func) { + SerializationOptions options = SerializationOptions::kRepresentativeQueryShapeSerializeOptions; + auto expCtx = make_intrusive<ExpressionContextForTest>(); + VariablesParseState vps = expCtx->variablesParseState; + + auto expr = func(expCtx.get(), elem, vps); + auto accum = expr.factory(); + return accum->serialize(expr.initializer, expr.argument, options); +} + +TEST(Accumulators, SerializeWithRedaction) { + auto jsReduce = + BSON("$accumulator" << BSON("init" + << "function() {}" + << "accumulateArgs" + << BSON_ARRAY("$a" + << "$b") + << "accumulate" + << "function(state, str1, str2) {return str1 + str2;}" + << "merge" + << "function(s1, s2) {return s1 || s2;}" + << "lang" + << "js")); + auto actual = parseAndSerializeAccum(jsReduce.firstElement(), &AccumulatorJs::parse); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$accumulator": { + "init": "?string", + "initArgs": "[]", + "accumulate": "?string", + "accumulateArgs": [ + "$HASH<a>", + "$HASH<b>" + ], + "merge": "?string", + "lang": "js" + } + })", + actual); + + auto topN = BSON("$topN" << BSON("n" << 3 << "output" + << "$output" + << "sortBy" << BSON("sortKey" << 1))); + actual = parseAndSerializeAccum( + topN.firstElement(), &AccumulatorTopBottomN<TopBottomSense::kTop, false>::parseTopBottomN); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$topN": { + "n": "?number", + "output": "$HASH<output>", + "sortBy": { + "HASH<sortKey>": 1 + } + } + })", + actual); + + auto addToSet = BSON("$addToSet" << BSON("a" << 5)); + actual = parseAndSerializeAccum(addToSet.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorAddToSet>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$addToSet":"?object"})", + actual); + + auto sum = BSON("$sum" << BSON_ARRAY(4 << 6)); + actual = parseAndSerializeAccum(sum.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorSum>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$sum": "?array<?number>"})", + actual); + + sum = BSON("$sum" << BSON_ARRAY("$a" << 5 << 3 << BSON("$sum" << BSON_ARRAY(4 << 6)))); + actual = parseAndSerializeAccum(sum.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorSum>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$sum":["$HASH<a>","?number","?number",{"$sum":"?array<?number>"}]})", + actual); + + auto mergeObjs = BSON("$mergeObjects" << BSON_ARRAY("$a" << BSON("b" + << "null"))); + actual = + parseAndSerializeAccum(mergeObjs.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorMergeObjects>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$mergeObjects":["$HASH<a>","?object"]})", + actual); + + auto push = BSON("$push" << BSON("$eq" << BSON_ARRAY("$str" + << "str2"))); + actual = parseAndSerializeAccum(push.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorPush>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$push":{"$eq":["$HASH<str>","?string"]}})", + actual); + + auto top = BSON("$top" << BSON("output" + << "$b" + << "sortBy" << BSON("sales" << 1))); + actual = parseAndSerializeAccum( + top.firstElement(), &AccumulatorTopBottomN<TopBottomSense::kTop, true>::parseTopBottomN); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$top": { + "output": "$HASH<b>", + "sortBy": { + "HASH<sales>": 1 + } + } + })", + actual); + + auto max = BSON("$max" << BSON_ARRAY( + "$a" << 2 << 3 << BSON("$max" << BSON_ARRAY(BSON_ARRAY("$b" << 4 << 5))))); + actual = parseAndSerializeAccum(max.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorMax>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$max": [ + "$HASH<a>", + "?number", + "?number", + { + "$max": [ + [ + "$HASH<b>", + "?number", + "?number" + ] + ] + } + ] + })", + actual); + + auto internalJsReduce = BSON( + "$_internalJsReduce" << BSON("data" + << "$emits" + << "eval" + << "function(key, values) {\n return Array.sum(values);\n")); + actual = parseAndSerializeAccum(internalJsReduce.firstElement(), + &AccumulatorInternalJsReduce::parseInternalJsReduce); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$_internalJsReduce":{"data":"$HASH<emits>","eval":"?string"}})", + actual); +} + +TEST(AccumulatorsToExpression, SerializeWithRedaction) { + auto maxN = BSON("$maxN" << BSON("n" << 3 << "input" << BSON_ARRAY(19 << 7 << 28 << 3 << 5))); + using Sense = AccumulatorMinMax::Sense; + auto actual = + parseAndSerializeAccumExpr(maxN, &AccumulatorMinMaxN::parseExpression<Sense::kMax>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$maxN":{"n":"?number","input":"?array<?number>"}})", + actual.getDocument()); + + auto firstN = BSON("$firstN" << BSON("input" + << "$sales" + << "n" + << "\'string\'")); + using FirstLastSense = AccumulatorFirstLastN::Sense; + actual = parseAndSerializeAccumExpr( + firstN, &AccumulatorFirstLastN::parseExpression<FirstLastSense::kFirst>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$firstN":{"n":"?string","input":"$HASH<sales>"}})", + actual.getDocument()); +} + /* ------------------------- AccumulatorMergeObjects -------------------------- */ TEST(AccumulatorMergeObjects, MergingZeroObjectsShouldReturnEmptyDocument) { @@ -1920,4 +2115,21 @@ TEST(AccumulatorMergeObjects, MergingWithEmptyDocumentShouldIgnore) { assertExpectedResults<AccumulatorMergeObjects>(&expCtx, {{{first, second}, expected}}); } +TEST(AccumulatorMergeObjects, RoundTripSerializationLiteral) { + auto mergeObjs = BSON("$mergeObjects" << BSON("$literal" << BSON_ARRAY(5 << true))); + auto actual = parseAndSerializeAccumRepresentative( + mergeObjs.firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorMergeObjects>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$mergeObjects":{"$const":[2,"or more types"]}})", + actual); + + auto roundTrip = parseAndSerializeAccumRepresentative( + actual.toBson().firstElement(), + &genericParseSingleExpressionAccumulator<AccumulatorMergeObjects>); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$mergeObjects":{"$const":[2,"or more types"]}})", + roundTrip); +} + } // namespace AccumulatorTests diff --git a/src/mongo/db/pipeline/aggregate_command.idl b/src/mongo/db/pipeline/aggregate_command.idl index f577931ff01..67c997dcd5d 100644 --- a/src/mongo/db/pipeline/aggregate_command.idl +++ b/src/mongo/db/pipeline/aggregate_command.idl @@ -66,11 +66,13 @@ types: structs: PassthroughToShardOptions: description: "options for commands requesting a per shard cursor" + query_shape_component: true fields: shard: description: "id of the shard to passthrough to" type: string unstable: true + query_shape: anonymize commands: aggregate: @@ -94,6 +96,14 @@ commands: - privilege: # $planCacheStats resource_pattern: exact_namespace action_type: planCacheRead + - privilege: # $queryStats + agg_stage: queryStats + resource_pattern: cluster + action_type: queryStatsRead + - privilege: # $queryStats + agg_stage: queryStats + resource_pattern: cluster + action_type: queryStatsReadTransformed - privilege: # $changeStream resource_pattern: exact_namespace action_type: changeStream diff --git a/src/mongo/db/pipeline/aggregation_context_fixture.h b/src/mongo/db/pipeline/aggregation_context_fixture.h index a4dfcfeb769..cce0283314c 100644 --- a/src/mongo/db/pipeline/aggregation_context_fixture.h +++ b/src/mongo/db/pipeline/aggregation_context_fixture.h @@ -31,8 +31,10 @@ #include <boost/intrusive_ptr.hpp> #include <memory> +#include <vector> #include "mongo/db/concurrency/locker_noop_client_observer.h" +#include "mongo/db/pipeline/document_source.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/service_context_test_fixture.h" #include "mongo/unittest/temp_dir.h" @@ -71,6 +73,42 @@ public: return _opCtx.get(); } + /* + * Serialize and redact a document source. + */ + BSONObj redact(const DocumentSource& docSource, + bool performRedaction = true, + boost::optional<ExplainOptions::Verbosity> verbosity = boost::none) { + SerializationOptions options; + options.verbosity = verbosity; + if (performRedaction) { + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + options.transformIdentifiersCallback = [](StringData s) -> std::string { + return str::stream() << "HASH<" << s << ">"; + }; + options.transformIdentifiers = true; + } + std::vector<Value> serialized; + docSource.serializeToArray(serialized, options); + ASSERT_EQ(1, serialized.size()); + return serialized[0].getDocument().toBson().getOwned(); + } + + std::vector<Value> redactToArray(const DocumentSource& docSource, + bool performRedaction = true) { + SerializationOptions options; + if (performRedaction) { + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + options.transformIdentifiersCallback = [](StringData s) -> std::string { + return str::stream() << "HASH<" << s << ">"; + }; + options.transformIdentifiers = true; + } + std::vector<Value> serialized; + docSource.serializeToArray(serialized, options); + return serialized; + } + private: ServiceContext::UniqueOperationContext _opCtx; boost::intrusive_ptr<ExpressionContextForTest> _expCtx; diff --git a/src/mongo/db/pipeline/change_stream_constants.h b/src/mongo/db/pipeline/change_stream_constants.h index 94fe6487b52..7b5104e93cd 100644 --- a/src/mongo/db/pipeline/change_stream_constants.h +++ b/src/mongo/db/pipeline/change_stream_constants.h @@ -34,8 +34,17 @@ namespace mongo { namespace change_stream_constants { +namespace stage_names { +static constexpr StringData kEnsureResumeTokenPresent = + "$_internalChangeStreamEnsureResumeTokenPresent"_sd; +static constexpr StringData kHandleTopologyChange = "$_internalChangeStreamHandleTopologyChange"_sd; +} // namespace stage_names -const BSONObj kSortSpec = BSON("_id._data" << 1); +static const BSONObj kSortSpec = BSON("_id._data" << 1); + +// Internal change stream stages that can appear in a router (mongoS) pipeline. +static const StringDataSet kChangeStreamRouterPipelineStages = { + stage_names::kEnsureResumeTokenPresent, stage_names::kHandleTopologyChange}; } // namespace change_stream_constants } // namespace mongo diff --git a/src/mongo/db/pipeline/change_stream_rewrite_helpers.cpp b/src/mongo/db/pipeline/change_stream_rewrite_helpers.cpp index f7703354560..ce4a9234c6a 100644 --- a/src/mongo/db/pipeline/change_stream_rewrite_helpers.cpp +++ b/src/mongo/db/pipeline/change_stream_rewrite_helpers.cpp @@ -227,7 +227,7 @@ boost::intrusive_ptr<Expression> exprRewriteOperationType( opCases.push_back(fromjson("{case: {$ne: ['$o.collMod', '$$REMOVE']}, then: 'modify'}")); // The default case, if nothing matches. - auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(false); + auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(); // Build the final expression object... BSONObjBuilder exprBuilder; @@ -417,7 +417,7 @@ boost::intrusive_ptr<Expression> exprRewriteDocumentKey( fromjson("{case: {$eq: ['$op', 'u']}, then: '" + updateAndReplacePath + "'}")); // The default case, if nothing matches. - auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(false); + auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(); // Build the expression BSON object. BSONObjBuilder exprBuilder; @@ -1159,7 +1159,7 @@ boost::intrusive_ptr<Expression> exprRewriteNs( collCases.push_back(fromjson("{case: {$ne: ['$o.collMod', '$$REMOVE']}, then: '$o.collMod'}")); // The default case, if nothing matches. - auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(false); + auto defaultCase = ExpressionConstant::create(expCtx.get(), Value())->serialize(); // Build the collection expression object... BSONObjBuilder collExprBuilder; @@ -1500,7 +1500,8 @@ std::unique_ptr<MatchExpression> rewriteMatchExpressionTree( case MatchExpression::EXPRESSION: { // Agg expressions are rewritten in-place, so we must clone the expression tree. auto origExprVal = - static_cast<const ExprMatchExpression*>(root)->getExpression()->serialize(false); + static_cast<const ExprMatchExpression*>(root)->getExpression()->serialize( + SerializationOptions{}); auto clonedExpr = Expression::parseOperand( expCtx.get(), BSON("" << origExprVal).firstElement(), expCtx->variablesParseState); @@ -1513,7 +1514,11 @@ std::unique_ptr<MatchExpression> rewriteMatchExpressionTree( } default: { if (auto pathME = dynamic_cast<const PathMatchExpression*>(root)) { - tassert(5687201, "Unexpected empty path", !pathME->path().empty()); + // Only attempt to rewrite non-empty paths. + if (pathME->path().empty()) { + return nullptr; + } + auto firstPath = pathME->fieldRef()->getPart(0).toString(); // Only attempt to rewrite paths that begin with one of the caller-requested fields. diff --git a/src/mongo/db/pipeline/document_path_support_test.cpp b/src/mongo/db/pipeline/document_path_support_test.cpp index 3df55d54e90..489575ba6aa 100644 --- a/src/mongo/db/pipeline/document_path_support_test.cpp +++ b/src/mongo/db/pipeline/document_path_support_test.cpp @@ -42,15 +42,6 @@ #include "mongo/db/pipeline/field_path.h" #include "mongo/unittest/unittest.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString(); \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace mongo { namespace document_path_support { diff --git a/src/mongo/db/pipeline/document_source.cpp b/src/mongo/db/pipeline/document_source.cpp index 55d400af60b..ea1cf479480 100644 --- a/src/mongo/db/pipeline/document_source.cpp +++ b/src/mongo/db/pipeline/document_source.cpp @@ -36,6 +36,7 @@ #include "mongo/db/commands/feature_compatibility_version_documentation.h" #include "mongo/db/exec/document_value/value.h" #include "mongo/db/matcher/expression_algo.h" +#include "mongo/db/pipeline/change_stream_constants.h" #include "mongo/db/pipeline/document_source_add_fields.h" #include "mongo/db/pipeline/document_source_group.h" #include "mongo/db/pipeline/document_source_internal_shard_filter.h" @@ -171,45 +172,77 @@ bool groupMatchSwapVerified(const DocumentSourceMatch& nextMatch, return !expression::hasExistencePredicateOnPath(*(nextMatch.getMatchExpression()), "_id"_sd); } +/** + * Returns 'true' if the given stage is an internal change stream stage that can appear in a router + * (mongoS) pipeline, or 'false' otherwise. + */ +bool isChangeStreamRouterPipelineStage(StringData stageName) { + return change_stream_constants::kChangeStreamRouterPipelineStages.contains(stageName); +} } // namespace bool DocumentSource::pushMatchBefore(Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { - auto nextMatch = dynamic_cast<DocumentSourceMatch*>((*std::next(itr)).get()); - auto thisGroup = dynamic_cast<DocumentSourceGroup*>(this); - if (constraints().canSwapWithMatch && nextMatch && !nextMatch->isTextQuery() && - (!thisGroup || groupMatchSwapVerified(*nextMatch, *thisGroup))) { - // We're allowed to swap with a $match and the stage after us is a $match. Furthermore, the - // $match does not contain a text search predicate, which we do not attempt to optimize - // because such a $match must already be the first stage in the pipeline. We can attempt to - // swap the $match or part of the $match before ourselves. - auto splitMatch = - DocumentSourceMatch::splitMatchByModifiedFields(nextMatch, getModifiedPaths()); - invariant(splitMatch.first || splitMatch.second); - - if (splitMatch.first) { - // At least part of the $match can be moved before this stage. Erase the original $match - // and put the independent part before this stage. If splitMatch.second is not null, - // then there is a new $match stage to insert after ourselves which is dependent on the - // modified fields. - LOGV2_DEBUG( - 5943503, + if (!constraints().canSwapWithMatch) { + return false; + } + + auto nextStageAsMatch = dynamic_cast<DocumentSourceMatch*>((*std::next(itr)).get()); + if (!nextStageAsMatch || nextStageAsMatch->isTextQuery()) { + // We do not need to attempt this optimization if the $match contains a text search + // predicate because, in that scenario, $match is already required to be the first stage in + // the pipeline. + return false; + } + + // At this point: + // 1) The next stage after 'this' is $match. + // 2) The $match stage does not contain a text search predicate. + + // TODO SERVER-55492: Remove the following workaround when there are rename checks for 'other' + // match expressions. + if (isChangeStreamRouterPipelineStage(this->getSourceName())) { + // Always move the $match stage ahead of internal change stream stages appearing in the + // router (mongoS) pipeline, because they do not access or modify any paths in the input + // document. + container->splice(itr, *container, std::next(itr)); + return true; + } + + auto thisStageAsGroup = dynamic_cast<DocumentSourceGroup*>(this); + if (thisStageAsGroup && !groupMatchSwapVerified(*nextStageAsMatch, *thisStageAsGroup)) { + return false; + } + + auto [renameableMatchPart, nonRenameableMatchPart] = + DocumentSourceMatch::splitMatchByModifiedFields(nextStageAsMatch, getModifiedPaths()); + invariant(renameableMatchPart || nonRenameableMatchPart); + if (!renameableMatchPart) { + return false; + } + + LOGV2_DEBUG(5943503, 5, "Swapping all or part of a $match stage in front of another stage: ", - "matchMovingBefore"_attr = redact(splitMatch.first->serializeToBSONForDebug()), + "matchMovingBefore"_attr = redact(renameableMatchPart->serializeToBSONForDebug()), "thisStage"_attr = redact(serializeToBSONForDebug()), "matchLeftAfter"_attr = redact( - splitMatch.second ? splitMatch.second->serializeToBSONForDebug() : BSONObj())); - container->erase(std::next(itr)); - container->insert(itr, std::move(splitMatch.first)); - if (splitMatch.second) { - container->insert(std::next(itr), std::move(splitMatch.second)); - } - - return true; - } + nonRenameableMatchPart ? nonRenameableMatchPart->serializeToBSONForDebug() + : BSONObj())); + + // At this point we know that at least part of the $match expression can be moved ahead of + // 'this'. So, we erase the original $match and move that renameable part ahead of 'this' stage. + container->erase(std::next(itr)); + container->insert(itr, std::move(renameableMatchPart)); + + // If 'nonRenameableMatchPart' is not null, the 'renameableMatchPart' of the $match expression + // was only one component of the original $match. So, we need to create a new $match stage for + // the remaining 'nonRenameableMatchPart' and insert it after 'this' - effectively keeping it in + // its original position in the pipeline. + if (nonRenameableMatchPart) { + container->insert(std::next(itr), std::move(nonRenameableMatchPart)); } - return false; + return true; } bool DocumentSource::pushSampleBefore(Pipeline::SourceContainer::iterator itr, @@ -227,7 +260,9 @@ bool DocumentSource::pushSampleBefore(Pipeline::SourceContainer::iterator itr, BSONObj DocumentSource::serializeToBSONForDebug() const { std::vector<Value> serialized; - serializeToArray(serialized, ExplainOptions::Verbosity::kQueryPlanner); + auto opts = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; + serializeToArray(serialized, opts); if (serialized.empty()) { LOGV2_DEBUG(5943501, 5, @@ -278,8 +313,8 @@ Pipeline::SourceContainer::iterator DocumentSource::optimizeAt( } void DocumentSource::serializeToArray(vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain) const { - Value entry = serialize(explain); + const SerializationOptions& opts) const { + Value entry = serialize(opts); if (!entry.missing()) { array.push_back(entry); } diff --git a/src/mongo/db/pipeline/document_source.h b/src/mongo/db/pipeline/document_source.h index faec2961599..4e4a5bfdadd 100644 --- a/src/mongo/db/pipeline/document_source.h +++ b/src/mongo/db/pipeline/document_source.h @@ -437,13 +437,9 @@ public: * * A subclass may choose to overwrite this, rather than serialize, if it should output multiple * stages (eg, $sort sometimes also outputs a $limit). - * - * The 'explain' parameter indicates the explain verbosity mode, or is equal boost::none if no - * explain is requested. */ - virtual void serializeToArray( - std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const; + virtual void serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts = SerializationOptions{}) const; /** * Shortcut method to get a BSONObj for debugging. Often useful in log messages, but is not @@ -794,12 +790,8 @@ private: * This is used by the default implementation of serializeToArray() to add this object * to a pipeline being serialized. Returning a missing() Value results in no entry * being added to the array for this stage (DocumentSource). - * - * The 'explain' parameter indicates the explain verbosity mode, or is equal boost::none if no - * explain is requested. */ - virtual Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const = 0; + virtual Value serialize(const SerializationOptions& opts = SerializationOptions{}) const = 0; }; /** diff --git a/src/mongo/db/pipeline/document_source_bucket.cpp b/src/mongo/db/pipeline/document_source_bucket.cpp index 4de8daee491..acf56f771b9 100644 --- a/src/mongo/db/pipeline/document_source_bucket.cpp +++ b/src/mongo/db/pipeline/document_source_bucket.cpp @@ -71,6 +71,12 @@ list<intrusive_ptr<DocumentSource>> DocumentSourceBucket::createFromBson( BSONElement groupByField; Value defaultValue; + // Validating the input boundaries and default value requires a correct collation. We may not + // have this at parse time if we are not parsing for the purpose of executing the queries (see + // SERVER-87166), so we should skip the validation for that case. + const bool shouldValidateInputRanges = + pExpCtx->mongoProcessInterface->isExpectedToExecuteQueries(); + bool outputFieldSpecified = false; for (auto&& argument : bucketObj) { const auto argName = argument.fieldNameStringData(); @@ -78,10 +84,10 @@ list<intrusive_ptr<DocumentSource>> DocumentSourceBucket::createFromBson( groupByField = argument; const bool groupByIsExpressionInObject = groupByField.type() == BSONType::Object && - groupByField.embeddedObject().firstElementFieldName()[0] == '$'; + groupByField.embeddedObject().firstElementFieldNameStringData().startsWith("$"); - const bool groupByIsPrefixedPath = - groupByField.type() == BSONType::String && groupByField.valueStringData()[0] == '$'; + const bool groupByIsPrefixedPath = groupByField.type() == BSONType::String && + groupByField.valueStringData().startsWith("$"); uassert(40202, str::stream() << "The $bucket 'groupBy' field must be defined as a $-prefixed " "path or an expression, but found: " @@ -129,7 +135,8 @@ list<intrusive_ptr<DocumentSource>> DocumentSourceBucket::createFromBson( << "The 'boundaries' option to $bucket must be sorted, but elements " << i - 1 << " and " << i << " are not in ascending order (" << lower.toString() << " is not less than " << upper.toString() << ").", - pExpCtx->getValueComparator().evaluate(lower < upper)); + !shouldValidateInputRanges || + pExpCtx->getValueComparator().evaluate(lower < upper)); } } else if ("default" == argName) { // If there is a default, make sure that it parses to a constant expression then add @@ -166,8 +173,9 @@ list<intrusive_ptr<DocumentSource>> DocumentSourceBucket::createFromBson( Value lowerValue = boundaryValues.front(); Value upperValue = boundaryValues.back(); - if (canonicalizeBSONType(defaultValue.getType()) == - canonicalizeBSONType(lowerValue.getType())) { + if (shouldValidateInputRanges && + canonicalizeBSONType(defaultValue.getType()) == + canonicalizeBSONType(lowerValue.getType())) { // If the default has the same canonical type as the bucket's boundaries, then make sure the // default is less than the lowest boundary or greater than or equal to the highest // boundary. diff --git a/src/mongo/db/pipeline/document_source_bucket_auto.cpp b/src/mongo/db/pipeline/document_source_bucket_auto.cpp index e3a00af3bd9..cec0be7b847 100644 --- a/src/mongo/db/pipeline/document_source_bucket_auto.cpp +++ b/src/mongo/db/pipeline/document_source_bucket_auto.cpp @@ -372,24 +372,25 @@ void DocumentSourceBucketAuto::doDispose() { _sortedInput.reset(); } -Value DocumentSourceBucketAuto::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceBucketAuto::serialize(const SerializationOptions& opts) const { MutableDocument insides; - insides["groupBy"] = _groupByExpression->serialize(static_cast<bool>(explain)); - insides["buckets"] = Value(_nBuckets); + insides["groupBy"] = _groupByExpression->serialize(opts); + insides["buckets"] = opts.serializeLiteral(_nBuckets); if (_granularityRounder) { - insides["granularity"] = Value(_granularityRounder->getName()); + //"granularity" only supports some strings, so a specific representative value is used if + // necessary. + insides["granularity"] = + opts.serializeLiteral(_granularityRounder->getName(), Value("R5"_sd)); } MutableDocument outputSpec(_accumulatedFields.size()); for (auto&& accumulatedField : _accumulatedFields) { intrusive_ptr<AccumulatorState> accum = accumulatedField.makeAccumulator(); - outputSpec[accumulatedField.fieldName] = - Value(accum->serialize(accumulatedField.expr.initializer, - accumulatedField.expr.argument, - static_cast<bool>(explain))); + outputSpec[opts.serializeFieldPathFromString(accumulatedField.fieldName)] = + Value(accum->serialize( + accumulatedField.expr.initializer, accumulatedField.expr.argument, opts)); } insides["output"] = outputSpec.freezeToValue(); diff --git a/src/mongo/db/pipeline/document_source_bucket_auto.h b/src/mongo/db/pipeline/document_source_bucket_auto.h index 6a85e21c642..6d8cd09089e 100644 --- a/src/mongo/db/pipeline/document_source_bucket_auto.h +++ b/src/mongo/db/pipeline/document_source_bucket_auto.h @@ -45,7 +45,8 @@ namespace mongo { class DocumentSourceBucketAuto final : public DocumentSource { public: static constexpr StringData kStageName = "$bucketAuto"_sd; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + DepsTracker::State getDependencies(DepsTracker* deps) const final; const char* getSourceName() const final; diff --git a/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp b/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp index 8e0be33e786..4bc3cb0e11a 100644 --- a/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp +++ b/src/mongo/db/pipeline/document_source_bucket_auto_test.cpp @@ -89,8 +89,9 @@ public: assertBucketAutoType(bucketAutoStage); vector<Value> explainedStages; - bucketAutoStage->serializeToArray(explainedStages, - ExplainOptions::Verbosity::kQueryPlanner); + bucketAutoStage->serializeToArray( + explainedStages, + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); ASSERT_EQUALS(explainedStages.size(), 1UL); Value expectedExplain = Value(expectedObj); @@ -868,5 +869,109 @@ TEST_F(BucketAutoTests, ShouldFailOnNegativeNumbersWhenGranularitySpecified) { AssertionException, 40260); } + +TEST_F(BucketAutoTests, RedactionWithoutOutputField) { + auto spec = fromjson(R"({ + $bucketAuto: { + groupBy: '$_id', + buckets: 5, + granularity: "R5" + } + })"); + auto docSource = DocumentSourceBucketAuto::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$bucketAuto": { + "groupBy": "$HASH<_id>", + "buckets": "?number", + "granularity": "?string", + "output": { + "HASH<count>": { + "$sum": "?number" + } + } + } + })", + redact(*docSource)); +} + +TEST_F(BucketAutoTests, RedactionWithOutputField) { + auto spec = fromjson(R"({ + $bucketAuto: { + groupBy: '$year', + buckets: 3, + output: { + count: { $sum: 1 }, + years: { $push: '$year' } + } + }})"); + auto docSource = DocumentSourceBucketAuto::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$bucketAuto": { + "groupBy": "$HASH<year>", + "buckets": "?number", + "output": { + "HASH<count>": { + "$sum": "?number" + }, + "HASH<years>": { + "$push": "$HASH<year>" + } + } + } + })", + redact(*docSource)); +} + +TEST_F(BucketAutoTests, QueryShapeReParseSerializedStage) { + auto expCtx = getExpCtx(); + auto spec = fromjson(R"({ + $bucketAuto: { + groupBy: '$year', + buckets: 3, + granularity: "E192", + output: { + count: { $sum: 1 }, + years: { $push: '$year' } + } + }})"); + + auto docSource = DocumentSourceBucketAuto::createFromBson(spec.firstElement(), expCtx); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + std::vector<Value> serialized; + docSource->serializeToArray(serialized, opts); + auto serializedDocSource = serialized[0].getDocument().toBson(); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$bucketAuto": { + "groupBy": "$year", + "buckets": 1, + "granularity": "R5", + "output": { + "count": { + "$sum": { + "$const":1 + } + }, + "years": { + "$push": "$year" + } + } + } + })", + serializedDocSource); + auto docSourceFromQueryShape = + DocumentSourceBucketAuto::createFromBson(serializedDocSource.firstElement(), expCtx); + + vector<Value> newSerialization; + docSourceFromQueryShape->serializeToArray(newSerialization, opts); + auto newSerializedDocSource = newSerialization[0].getDocument().toBson(); + ASSERT_BSONOBJ_EQ(serializedDocSource, newSerializedDocSource); +} + + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_bucket_test.cpp b/src/mongo/db/pipeline/document_source_bucket_test.cpp index 1036c456ef3..e7702d58d28 100644 --- a/src/mongo/db/pipeline/document_source_bucket_test.cpp +++ b/src/mongo/db/pipeline/document_source_bucket_test.cpp @@ -79,7 +79,8 @@ public: // Serialize the DocumentSourceGroup and DocumentSourceSort from $bucket so that we can // check the explain output to make sure $group and $sort have the correct fields. - auto explain = ExplainOptions::Verbosity::kQueryPlanner; + auto explain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; vector<Value> explainedStages; groupStage->serializeToArray(explainedStages, explain); sortStage->serializeToArray(explainedStages, explain); @@ -227,9 +228,35 @@ TEST_F(BucketReturnsGroupAndSort, BucketSucceedsWithMultipleBoundaryValues) { testCreateFromBsonResult(spec, expectedGroupExplain); } +TEST_F(BucketReturnsGroupAndSort, BucketWithEmptyGroupByStrDoesNotAccessPastEndOfString) { + // Verify that {groupBy: ''} is rejected _without_ attempting to read past the end of the empty + // string. + const auto spec = + fromjson("{$bucket : {groupBy : '', boundaries : [ 1, 5, 8 ], default : 'other'}}"); + + // Under a debug build, this would previously fail if an empty str for groupBy led to access + // past the end of the string, with pos() > size() in StringData::operator[]. + // Verify that this reaches the intended uassert, rejecting the empty string, _without_ first + // trying to read past the end of the string. + ASSERT_THROWS_CODE(DocumentSourceBucket::createFromBson(spec.firstElement(), getExpCtx()), + AssertionException, + 40202); +} + +/* + * Override the stub interface to allow full execution in these tests. + */ +class ExecutableStubMongoProcessInterface : public StubMongoProcessInterface { + bool isExpectedToExecuteQueries() override { + return true; + } +}; + class InvalidBucketSpec : public AggregationContextFixture { public: list<intrusive_ptr<DocumentSource>> createBucket(BSONObj bucketSpec) { + getExpCtx()->mongoProcessInterface = + std::make_unique<ExecutableStubMongoProcessInterface>(); auto sources = DocumentSourceBucket::createFromBson(bucketSpec.firstElement(), getExpCtx()); return sources; } diff --git a/src/mongo/db/pipeline/document_source_change_stream.cpp b/src/mongo/db/pipeline/document_source_change_stream.cpp index 3f19d4ac4ee..7602e29417e 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream.cpp @@ -352,7 +352,7 @@ std::list<boost::intrusive_ptr<DocumentSource>> DocumentSourceChangeStream::_bui // If 'showExpandedEvents' is NOT set, add a filter that returns only classic change events. if (!spec.getShowExpandedEvents()) { - stages.push_back(DocumentSourceMatch::create( + stages.push_back(DocumentSourceInternalChangeStreamMatch::create( change_stream_filter::getMatchFilterForClassicOperationTypes(), expCtx)); } return stages; @@ -369,6 +369,12 @@ void DocumentSourceChangeStream::assertIsLegalSpecification( (replCoord && replCoord->getReplicationMode() == repl::ReplicationCoordinator::Mode::modeReplSet)); + // We will not validate user specified options when we are not expecting to execute queries, + // such as during $queryStats. + if (!expCtx->mongoProcessInterface->isExpectedToExecuteQueries()) { + return; + } + // If 'allChangesForCluster' is true, the stream must be opened on the 'admin' database with // {aggregate: 1}. uassert(ErrorCodes::InvalidOptions, diff --git a/src/mongo/db/pipeline/document_source_change_stream.h b/src/mongo/db/pipeline/document_source_change_stream.h index 5a76a5205e2..1a75d5b4300 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.h +++ b/src/mongo/db/pipeline/document_source_change_stream.h @@ -336,4 +336,29 @@ public: } }; +/** + * A DocumentSource class for all internal change stream stages. This class is useful for + * shared logic between all of the internal change stream stages. For internally created match + * stages see 'DocumentSourceInternalChangeStreamMatch'. + */ +class DocumentSourceInternalChangeStreamStage : public DocumentSource { +public: + DocumentSourceInternalChangeStreamStage(StringData stageName, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSource(stageName, expCtx) {} + + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const override { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || + opts.transformIdentifiers) { + // Stages made internally by 'DocumentSourceChangeStream' should not be serialized for + // query stats. For query stats we will serialize only the user specified $changeStream + // stage. + return Value(); + } + return doSerialize(opts); + } + + virtual Value doSerialize(const SerializationOptions& opts) const = 0; +}; + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream.idl b/src/mongo/db/pipeline/document_source_change_stream.idl index e1b8227a0fc..de5473afb58 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.idl +++ b/src/mongo/db/pipeline/document_source_change_stream.idl @@ -30,22 +30,10 @@ global: cpp_namespace: "mongo" - cpp_includes: - - "mongo/db/pipeline/resume_token.h" imports: - "mongo/idl/basic_types.idl" - -types: - # A resume token is an opaque document we return to the user that contains all the information - # needed to resume a stream where they left off. It also provides the ordering of streams - # from multiple shards. - resumeToken: - bson_serialization_type: object - description: An object representing a resume token for a change stream - cpp_type: ResumeToken - serializer: ResumeToken::toBSON - deserializer: ResumeToken::parse + - "mongo/db/pipeline/resume_token.idl" enums: FullDocumentBeforeChangeMode: @@ -86,6 +74,7 @@ structs: description: A document used to specify the $changeStream stage of an aggregation pipeline. The same spec is also used by the $_internalChangeStreamTransform stage. + query_shape_component: true fields: resumeAfter: cpp_name: resumeAfter @@ -94,6 +83,7 @@ structs: description: An object representing the point at which we should resume reporting changes from. Only one of resumeAfter, startAfter, and startAtOperationTime should be specified. + query_shape: custom startAfter: cpp_name: startAfter @@ -103,6 +93,7 @@ structs: changes from. This is allowed to be a token from an invalidating command. Only one of resumeAfter, startAfter, and startAtOperationTime should be specified. + query_shape: custom startAtOperationTime: cpp_name: startAtOperationTime @@ -111,6 +102,7 @@ structs: description: The operation time after which we should start reporting changes. Only one of resumeAfter, startAfter, and startAtOperationTime should be specified. + query_shape: literal fullDocument: cpp_name: fullDocument @@ -118,6 +110,7 @@ structs: default: kDefault description: A string '"updateLookup"' or '"default"', indicating whether or not we should return a full document or just changes for an update. + query_shape: parameter fullDocumentBeforeChange: cpp_name: fullDocumentBeforeChange @@ -132,6 +125,7 @@ structs: set to "required", then the "fullDocumentBeforeChange" field is always populated and an exception is thrown if the pre-image is not available. + query_shape: parameter allChangesForCluster: cpp_name: allChangesForCluster @@ -139,6 +133,7 @@ structs: description: A flag indicating whether the stream should report all changes that occur on the deployment, aside from those on internal databases or collections. + query_shape: literal showMigrationEvents: cpp_name: showMigrationEvents @@ -149,12 +144,14 @@ structs: deletes may appear that do not reflect actual deletions or insertions of data. Instead they reflect this data moving from one shard to another. + query_shape: literal showSystemEvents: cpp_name: showSystemEvents type: optionalBool description: A flag indicating whether the stream should report events on system collections. + query_shape: literal allowToRunOnConfigDB: cpp_name: allowToRunOnConfigDB @@ -163,17 +160,20 @@ structs: 'config' database, which is usually banned. This flag is used internally to allow mongoS to open a stream on 'config.shards', in order to monitor for the addition of new shards to the cluster. + query_shape: literal allowToRunOnSystemNS: cpp_name: allowToRunOnSystemNS type: optionalBool description: An internal flag indicating whether the change stream may be opened on a system collection. + query_shape: literal showExpandedEvents: type: optionalBool description: A flag indicating whether the change stream output should include the expanded events and parameters that may not be part of the stable API. + query_shape: literal showRawUpdateDescription: cpp_name: showRawUpdateDescription @@ -183,6 +183,7 @@ structs: raw update description from the oplog entry for the corresponding update command. When this flag is set to true, the standard "updateDescription" field in update events will be omitted. + query_shape: literal DocumentSourceChangeStreamOplogMatchSpec: strict: true @@ -206,7 +207,7 @@ structs: startAfterInvalidate: type: resumeToken optional: true - + DocumentSourceChangeStreamCheckResumabilitySpec: strict: true description: A document used to specify the $_internalChangeStreamCheckResumability stage of @@ -231,3 +232,12 @@ structs: fields: fullDocument: type: FullDocumentMode + + DocumentSourceChangeStreamHandleTopologyChangeSpec: + strict: true + description: A document used to specify the $_internalChangeStreamHandleTopologyChange stage of + an aggregation pipeline. + fields: + originalAggregateCommand: + type: object + optional: true diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp index aa663dd131f..437ec96b386 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.cpp @@ -217,9 +217,8 @@ boost::optional<Document> DocumentSourceChangeStreamAddPostImage::lookupLatestPo pExpCtx, nss, *resumeTokenData.uuid, documentKey, std::move(readConcern)); } -Value DocumentSourceChangeStreamAddPostImage::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return explain +Value DocumentSourceChangeStreamAddPostImage::doSerialize(const SerializationOptions& opts) const { + return opts.verbosity ? Value(Document{ {DocumentSourceChangeStream::kStageName, Document{{"stage"_sd, kStageName}, @@ -227,5 +226,4 @@ Value DocumentSourceChangeStreamAddPostImage::serialize( : Value(Document{{kStageName, DocumentSourceChangeStreamAddPostImageSpec(_fullDocumentMode).toBSON()}}); } - } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h index d92994064e2..03c5459d15d 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h +++ b/src/mongo/db/pipeline/document_source_change_stream_add_post_image.h @@ -38,7 +38,8 @@ namespace mongo { * Part of the change stream API machinery used to look up the post-image of a document. Uses the * "documentKey" field of the input to look up the new version of the document. */ -class DocumentSourceChangeStreamAddPostImage final : public DocumentSource { +class DocumentSourceChangeStreamAddPostImage final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamAddPostImage"_sd; static constexpr StringData kFullDocumentFieldName = @@ -114,7 +115,7 @@ public: return DepsTracker::State::SEE_NEXT; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; const char* getSourceName() const final { return kStageName.rawData(); @@ -123,7 +124,8 @@ public: private: DocumentSourceChangeStreamAddPostImage(const boost::intrusive_ptr<ExpressionContext>& expCtx, const FullDocumentModeEnum fullDocumentMode) - : DocumentSource(kStageName, expCtx), _fullDocumentMode(fullDocumentMode) { + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx), + _fullDocumentMode(fullDocumentMode) { tassert(5842300, "the 'fullDocument' field cannot be 'default'", _fullDocumentMode != FullDocumentModeEnum::kDefault); diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_post_image_test.cpp b/src/mongo/db/pipeline/document_source_change_stream_add_post_image_test.cpp index 25ca17bbbfa..c97a393d4d7 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_post_image_test.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_add_post_image_test.cpp @@ -106,7 +106,9 @@ TEST_F(DocumentSourceChangeStreamAddPostImageTest, ShouldSerializeAsExpectedForE Document{{"stage"_sd, DocumentSourceChangeStreamAddPostImage::kStageName}, {"fullDocument"_sd, "updateLookup"_sd}}}}); - ASSERT_VALUE_EQ(stage->serialize({ExplainOptions::Verbosity::kQueryPlanner}), expectedOutput); + ASSERT_VALUE_EQ(stage->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedOutput); } TEST_F(DocumentSourceChangeStreamAddPostImageTest, ShouldSerializeAsExpectedForDispatch) { diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp index 42b0cc20bfb..f3c47c58d9a 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.cpp @@ -152,9 +152,8 @@ boost::optional<Document> DocumentSourceChangeStreamAddPreImage::lookupPreImage( return preImageField.getDocument().getOwned(); } -Value DocumentSourceChangeStreamAddPreImage::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return explain +Value DocumentSourceChangeStreamAddPreImage::doSerialize(const SerializationOptions& opts) const { + return opts.verbosity ? Value(Document{ {DocumentSourceChangeStream::kStageName, Document{{"stage"_sd, "internalAddPreImage"_sd}, diff --git a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h index e04ac9a30ef..2c4918ac34b 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h +++ b/src/mongo/db/pipeline/document_source_change_stream_add_pre_image.h @@ -41,7 +41,7 @@ namespace mongo { * its "fullDocumentBeforeChange" field shall be the optime of the noop oplog entry containing the * pre-image. This stage replaces that field with the actual pre-image document. */ -class DocumentSourceChangeStreamAddPreImage final : public DocumentSource { +class DocumentSourceChangeStreamAddPreImage final : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamAddPreImage"_sd; static constexpr StringData kFullDocumentBeforeChangeFieldName = @@ -68,7 +68,8 @@ public: DocumentSourceChangeStreamAddPreImage(const boost::intrusive_ptr<ExpressionContext>& expCtx, FullDocumentBeforeChangeModeEnum mode) - : DocumentSource(kStageName, expCtx), _fullDocumentBeforeChangeMode(mode) { + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx), + _fullDocumentBeforeChangeMode(mode) { // This stage should never be created with FullDocumentBeforeChangeMode::kOff. invariant(_fullDocumentBeforeChangeMode != FullDocumentBeforeChangeModeEnum::kOff); } @@ -108,7 +109,7 @@ public: return DepsTracker::State::SEE_NEXT; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; const char* getSourceName() const final { return kStageName.rawData(); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp index 4e85815b049..935983e8ba5 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.cpp @@ -180,18 +180,20 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamCheckInvalidate::doGetNe return nextInput; } -Value DocumentSourceChangeStreamCheckInvalidate::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { - return Value(Document{{DocumentSourceChangeStream::kStageName, - Document{{"stage"_sd, "internalCheckInvalidate"_sd}}}}); +Value DocumentSourceChangeStreamCheckInvalidate::doSerialize( + const SerializationOptions& opts) const { + BSONObjBuilder builder; + if (opts.verbosity) { + BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); + sub.append("stage"_sd, kStageName); + sub.done(); } - DocumentSourceChangeStreamCheckInvalidateSpec spec; if (_startAfterInvalidate) { spec.setStartAfterInvalidate(ResumeToken(*_startAfterInvalidate)); } - return Value(Document{{DocumentSourceChangeStreamCheckInvalidate::kStageName, spec.toBSON()}}); + builder.append(DocumentSourceChangeStreamCheckInvalidate::kStageName, spec.toBSON()); + return Value(builder.obj()); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h index 166604c1d10..764e8531ccd 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h +++ b/src/mongo/db/pipeline/document_source_change_stream_check_invalidate.h @@ -39,7 +39,8 @@ namespace mongo { * "invalidate" entry for commands that should invalidate the change stream (e.g. collection drop * for a single-collection change stream). It is not intended to be created by the user. */ -class DocumentSourceChangeStreamCheckInvalidate final : public DocumentSource { +class DocumentSourceChangeStreamCheckInvalidate final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamCheckInvalidate"_sd; @@ -64,7 +65,7 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; static boost::intrusive_ptr<DocumentSourceChangeStreamCheckInvalidate> createFromBson( BSONElement spec, const boost::intrusive_ptr<ExpressionContext>& expCtx); @@ -79,7 +80,7 @@ private: */ DocumentSourceChangeStreamCheckInvalidate(const boost::intrusive_ptr<ExpressionContext>& expCtx, boost::optional<ResumeTokenData> startAfterInvalidate) - : DocumentSource(kStageName, expCtx), + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx), _startAfterInvalidate(std::move(startAfterInvalidate)) { invariant(!_startAfterInvalidate || _startAfterInvalidate->fromInvalidate == ResumeTokenData::kFromInvalidate); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp index e7e05dcd756..5d704b00c84 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.cpp @@ -129,7 +129,8 @@ DocumentSourceChangeStreamCheckResumability::compareAgainstClientResumeToken( DocumentSourceChangeStreamCheckResumability::DocumentSourceChangeStreamCheckResumability( const intrusive_ptr<ExpressionContext>& expCtx, ResumeTokenData token) - : DocumentSource(getSourceName(), expCtx), _tokenFromClient(std::move(token)) {} + : DocumentSourceInternalChangeStreamStage(getSourceName(), expCtx), + _tokenFromClient(std::move(token)) {} intrusive_ptr<DocumentSourceChangeStreamCheckResumability> DocumentSourceChangeStreamCheckResumability::create(const intrusive_ptr<ExpressionContext>& expCtx, @@ -205,17 +206,21 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamCheckResumability::doGet MONGO_UNREACHABLE; } -Value DocumentSourceChangeStreamCheckResumability::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return explain - ? Value(DOC(DocumentSourceChangeStream::kStageName - << DOC("stage" - << "internalCheckResumability"_sd - << "resumeToken" << ResumeToken(_tokenFromClient).toDocument()))) - : Value(Document{ - {DocumentSourceChangeStreamCheckResumability::kStageName, - DocumentSourceChangeStreamCheckResumabilitySpec(ResumeToken(_tokenFromClient)) - .toBSON()}}); +Value DocumentSourceChangeStreamCheckResumability::doSerialize( + const SerializationOptions& opts) const { + BSONObjBuilder builder; + if (opts.verbosity) { + BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); + sub.append("stage"_sd, kStageName); + sub << "resumeToken"_sd << Value(ResumeToken(_tokenFromClient).toDocument(opts)); + sub.done(); + } else { + builder.append( + kStageName, + DocumentSourceChangeStreamCheckResumabilitySpec(ResumeToken(_tokenFromClient)) + .toBSON()); + } + return Value(builder.obj()); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h index 59946310ab1..d0409198d22 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h +++ b/src/mongo/db/pipeline/document_source_change_stream_check_resumability.h @@ -59,7 +59,7 @@ namespace mongo { * - Otherwise we cannot resume, as we do not know if there were any events between the resume token * and the first matching document in the oplog. */ -class DocumentSourceChangeStreamCheckResumability : public DocumentSource { +class DocumentSourceChangeStreamCheckResumability : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamCheckResumability"_sd; @@ -90,7 +90,7 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const override; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const override; static boost::intrusive_ptr<DocumentSourceChangeStreamCheckResumability> createFromBson( BSONElement spec, const boost::intrusive_ptr<ExpressionContext>& expCtx); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp index 26cdcb77181..912c691d8f2 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.cpp @@ -86,9 +86,9 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamCheckTopologyChange::doG return nextInput; } -Value DocumentSourceChangeStreamCheckTopologyChange::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +Value DocumentSourceChangeStreamCheckTopologyChange::doSerialize( + const SerializationOptions& opts) const { + if (opts.verbosity) { return Value(DOC(DocumentSourceChangeStream::kStageName << DOC("stage" << "internalCheckTopologyChange"_sd))); diff --git a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h index 7629e5d8063..9bf5b698a7c 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h +++ b/src/mongo/db/pipeline/document_source_change_stream_check_topology_change.h @@ -45,7 +45,8 @@ namespace mongo { * that previously may not have held any data for the collection being watched, and they contain the * information necessary for the mongoS to include the new shard in the merged change stream. */ -class DocumentSourceChangeStreamCheckTopologyChange final : public DocumentSource { +class DocumentSourceChangeStreamCheckTopologyChange final + : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamCheckTopologyChange"_sd; @@ -67,12 +68,12 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; private: DocumentSourceChangeStreamCheckTopologyChange( const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSource(kStageName, expCtx) {} + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx) {} GetNextResult doGetNext() final; }; diff --git a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp index a6c9a3fee68..b5c24a36af8 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.cpp @@ -153,16 +153,20 @@ DocumentSource::GetNextResult DocumentSourceChangeStreamEnsureResumeTokenPresent } } -Value DocumentSourceChangeStreamEnsureResumeTokenPresent::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - // We only serialize this stage in the context of explain. - if (explain) { - return Value(DOC(DocumentSourceChangeStream::kStageName - << DOC("stage" - << "internalEnsureResumeTokenPresent"_sd - << "resumeToken" << ResumeToken(_tokenFromClient).toDocument()))); +Value DocumentSourceChangeStreamEnsureResumeTokenPresent::doSerialize( + const SerializationOptions& opts) const { + BSONObjBuilder builder; + if (opts.verbosity) { + BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); + sub.append("stage"_sd, kStageName); + sub << "resumeToken"_sd << Value(ResumeToken(_tokenFromClient).toDocument(opts)); + sub.done(); + } else { + BSONObjBuilder sub(builder.subobjStart(kStageName)); + sub << "resumeToken"_sd << Value(ResumeToken(_tokenFromClient).toDocument(opts)); + sub.done(); } - MONGO_UNREACHABLE_TASSERT(5467611); + return Value(builder.obj()); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h index 6fcabf4c0b6..0482e128379 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h +++ b/src/mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h @@ -39,7 +39,8 @@ namespace mongo { class DocumentSourceChangeStreamEnsureResumeTokenPresent final : public DocumentSourceChangeStreamCheckResumability { public: - static constexpr StringData kStageName = "$_internalChangeStreamEnsureResumeTokenPresent"_sd; + static constexpr StringData kStageName = + change_stream_constants::stage_names::kEnsureResumeTokenPresent; const char* getSourceName() const final; @@ -54,7 +55,7 @@ public: const boost::intrusive_ptr<ExpressionContext>& expCtx, const DocumentSourceChangeStreamSpec& spec); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; private: /** diff --git a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp index 9f83f3df203..4edb7f0c26a 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp @@ -46,6 +46,11 @@ namespace mongo { namespace { +REGISTER_INTERNAL_DOCUMENT_SOURCE(_internalChangeStreamHandleTopologyChange, + LiteParsedDocumentSourceChangeStreamInternal::parse, + DocumentSourceChangeStreamHandleTopologyChange::createFromBson, + true); + // Failpoint to throw an exception when the 'kNewShardDetected' event is observed. MONGO_FAIL_POINT_DEFINE(throwChangeStreamTopologyChangeExceptionToClient); @@ -110,6 +115,15 @@ bool isShardConfigEvent(const Document& eventDoc) { } // namespace boost::intrusive_ptr<DocumentSourceChangeStreamHandleTopologyChange> +DocumentSourceChangeStreamHandleTopologyChange::createFromBson( + const BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx) { + uassert(8131300, + str::stream() << "the '" << kStageName << "' spec must be an empty object", + elem.type() == Object && elem.Obj().isEmpty()); + return new DocumentSourceChangeStreamHandleTopologyChange(expCtx); +} + +boost::intrusive_ptr<DocumentSourceChangeStreamHandleTopologyChange> DocumentSourceChangeStreamHandleTopologyChange::create( const boost::intrusive_ptr<ExpressionContext>& expCtx) { return new DocumentSourceChangeStreamHandleTopologyChange(expCtx); @@ -117,7 +131,7 @@ DocumentSourceChangeStreamHandleTopologyChange::create( DocumentSourceChangeStreamHandleTopologyChange::DocumentSourceChangeStreamHandleTopologyChange( const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSource(kStageName, expCtx) {} + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx) {} StageConstraints DocumentSourceChangeStreamHandleTopologyChange::constraints( Pipeline::SplitState) const { @@ -257,9 +271,9 @@ BSONObj DocumentSourceChangeStreamHandleTopologyChange::replaceResumeTokenInComm return newCmd.freeze().toBson(); } -Value DocumentSourceChangeStreamHandleTopologyChange::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +Value DocumentSourceChangeStreamHandleTopologyChange::doSerialize( + const SerializationOptions& opts) const { + if (opts.verbosity) { return Value(DOC(DocumentSourceChangeStream::kStageName << DOC("stage" << "internalHandleTopologyChange"_sd))); diff --git a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h index 4c07368aff9..b88c50d75ce 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h +++ b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.h @@ -46,10 +46,14 @@ namespace mongo { * the first time. When this event is detected, this stage will establish a new cursor on that * shard and add it to the cursors being merged. */ -class DocumentSourceChangeStreamHandleTopologyChange final : public DocumentSource { +class DocumentSourceChangeStreamHandleTopologyChange final + : public DocumentSourceInternalChangeStreamStage { public: - static constexpr StringData kStageName = "$_internalChangeStreamHandleTopologyChange"_sd; + static constexpr StringData kStageName = + change_stream_constants::stage_names::kHandleTopologyChange; + static boost::intrusive_ptr<DocumentSourceChangeStreamHandleTopologyChange> createFromBson( + BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx); /** * Creates a new stage which will establish a new cursor and add it to the cursors being merged * by 'mergeCursorsStage' whenever a new shard is detected by a change stream. @@ -61,7 +65,7 @@ public: return kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; StageConstraints constraints(Pipeline::SplitState) const final; diff --git a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp index 658861a9a3f..8656ad68b04 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.cpp @@ -96,8 +96,8 @@ std::unique_ptr<MatchExpression> buildOplogMatchFilter( DocumentSourceChangeStreamOplogMatch::DocumentSourceChangeStreamOplogMatch( Timestamp clusterTime, const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSourceMatch(change_stream_filter::buildOplogMatchFilter(expCtx, clusterTime), - expCtx) { + : DocumentSourceInternalChangeStreamMatch( + change_stream_filter::buildOplogMatchFilter(expCtx, clusterTime), expCtx) { _clusterTime = clusterTime; expCtx->tailableMode = TailableModeEnum::kTailableAndAwaitData; } @@ -204,16 +204,23 @@ Pipeline::SourceContainer::iterator DocumentSourceChangeStreamOplogMatch::doOpti return nextChangeStreamStageItr; } -Value DocumentSourceChangeStreamOplogMatch::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { - return Value( - Document{{DocumentSourceChangeStream::kStageName, - Document{{"stage"_sd, "internalOplogMatch"_sd}, {"filter"_sd, _predicate}}}}); +Value DocumentSourceChangeStreamOplogMatch::doSerialize(const SerializationOptions& opts) const { + BSONObjBuilder builder; + if (opts.verbosity) { + BSONObjBuilder sub(builder.subobjStart(DocumentSourceChangeStream::kStageName)); + sub.append("stage"_sd, kStageName); + sub.append(DocumentSourceChangeStreamOplogMatchSpec::kFilterFieldName, + getMatchExpression()->serialize(opts)); + sub.done(); + } else { + BSONObjBuilder sub(builder.subobjStart(kStageName)); + + // 'SerializationOptions' are not required here, since serialization for explain and query + // stats occur before this function call. + DocumentSourceChangeStreamOplogMatchSpec(_predicate).serialize(&sub); + sub.done(); } - - DocumentSourceChangeStreamOplogMatchSpec spec(_predicate); - return Value(Document{{DocumentSourceChangeStreamOplogMatch::kStageName, spec.toBSON()}}); + return Value(builder.obj()); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h index 54d15f363e6..6138fdaae31 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h +++ b/src/mongo/db/pipeline/document_source_change_stream_oplog_match.h @@ -36,7 +36,7 @@ namespace mongo { * A custom subclass of DocumentSourceMatch which is used to generate a $match stage to be applied * on the oplog. The stage requires itself to be the first stage in the pipeline. */ -class DocumentSourceChangeStreamOplogMatch final : public DocumentSourceMatch { +class DocumentSourceChangeStreamOplogMatch final : public DocumentSourceInternalChangeStreamMatch { public: static constexpr StringData kStageName = "$_internalChangeStreamOplogMatch"_sd; @@ -45,7 +45,7 @@ public: DocumentSourceChangeStreamOplogMatch(const DocumentSourceChangeStreamOplogMatch& other, const boost::intrusive_ptr<ExpressionContext>& newExpCtx) - : DocumentSourceMatch(other, newExpCtx) { + : DocumentSourceInternalChangeStreamMatch(other, newExpCtx) { _clusterTime = other._clusterTime; _optimizedEndOfPipeline = other._optimizedEndOfPipeline; } @@ -74,7 +74,7 @@ public: StageConstraints constraints(Pipeline::SplitState pipeState) const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts) const final; protected: Pipeline::SourceContainer::iterator doOptimizeAt(Pipeline::SourceContainer::iterator itr, @@ -88,7 +88,8 @@ private: */ DocumentSourceChangeStreamOplogMatch(BSONObj filter, const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSourceMatch(std::move(filter), expCtx), _optimizedEndOfPipeline(true) { + : DocumentSourceInternalChangeStreamMatch(std::move(filter), expCtx), + _optimizedEndOfPipeline(true) { expCtx->tailableMode = TailableModeEnum::kTailableAndAwaitData; } diff --git a/src/mongo/db/pipeline/document_source_change_stream_split_large_event.cpp b/src/mongo/db/pipeline/document_source_change_stream_split_large_event.cpp index b2ac5353325..1cf2ef0ed58 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_split_large_event.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_split_large_event.cpp @@ -31,8 +31,7 @@ #include "mongo/db/pipeline/change_stream_helpers.h" #include "mongo/db/pipeline/change_stream_split_event_helpers.h" -#include "mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h" -#include "mongo/db/pipeline/document_source_change_stream_handle_topology_change.h" +#include "mongo/db/pipeline/document_source_change_stream_check_resumability.h" namespace mongo { namespace { @@ -82,8 +81,7 @@ DocumentSourceChangeStreamSplitLargeEvent::DocumentSourceChangeStreamSplitLargeE !_resumeAfterSplit || _resumeAfterSplit->fragmentNum); } -Value DocumentSourceChangeStreamSplitLargeEvent::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceChangeStreamSplitLargeEvent::serialize(const SerializationOptions& opts) const { return Value(Document{{DocumentSourceChangeStreamSplitLargeEvent::kStageName, Document{}}}); } @@ -190,24 +188,19 @@ size_t DocumentSourceChangeStreamSplitLargeEvent::_handleResumeAfterSplit(const return fragmentNum; } -namespace { -// During pipeline optimization, the split stage must move ahead of these change stream stages. -static const std::set<StringData> kStagesToMoveAheadOf = { - DocumentSourceChangeStreamEnsureResumeTokenPresent::kStageName, - DocumentSourceChangeStreamHandleTopologyChange::kStageName}; -} // namespace - Pipeline::SourceContainer::iterator DocumentSourceChangeStreamSplitLargeEvent::doOptimizeAt( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { // Helper to determine whether the iterator has reached its final position in the pipeline. // Checks whether $changeStreamSplitLargeEvent should move ahead of the given stage. auto shouldMoveAheadOf = [](const auto& stagePtr) { - return kStagesToMoveAheadOf.count(stagePtr->getSourceName()); + return change_stream_constants::kChangeStreamRouterPipelineStages.contains( + stagePtr->getSourceName()); }; // Find the point in the pipeline that the stage should move to. - for (auto it = itr; it != container->begin() && shouldMoveAheadOf(*std::prev(it)); --it) { - std::swap(*it, *std::prev(it)); + for (auto it = itr; it != container->begin() && shouldMoveAheadOf(*std::prev(it));) { + // Swap 'it' with the previous stage. + container->splice(std::prev(it), *container, it); } // Return an iterator pointing to the next stage to be optimized. @@ -218,6 +211,7 @@ void DocumentSourceChangeStreamSplitLargeEvent::validatePipelinePosition( bool alreadyOptimized, Pipeline::SourceContainer::const_iterator pos, const Pipeline::SourceContainer& container) const { + // The $changeStreamSplitLargeEvent stage must be the final stage in the pipeline before // optimization. uassert(7182802, @@ -230,7 +224,8 @@ void DocumentSourceChangeStreamSplitLargeEvent::validatePipelinePosition( str::stream() << getSourceName() << " is at the wrong position in the pipeline after optimization", !alreadyOptimized || std::none_of(container.begin(), pos, [](const auto& stage) { - return kStagesToMoveAheadOf.count(stage->getSourceName()); + return change_stream_constants::kChangeStreamRouterPipelineStages.contains( + stage->getSourceName()); })); }; } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_split_large_event.h b/src/mongo/db/pipeline/document_source_change_stream_split_large_event.h index 1dfab7b85a1..44ed8b41955 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_split_large_event.h +++ b/src/mongo/db/pipeline/document_source_change_stream_split_large_event.h @@ -78,10 +78,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - // This stage does not reference any user or system variables. - void addVariableRefs(std::set<Variables::Id>* refs) const {} - - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/pipeline/document_source_change_stream_test.cpp b/src/mongo/db/pipeline/document_source_change_stream_test.cpp index 50e0f0cdcac..8bcdb7e7f9d 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_test.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_test.cpp @@ -27,6 +27,7 @@ * it in the license file. */ +#include "mongo/bson/bsontypes.h" #include "mongo/platform/basic.h" #include <boost/intrusive_ptr.hpp> @@ -52,8 +53,11 @@ #include "mongo/db/pipeline/document_source_change_stream_add_pre_image.h" #include "mongo/db/pipeline/document_source_change_stream_check_invalidate.h" #include "mongo/db/pipeline/document_source_change_stream_check_resumability.h" +#include "mongo/db/pipeline/document_source_change_stream_check_topology_change.h" #include "mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h" +#include "mongo/db/pipeline/document_source_change_stream_handle_topology_change.h" #include "mongo/db/pipeline/document_source_change_stream_oplog_match.h" +#include "mongo/db/pipeline/document_source_change_stream_split_large_event.h" #include "mongo/db/pipeline/document_source_change_stream_transform.h" #include "mongo/db/pipeline/document_source_change_stream_unwind_transaction.h" #include "mongo/db/pipeline/document_source_limit.h" @@ -88,15 +92,26 @@ using V = Value; using DSChangeStream = DocumentSourceChangeStream; +// Deterministic values used for testing +const UUID testConstUuid = UUID::parse("6948DF80-14BD-4E04-8842-7668D9C001F5").getValue(); + +class ExecutableStubMongoProcessInterface : public StubMongoProcessInterface { + bool isExpectedToExecuteQueries() override { + return true; + } +}; + class ChangeStreamStageTestNoSetup : public AggregationContextFixture { public: ChangeStreamStageTestNoSetup() : ChangeStreamStageTestNoSetup(nss) {} explicit ChangeStreamStageTestNoSetup(NamespaceString nsString) - : AggregationContextFixture(nsString) {} + : AggregationContextFixture(nsString) { + getExpCtx()->mongoProcessInterface = + std::make_unique<ExecutableStubMongoProcessInterface>(); + }; }; -struct MockMongoInterface final : public StubMongoProcessInterface { - +struct MockMongoInterface final : public ExecutableStubMongoProcessInterface { // Used by operations which need to obtain the oplog's UUID. static const UUID& oplogUuid() { static const UUID* oplog_uuid = new UUID(UUID::gen()); @@ -4648,5 +4663,308 @@ TEST_F(MultiTokenFormatVersionTest, CanResumeFromV2HighWaterMark) { next = lastStage->getNext(); ASSERT_FALSE(next.isAdvanced()); } + +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamAddPostImageEmptyForQueryStats) { + auto spec = DocumentSourceChangeStreamSpec(); + spec.setFullDocument(FullDocumentModeEnum::kUpdateLookup); + + auto docSource = DocumentSourceChangeStreamAddPostImage::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalChangeStreamAddPostImage":{"fullDocument":"updateLookup"}})", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamAddPreImageEmptyForQueryStats) { + auto docSource = DocumentSourceChangeStreamAddPreImage{ + getExpCtx(), FullDocumentBeforeChangeModeEnum::kWhenAvailable}; + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamAddPreImage": { + "fullDocumentBeforeChange": "whenAvailable" + } + })", + docSource.serialize().getDocument().toBson()); + + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource.serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamCheckInvalidateEmptyForQueryStats) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse(makeResumeToken(Timestamp(), + testConstUuid, + BSON("_id" << 1 << "x" << 2), + ResumeTokenData::kFromInvalidate))); + + auto docSource = DocumentSourceChangeStreamCheckInvalidate::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamCheckInvalidate": { + "startAfterInvalidate": { + "_data": "8200000000000000002B022C0100296F5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + } + } + })", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamCheckResumabilityEmptyForQueryStats) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamCheckResumability::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamCheckResumability": { + "resumeToken": { + "_data": "8200000000000000002B022C0100296E5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + } + } + })", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamCheckTopologyChangeEmptyForQueryStats) { + auto docSource = DocumentSourceChangeStreamCheckTopologyChange::create(getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalChangeStreamCheckTopologyChange":{}})", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamEnsureResumeTokenPresentEmptyForQueryStats) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamEnsureResumeTokenPresent::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamEnsureResumeTokenPresent": { + "resumeToken": { + "_data": "8200000000000000002B022C0100296E5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + } + } + })", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamHandleTopologyChangeEmptyForQueryStats) { + auto docSource = DocumentSourceChangeStreamHandleTopologyChange::create(getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalChangeStreamHandleTopologyChange":{}})", + docSource->serialize().getDocument().toBson()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamSplitLargeEvent) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamSplitLargeEvent::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$changeStreamSplitLargeEvent":{}})", + docSource->serialize().getDocument().toBson()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$changeStreamSplitLargeEvent":{}})", + redact(*docSource)); +} + +TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamTransform) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamTransform::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamTransform": { + "resumeAfter": { + "_data": "8200000000000000002B022C0100296E5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + }, + "fullDocument": "default", + "fullDocumentBeforeChange": "off" + } + })", + docSource->serialize().getDocument().toBson()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "resumeAfter": { + "_data": "?string" + }, + "fullDocument": "default", + "fullDocumentBeforeChange": "off" + } + })", + redact(*docSource)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "resumeAfter": { + "_data": "8200000000000000002B0229296E04" + }, + "fullDocument": "default", + "fullDocumentBeforeChange": "off" + } + })", + docSource + ->serialize( + SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}) + .getDocument() + .toBson()); +} + + +TEST_F(ChangeStreamStageTestNoSetup, RedactDocumentSourceChangeStreamTransformMoreFields) { + DocumentSourceChangeStreamSpec spec; + spec.setStartAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + spec.setFullDocument(FullDocumentModeEnum::kRequired); + spec.setFullDocumentBeforeChange(FullDocumentBeforeChangeModeEnum::kWhenAvailable); + spec.setShowExpandedEvents(true); + + auto docSource = DocumentSourceChangeStreamTransform::create(getExpCtx(), spec); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalChangeStreamTransform": { + "startAfter": { + "_data": "8200000000000000002B022C0100296E5A10046948DF8014BD4E0488427668D9C001F5461E5F6964002B021E78002B040004" + }, + "fullDocument": "required", + "fullDocumentBeforeChange": "whenAvailable", + "showExpandedEvents": true + } + })", + docSource->serialize().getDocument().toBson()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "startAfter": { + "_data": "?string" + }, + "fullDocument": "required", + "fullDocumentBeforeChange": "whenAvailable", + "showExpandedEvents": true + } + })", + redact(*docSource)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$changeStream": { + "startAfter": { + "_data": "8200000000000000002B0229296E04" + }, + "fullDocument": "required", + "fullDocumentBeforeChange": "whenAvailable", + "showExpandedEvents": true + } + })", + docSource + ->serialize( + SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}) + .getDocument() + .toBson()); +} + +// For DocumentSource types which contain an arbitrarily internal +// MatchExpression, we don't want match the entire structure. This +// assertion allows us to check some basic structure. +void assertRedactedMatchExpressionContainsOperatorsAndRedactedFieldPaths(BSONElement el) { + // Walk the redacted BSON and assert that we have some ops and + // redacted field paths. + auto opCount = 0; + auto redactedFieldPaths = 0; + while (true) { + if (el.type() == mongo::Array) { + auto array = el.Array(); + if (array.empty()) { + break; + } + el = array[0]; + } else if (el.type() == mongo::Object) { + auto obj = el.Obj(); + if (obj.begin() == obj.end()) { + break; + } + el = obj.firstElement(); + + // Field name should be an operator or a redacted field path. + if (el.fieldName()[0] == '$') { + opCount++; + } else if (!strcmp(el.fieldName(), "$regularExpression")) { + opCount++; + // Skip $regularExpression. + continue; + } else { + if (strstr(el.fieldName(), "HASH<") != el.fieldName()) { + FAIL(std::string("Expected redacted field path: ") + el.fieldName()); + } + redactedFieldPaths++; + } + } else { + break; + } + } + + ASSERT(opCount > 0); + ASSERT(redactedFieldPaths > 0); +} + +TEST_F(ChangeStreamStageTestNoSetup, + DocumentSourceChangeStreamUnwindTransactionEmptyForQueryStats) { + auto docSource = DocumentSourceChangeStreamUnwindTransaction::create(getExpCtx()); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + +TEST_F(ChangeStreamStageTestNoSetup, DocumentSourceChangeStreamOplogMatchEmptyForQueryStats) { + DocumentSourceChangeStreamSpec spec; + spec.setResumeAfter(ResumeToken::parse( + makeResumeToken(Timestamp(), testConstUuid, BSON("_id" << 1 << "x" << 2)))); + + auto docSource = DocumentSourceChangeStreamOplogMatch::create(getExpCtx(), spec); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + ASSERT(docSource->serialize(opts).missing()); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_change_stream_transform.cpp b/src/mongo/db/pipeline/document_source_change_stream_transform.cpp index 7434190a609..3ea1312f51c 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_transform.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_transform.cpp @@ -73,7 +73,8 @@ DocumentSourceChangeStreamTransform::createFromBson( DocumentSourceChangeStreamTransform::DocumentSourceChangeStreamTransform( const boost::intrusive_ptr<ExpressionContext>& expCtx, DocumentSourceChangeStreamSpec spec) - : DocumentSource(DocumentSourceChangeStreamTransform::kStageName, expCtx), + : DocumentSourceInternalChangeStreamStage(DocumentSourceChangeStreamTransform::kStageName, + expCtx), _changeStreamSpec(std::move(spec)), _transformer(expCtx, _changeStreamSpec), _isIndependentOfAnyCollection(expCtx->ns.isCollectionlessAggregateNS()) { @@ -103,16 +104,115 @@ StageConstraints DocumentSourceChangeStreamTransform::constraints( return constraints; } -Value DocumentSourceChangeStreamTransform::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +namespace { + +template <typename T> +void serializeSpecField(BSONObjBuilder* builder, + const SerializationOptions& opts, + const StringData& fieldName, + const boost::optional<T>& value) { + if (value) { + opts.serializeLiteral((*value).toBSON()).addToBsonObj(builder, fieldName); + } +} + +template <> +void serializeSpecField(BSONObjBuilder* builder, + const SerializationOptions& opts, + const StringData& fieldName, + const boost::optional<Timestamp>& value) { + if (value) { + opts.serializeLiteral(*value).addToBsonObj(builder, fieldName); + } +} + +template <typename T> +void serializeSpecField(BSONObjBuilder* builder, + const SerializationOptions& opts, + const StringData& fieldName, + const T& value) { + opts.appendLiteral(builder, fieldName, value); +} + +template <> +void serializeSpecField(BSONObjBuilder* builder, + const SerializationOptions& opts, + const StringData& fieldName, + const mongo::OptionalBool& value) { + if (value.has_value()) { + opts.appendLiteral(builder, fieldName, value.value_or(true)); + } +} + +void serializeSpec(const DocumentSourceChangeStreamSpec& spec, + const SerializationOptions& opts, + BSONObjBuilder* builder) { + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kResumeAfterFieldName, + spec.getResumeAfter()); + serializeSpecField( + builder, opts, DocumentSourceChangeStreamSpec::kStartAfterFieldName, spec.getStartAfter()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kStartAtOperationTimeFieldName, + spec.getStartAtOperationTime()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kFullDocumentFieldName, + ::mongo::FullDocumentMode_serializer(spec.getFullDocument())); + serializeSpecField( + builder, + opts, + DocumentSourceChangeStreamSpec::kFullDocumentBeforeChangeFieldName, + ::mongo::FullDocumentBeforeChangeMode_serializer(spec.getFullDocumentBeforeChange())); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kAllChangesForClusterFieldName, + spec.getAllChangesForCluster()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kShowMigrationEventsFieldName, + spec.getShowMigrationEvents()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kShowSystemEventsFieldName, + spec.getShowSystemEvents()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kAllowToRunOnConfigDBFieldName, + spec.getAllowToRunOnConfigDB()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kAllowToRunOnSystemNSFieldName, + spec.getAllowToRunOnSystemNS()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName, + spec.getShowExpandedEvents()); + serializeSpecField(builder, + opts, + DocumentSourceChangeStreamSpec::kShowRawUpdateDescriptionFieldName, + spec.getShowRawUpdateDescription()); +} + +} // namespace + +Value DocumentSourceChangeStreamTransform::serialize(const SerializationOptions& opts) const { + if (opts.verbosity) { return Value(Document{{DocumentSourceChangeStream::kStageName, Document{{"stage"_sd, "internalTransform"_sd}, - {"options"_sd, _changeStreamSpec.toBSON()}}}}); + {"options"_sd, _changeStreamSpec.toBSON(opts)}}}}); } - return Value( - Document{{DocumentSourceChangeStreamTransform::kStageName, _changeStreamSpec.toBSON()}}); + // Internal change stream stages are not serialized for query stats. Query stats uses this stage + // to serialize the user specified stage, and therefore if serializing for query stats, we + // should use the '$changeStream' stage name. + auto stageName = + (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || opts.transformIdentifiers) + ? DocumentSourceChangeStream::kStageName + : DocumentSourceChangeStreamTransform::kStageName; + return Value(Document{{stageName, _changeStreamSpec.toBSON(opts)}}); } DepsTracker::State DocumentSourceChangeStreamTransform::getDependencies(DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/document_source_change_stream_transform.h b/src/mongo/db/pipeline/document_source_change_stream_transform.h index 71941800da9..52284954d70 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_transform.h +++ b/src/mongo/db/pipeline/document_source_change_stream_transform.h @@ -34,7 +34,7 @@ namespace mongo { -class DocumentSourceChangeStreamTransform : public DocumentSource { +class DocumentSourceChangeStreamTransform : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamTransform"_sd; @@ -54,7 +54,14 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + + /** + * This function should never be called, since this DocumentSource has its own serialize method. + */ + Value doSerialize(const SerializationOptions& opts) const final { + MONGO_UNREACHABLE; + } StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp index ae65c696d65..6f4877f878c 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.cpp @@ -104,7 +104,7 @@ DocumentSourceChangeStreamUnwindTransaction::createFromBson( DocumentSourceChangeStreamUnwindTransaction::DocumentSourceChangeStreamUnwindTransaction( const BSONObj& filter, const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSource(kStageName, expCtx) { + : DocumentSourceInternalChangeStreamStage(kStageName, expCtx) { rebuild(filter); } @@ -126,19 +126,23 @@ StageConstraints DocumentSourceChangeStreamUnwindTransaction::constraints( ChangeStreamRequirement::kChangeStreamStage); } -Value DocumentSourceChangeStreamUnwindTransaction::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - tassert(5467604, "expression has not been initialized", _expression); +Value DocumentSourceChangeStreamUnwindTransaction::doSerialize( + const SerializationOptions& opts) const { + tassert(7481400, "expression has not been initialized", _expression); - if (explain) { - return Value( - DOC(DocumentSourceChangeStream::kStageName << DOC("stage" - << "internalUnwindTransaction"_sd - << "filter" << _filter))); + if (opts.verbosity) { + BSONObjBuilder builder; + builder.append("stage"_sd, "internalUnwindTransaction"_sd); + builder.append(DocumentSourceChangeStreamUnwindTransactionSpec::kFilterFieldName, + _expression->serialize(opts)); + + return Value(DOC(DocumentSourceChangeStream::kStageName << builder.obj())); } - DocumentSourceChangeStreamUnwindTransactionSpec spec(_filter); - return Value(Document{{kStageName, Value(spec.toBSON())}}); + // 'SerializationOptions' are not required here, since serialization for explain and query + // stats occur before this function call. + return Value(Document{ + {kStageName, Value{DocumentSourceChangeStreamUnwindTransactionSpec{_filter}.toBSON()}}}); } DepsTracker::State DocumentSourceChangeStreamUnwindTransaction::getDependencies( diff --git a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h index ce7278b3b3e..d9e794f5551 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h +++ b/src/mongo/db/pipeline/document_source_change_stream_unwind_transaction.h @@ -41,7 +41,7 @@ namespace mongo { * output, but all other entries pass through unmodified. Note that the namespace filter applies * only to unwound transaction operations, not to any other entries. */ -class DocumentSourceChangeStreamUnwindTransaction : public DocumentSource { +class DocumentSourceChangeStreamUnwindTransaction : public DocumentSourceInternalChangeStreamStage { public: static constexpr StringData kStageName = "$_internalChangeStreamUnwindTransaction"_sd; @@ -55,7 +55,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value doSerialize(const SerializationOptions& opts = SerializationOptions{}) const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/pipeline/document_source_coll_stats.cpp b/src/mongo/db/pipeline/document_source_coll_stats.cpp index 2881425d5e4..a947f1589e6 100644 --- a/src/mongo/db/pipeline/document_source_coll_stats.cpp +++ b/src/mongo/db/pipeline/document_source_coll_stats.cpp @@ -129,8 +129,8 @@ DocumentSource::GetNextResult DocumentSourceCollStats::doGetNext() { return {Document(makeStatsForNs(pExpCtx, pExpCtx->ns, _collStatsSpec))}; } -Value DocumentSourceCollStats::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{getSourceName(), _collStatsSpec.toBSON()}}); +Value DocumentSourceCollStats::serialize(const SerializationOptions& opts) const { + return Value(Document{{getSourceName(), _collStatsSpec.toBSON(opts)}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_coll_stats.h b/src/mongo/db/pipeline/document_source_coll_stats.h index 074683367e1..5c5f3d5482b 100644 --- a/src/mongo/db/pipeline/document_source_coll_stats.h +++ b/src/mongo/db/pipeline/document_source_coll_stats.h @@ -113,7 +113,7 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; static boost::intrusive_ptr<DocumentSource> createFromBson( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); diff --git a/src/mongo/db/pipeline/document_source_coll_stats.idl b/src/mongo/db/pipeline/document_source_coll_stats.idl index 264a4004a5c..b432912392a 100644 --- a/src/mongo/db/pipeline/document_source_coll_stats.idl +++ b/src/mongo/db/pipeline/document_source_coll_stats.idl @@ -39,36 +39,50 @@ structs: LatencyStatsSpec: description: Represents the 'latencyStats' argument to the $collStats stage. strict: true - fields: + query_shape_component: true + fields: histograms: description: Adds latency histogram information to the embedded documents in latencyStats if true. type: optionalBool + # Do not abstract this literal, since it is parameterizing the stage like an enum rather than representing + # real user input. + query_shape: parameter DocumentSourceCollStatsSpec: description: Specification for a $collStats stage. strict: true + query_shape_component: true fields: latencyStats: description: A request to include latency stats in the $collStats output. type: LatencyStatsSpec optional: true + query_shape: literal storageStats: description: Adds storage statistics to the return document. type: StorageStatsSpec optional: true + query_shape: literal count: description: Adds the total number of documents in the collection to the return document. type: object validator: callback: validateObjectIsEmpty optional: true + # Treat this value as a parameter (meaning it won't be abstracted or transformed) because + # the _only_ accepted value is the empty object. There's no transformation needed. + query_shape: parameter queryExecStats: description: Adds query execution statistics to the return document. type: object validator: callback: validateObjectIsEmpty optional: true + # Treat this value as a parameter (meaning it won't be abstracted or transformed) because + # the _only_ accepted value is the empty object. There's no transformation needed. + query_shape: parameter $_requestOnTimeseriesView: description: When set to true, $collStats stage requests statistics from the view namespace. When set to false, $collStats stage requests statistics from the underlying collection. cpp_name: requestOnTimeseriesView type: optionalBool + query_shape: parameter diff --git a/src/mongo/db/pipeline/document_source_coll_stats_test.cpp b/src/mongo/db/pipeline/document_source_coll_stats_test.cpp new file mode 100644 index 00000000000..f958307c8be --- /dev/null +++ b/src/mongo/db/pipeline/document_source_coll_stats_test.cpp @@ -0,0 +1,129 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_coll_stats.h" +#include "mongo/db/pipeline/document_source_coll_stats_gen.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/unittest/bson_test_util.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { +using DocumentSourceCollStatsTest = AggregationContextFixture; + +auto representativeShape(const DocumentSourceCollStats& collStatsStage) { + SerializationOptions opts{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + return collStatsStage.serialize(opts).getDocument().toBson(); +} + +TEST_F(DocumentSourceCollStatsTest, QueryShape) { + auto spec = DocumentSourceCollStatsSpec(); + + auto stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$collStats":{}})", + redact(*stage)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$collStats":{}})", + representativeShape(*stage)); + + spec.setCount(BSONObj()); + spec.setQueryExecStats(BSONObj()); + stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$collStats":{"count":{},"queryExecStats":{}}})", + redact(*stage)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$collStats":{"count":{},"queryExecStats":{}}})", + representativeShape(*stage)); + + auto latencyStats = LatencyStatsSpec(); + latencyStats.setHistograms(true); + spec.setLatencyStats(latencyStats); + stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$collStats": { + "latencyStats": { + "histograms": true + }, + "count": {}, + "queryExecStats": {} + } + })", + redact(*stage)); + + auto storageStats = StorageStatsSpec(); + storageStats.setScale(2); + storageStats.setVerbose(true); + spec.setStorageStats(storageStats); + stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$collStats": { + "latencyStats": { + "histograms": true + }, + "storageStats": { + "scale": "?number", + "verbose": true, + "waitForLock": true, + "numericOnly": false + }, + "count": {}, + "queryExecStats": {} + } + })", + redact(*stage)); + + storageStats.setWaitForLock(false); + storageStats.setNumericOnly(false); + spec.setStorageStats(storageStats); + stage = make_intrusive<DocumentSourceCollStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$collStats": { + "latencyStats": { + "histograms": true + }, + "storageStats": { + "scale": "?number", + "verbose": true, + "waitForLock": false, + "numericOnly": false + }, + "count": {}, + "queryExecStats": {} + } + })", + redact(*stage)); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_count_test.cpp b/src/mongo/db/pipeline/document_source_count_test.cpp index 9ffd8a6d120..399086b0b52 100644 --- a/src/mongo/db/pipeline/document_source_count_test.cpp +++ b/src/mongo/db/pipeline/document_source_count_test.cpp @@ -65,10 +65,13 @@ public: dynamic_cast<DocumentSourceSingleDocumentTransformation*>(result.back().get()); ASSERT(projectStage); - auto explain = ExplainOptions::Verbosity::kQueryPlanner; vector<Value> explainedStages; - groupStage->serializeToArray(explainedStages, explain); - projectStage->serializeToArray(explainedStages, explain); + groupStage->serializeToArray( + explainedStages, + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); + projectStage->serializeToArray( + explainedStages, + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); ASSERT_EQUALS(explainedStages.size(), 2UL); StringData countName = countSpec.firstElement().valueStringData(); diff --git a/src/mongo/db/pipeline/document_source_current_op.cpp b/src/mongo/db/pipeline/document_source_current_op.cpp index 6ef6e2c762e..fc5c8b32a1f 100644 --- a/src/mongo/db/pipeline/document_source_current_op.cpp +++ b/src/mongo/db/pipeline/document_source_current_op.cpp @@ -70,8 +70,8 @@ std::unique_ptr<DocumentSourceCurrentOp::LiteParsed> DocumentSourceCurrentOp::Li << typeName(spec.type())); } - auto allUsers = UserMode::kExcludeOthers; - auto localOps = LocalOpsMode::kRemoteShardOps; + auto allUsers = kDefaultUserMode; + auto localOps = kDefaultLocalOpsMode; // Check the spec for all fields named 'allUsers'. If any of them are 'true', we require // the 'inprog' privilege. This avoids the possibility that a spec with multiple @@ -113,13 +113,14 @@ const char* DocumentSourceCurrentOp::getSourceName() const { DocumentSource::GetNextResult DocumentSourceCurrentOp::doGetNext() { if (_ops.empty()) { - _ops = pExpCtx->mongoProcessInterface->getCurrentOps(pExpCtx, - _includeIdleConnections, - _includeIdleSessions, - _includeOpsFromAllUsers, - _truncateOps, - _idleCursors, - _backtrace); + _ops = pExpCtx->mongoProcessInterface->getCurrentOps( + pExpCtx, + _includeIdleConnections.value_or(kDefaultConnMode), + _includeIdleSessions.value_or(kDefaultSessionMode), + _includeOpsFromAllUsers.value_or(kDefaultUserMode), + _truncateOps.value_or(kDefaultTruncationMode), + _idleCursors.value_or(kDefaultCursorMode), + _backtrace.value_or(kDefaultBacktraceMode)); _opsIter = _ops.begin(); @@ -191,13 +192,13 @@ intrusive_ptr<DocumentSource> DocumentSourceCurrentOp::createFromBson( "$currentOp must be run against the 'admin' database with {aggregate: 1}", nss.db() == NamespaceString::kAdminDb && nss.isCollectionlessAggregateNS()); - ConnMode includeIdleConnections = ConnMode::kExcludeIdle; - SessionMode includeIdleSessions = SessionMode::kIncludeIdle; - UserMode includeOpsFromAllUsers = UserMode::kExcludeOthers; - LocalOpsMode showLocalOpsOnMongoS = LocalOpsMode::kRemoteShardOps; - TruncationMode truncateOps = TruncationMode::kNoTruncation; - CursorMode idleCursors = CursorMode::kExcludeCursors; - BacktraceMode backtrace = BacktraceMode::kExcludeBacktrace; + boost::optional<ConnMode> includeIdleConnections; + boost::optional<SessionMode> includeIdleSessions; + boost::optional<UserMode> includeOpsFromAllUsers; + boost::optional<LocalOpsMode> showLocalOpsOnMongoS; + boost::optional<TruncationMode> truncateOps; + boost::optional<CursorMode> idleCursors; + boost::optional<BacktraceMode> backtrace; for (auto&& elem : spec.embeddedObject()) { const auto fieldName = elem.fieldNameStringData(); @@ -278,13 +279,13 @@ intrusive_ptr<DocumentSource> DocumentSourceCurrentOp::createFromBson( intrusive_ptr<DocumentSourceCurrentOp> DocumentSourceCurrentOp::create( const boost::intrusive_ptr<ExpressionContext>& pExpCtx, - ConnMode includeIdleConnections, - SessionMode includeIdleSessions, - UserMode includeOpsFromAllUsers, - LocalOpsMode showLocalOpsOnMongoS, - TruncationMode truncateOps, - CursorMode idleCursors, - BacktraceMode backtrace) { + boost::optional<ConnMode> includeIdleConnections, + boost::optional<SessionMode> includeIdleSessions, + boost::optional<UserMode> includeOpsFromAllUsers, + boost::optional<LocalOpsMode> showLocalOpsOnMongoS, + boost::optional<TruncationMode> truncateOps, + boost::optional<CursorMode> idleCursors, + boost::optional<BacktraceMode> backtrace) { return new DocumentSourceCurrentOp(pExpCtx, includeIdleConnections, includeIdleSessions, @@ -295,22 +296,38 @@ intrusive_ptr<DocumentSourceCurrentOp> DocumentSourceCurrentOp::create( backtrace); } -Value DocumentSourceCurrentOp::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceCurrentOp::serialize(const SerializationOptions& opts) const { return Value(Document{ {getSourceName(), - Document{{kIdleConnectionsFieldName, - _includeIdleConnections == ConnMode::kIncludeIdle ? Value(true) : Value()}, - {kIdleSessionsFieldName, - _includeIdleSessions == SessionMode::kExcludeIdle ? Value(false) : Value()}, - {kAllUsersFieldName, - _includeOpsFromAllUsers == UserMode::kIncludeAll ? Value(true) : Value()}, - {kLocalOpsFieldName, - _showLocalOpsOnMongoS == LocalOpsMode::kLocalMongosOps ? Value(true) : Value()}, - {kTruncateOpsFieldName, - _truncateOps == TruncationMode::kTruncateOps ? Value(true) : Value()}, - {kIdleCursorsFieldName, - _idleCursors == CursorMode::kIncludeCursors ? Value(true) : Value()}, - {kBacktraceFieldName, - _backtrace == BacktraceMode::kIncludeBacktrace ? Value(true) : Value()}}}}); + Document{ + {kIdleConnectionsFieldName, + _includeIdleConnections.has_value() + ? opts.serializeLiteral(_includeIdleConnections.value() == ConnMode::kIncludeIdle) + : Value()}, + {kIdleSessionsFieldName, + _includeIdleSessions.has_value() + ? opts.serializeLiteral(_includeIdleSessions.value() == SessionMode::kIncludeIdle) + : Value()}, + {kAllUsersFieldName, + _includeOpsFromAllUsers.has_value() + ? opts.serializeLiteral(_includeOpsFromAllUsers.value() == UserMode::kIncludeAll) + : Value()}, + {kLocalOpsFieldName, + _showLocalOpsOnMongoS.has_value() + ? opts.serializeLiteral(_showLocalOpsOnMongoS.value() == + LocalOpsMode::kLocalMongosOps) + : Value()}, + {kTruncateOpsFieldName, + _truncateOps.has_value() + ? opts.serializeLiteral(_truncateOps.value() == TruncationMode::kTruncateOps) + : Value()}, + {kIdleCursorsFieldName, + _idleCursors.has_value() + ? opts.serializeLiteral(_idleCursors.value() == CursorMode::kIncludeCursors) + : Value()}, + {kBacktraceFieldName, + _backtrace.has_value() + ? opts.serializeLiteral(_backtrace.value() == BacktraceMode::kIncludeBacktrace) + : Value()}}}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_current_op.h b/src/mongo/db/pipeline/document_source_current_op.h index 49e0be6c35d..4cc53475e4b 100644 --- a/src/mongo/db/pipeline/document_source_current_op.h +++ b/src/mongo/db/pipeline/document_source_current_op.h @@ -45,6 +45,14 @@ public: static constexpr StringData kStageName = "$currentOp"_sd; + static constexpr ConnMode kDefaultConnMode = ConnMode::kExcludeIdle; + static constexpr SessionMode kDefaultSessionMode = SessionMode::kIncludeIdle; + static constexpr UserMode kDefaultUserMode = UserMode::kExcludeOthers; + static constexpr LocalOpsMode kDefaultLocalOpsMode = LocalOpsMode::kRemoteShardOps; + static constexpr TruncationMode kDefaultTruncationMode = TruncationMode::kNoTruncation; + static constexpr CursorMode kDefaultCursorMode = CursorMode::kExcludeCursors; + static constexpr BacktraceMode kDefaultBacktraceMode = BacktraceMode::kExcludeBacktrace; + class LiteParsed final : public LiteParsedDocumentSource { public: static std::unique_ptr<LiteParsed> parse(const NamespaceString& nss, @@ -98,29 +106,28 @@ public: static boost::intrusive_ptr<DocumentSourceCurrentOp> create( const boost::intrusive_ptr<ExpressionContext>& pExpCtx, - ConnMode includeIdleConnections = ConnMode::kExcludeIdle, - SessionMode includeIdleSessions = SessionMode::kIncludeIdle, - UserMode includeOpsFromAllUsers = UserMode::kExcludeOthers, - LocalOpsMode showLocalOpsOnMongoS = LocalOpsMode::kRemoteShardOps, - TruncationMode truncateOps = TruncationMode::kNoTruncation, - CursorMode idleCursors = CursorMode::kExcludeCursors, - BacktraceMode backtrace = BacktraceMode::kExcludeBacktrace); + boost::optional<ConnMode> includeIdleConnections = boost::none, + boost::optional<SessionMode> includeIdleSessions = boost::none, + boost::optional<UserMode> includeOpsFromAllUsers = boost::none, + boost::optional<LocalOpsMode> showLocalOpsOnMongoS = boost::none, + boost::optional<TruncationMode> truncateOps = boost::none, + boost::optional<CursorMode> idleCursors = boost::none, + boost::optional<BacktraceMode> backtrace = boost::none); const char* getSourceName() const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final { - StageConstraints constraints(StreamType::kStreaming, - PositionRequirement::kFirst, - (_showLocalOpsOnMongoS == LocalOpsMode::kLocalMongosOps - ? HostTypeRequirement::kLocalOnly - : HostTypeRequirement::kAnyShard), - DiskUseRequirement::kNoDiskUse, - FacetRequirement::kNotAllowed, - TransactionRequirement::kNotAllowed, - LookupRequirement::kAllowed, - (_showLocalOpsOnMongoS == LocalOpsMode::kLocalMongosOps - ? UnionRequirement::kNotAllowed - : UnionRequirement::kAllowed)); + bool showLocalOps = + _showLocalOpsOnMongoS.value_or(kDefaultLocalOpsMode) == LocalOpsMode::kLocalMongosOps; + StageConstraints constraints( + StreamType::kStreaming, + PositionRequirement::kFirst, + (showLocalOps ? HostTypeRequirement::kLocalOnly : HostTypeRequirement::kAnyShard), + DiskUseRequirement::kNoDiskUse, + FacetRequirement::kNotAllowed, + TransactionRequirement::kNotAllowed, + LookupRequirement::kAllowed, + (showLocalOps ? UnionRequirement::kNotAllowed : UnionRequirement::kAllowed)); constraints.isIndependentOfAnyCollection = true; constraints.requiresInputDocSource = false; @@ -134,17 +141,17 @@ public: static boost::intrusive_ptr<DocumentSource> createFromBson( BSONElement spec, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; private: DocumentSourceCurrentOp(const boost::intrusive_ptr<ExpressionContext>& pExpCtx, - ConnMode includeIdleConnections, - SessionMode includeIdleSessions, - UserMode includeOpsFromAllUsers, - LocalOpsMode showLocalOpsOnMongoS, - TruncationMode truncateOps, - CursorMode idleCursors, - BacktraceMode backtrace) + boost::optional<ConnMode> includeIdleConnections, + boost::optional<SessionMode> includeIdleSessions, + boost::optional<UserMode> includeOpsFromAllUsers, + boost::optional<LocalOpsMode> showLocalOpsOnMongoS, + boost::optional<TruncationMode> truncateOps, + boost::optional<CursorMode> idleCursors, + boost::optional<BacktraceMode> backtrace) : DocumentSource(kStageName, pExpCtx), _includeIdleConnections(includeIdleConnections), _includeIdleSessions(includeIdleSessions), @@ -156,13 +163,13 @@ private: GetNextResult doGetNext() final; - ConnMode _includeIdleConnections = ConnMode::kExcludeIdle; - SessionMode _includeIdleSessions = SessionMode::kIncludeIdle; - UserMode _includeOpsFromAllUsers = UserMode::kExcludeOthers; - LocalOpsMode _showLocalOpsOnMongoS = LocalOpsMode::kRemoteShardOps; - TruncationMode _truncateOps = TruncationMode::kNoTruncation; - CursorMode _idleCursors = CursorMode::kExcludeCursors; - BacktraceMode _backtrace = BacktraceMode::kExcludeBacktrace; + boost::optional<ConnMode> _includeIdleConnections; + boost::optional<SessionMode> _includeIdleSessions; + boost::optional<UserMode> _includeOpsFromAllUsers; + boost::optional<LocalOpsMode> _showLocalOpsOnMongoS; + boost::optional<TruncationMode> _truncateOps; + boost::optional<CursorMode> _idleCursors; + boost::optional<BacktraceMode> _backtrace; std::string _shardName; diff --git a/src/mongo/db/pipeline/document_source_current_op_test.cpp b/src/mongo/db/pipeline/document_source_current_op_test.cpp index 260bfbf0829..440205960f2 100644 --- a/src/mongo/db/pipeline/document_source_current_op_test.cpp +++ b/src/mongo/db/pipeline/document_source_current_op_test.cpp @@ -154,10 +154,10 @@ TEST_F(DocumentSourceCurrentOpTest, ShouldFailToParseIfUnrecognisedParameterSpec ErrorCodes::FailedToParse); } -TEST_F(DocumentSourceCurrentOpTest, ShouldParseAndSerializeNonDefaultOptionalArguments) { +TEST_F(DocumentSourceCurrentOpTest, ShouldParseAndSerializeAllExplicitlySpecifiedArguments) { const auto specObj = fromjson( - "{$currentOp:{idleConnections:true, idleSessions:false, allUsers:true, localOps:true, " - "truncateOps:true}}"); + "{$currentOp:{idleConnections:false, idleSessions:false, allUsers:true, localOps:true, " + "truncateOps:false}}"); const auto parsed = DocumentSourceCurrentOp::createFromBson(specObj.firstElement(), getExpCtx()); @@ -165,19 +165,42 @@ TEST_F(DocumentSourceCurrentOpTest, ShouldParseAndSerializeNonDefaultOptionalArg const auto currentOp = static_cast<DocumentSourceCurrentOp*>(parsed.get()); const auto expectedOutput = Document{{"$currentOp", - Document{{"idleConnections", true}, + Document{{"idleConnections", false}, {"idleSessions", false}, {"allUsers", true}, {"localOps", true}, - {"truncateOps", true}}}}; + {"truncateOps", false}}}}; ASSERT_DOCUMENT_EQ(currentOp->serialize().getDocument(), expectedOutput); } -TEST_F(DocumentSourceCurrentOpTest, ShouldParseButNotSerializeDefaultOptionalArguments) { - const auto specObj = fromjson( - "{$currentOp:{idleConnections:false, idleSessions:true, allUsers:false, localOps:false, " - "truncateOps:false}}"); +TEST_F(DocumentSourceCurrentOpTest, + ShouldParseAndSerializeAllExplicitlySpecifiedArgumentsWithRedaction) { + auto spec = fromjson( + R"({ + $currentOp: { + idleConnections: true, + allUsers: false, + idleSessions: false, + localOps: true + } + })"); + auto docSource = DocumentSourceCurrentOp::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$currentOp": { + "idleConnections": "?bool", + "idleSessions": "?bool", + "allUsers": "?bool", + "localOps": "?bool" + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceCurrentOpTest, ShouldNotSerializeOmittedOptionalArguments) { + const auto specObj = fromjson("{$currentOp:{}}"); const auto parsed = DocumentSourceCurrentOp::createFromBson(specObj.firstElement(), getExpCtx()); @@ -189,17 +212,17 @@ TEST_F(DocumentSourceCurrentOpTest, ShouldParseButNotSerializeDefaultOptionalArg ASSERT_DOCUMENT_EQ(currentOp->serialize().getDocument(), expectedOutput); } -TEST_F(DocumentSourceCurrentOpTest, ShouldNotSerializeOmittedOptionalArguments) { +TEST_F(DocumentSourceCurrentOpTest, ShouldNotSerializeOmittedOptionalArgumentsWithRedaction) { const auto specObj = fromjson("{$currentOp:{}}"); - const auto parsed = + const auto docSource = DocumentSourceCurrentOp::createFromBson(specObj.firstElement(), getExpCtx()); - const auto currentOp = static_cast<DocumentSourceCurrentOp*>(parsed.get()); - - const auto expectedOutput = Document{{"$currentOp", Document{}}}; - - ASSERT_DOCUMENT_EQ(currentOp->serialize().getDocument(), expectedOutput); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$currentOp": {} + })", + redact(*docSource)); } TEST_F(DocumentSourceCurrentOpTest, ShouldReturnEOFImmediatelyIfNoCurrentOps) { diff --git a/src/mongo/db/pipeline/document_source_cursor.cpp b/src/mongo/db/pipeline/document_source_cursor.cpp index 55bacba9d7f..520c32f74cc 100644 --- a/src/mongo/db/pipeline/document_source_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_cursor.cpp @@ -206,9 +206,12 @@ void DocumentSourceCursor::recordPlanSummaryStats() { _exec->getPlanExplainer().getSummaryStats(&_stats.planSummaryStats); } -Value DocumentSourceCursor::serialize(boost::optional<ExplainOptions::Verbosity> verbosity) const { - // We never parse a DocumentSourceCursor, so we only serialize for explain. - if (!verbosity) +Value DocumentSourceCursor::serialize(const SerializationOptions& opts) const { + auto verbosity = opts.verbosity; + // We never parse a DocumentSourceCursor, so we only serialize for explain. Since it's never + // part of user input, there's no need to compute its query shape. + if (!verbosity || opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) return Value(); invariant(_exec); diff --git a/src/mongo/db/pipeline/document_source_cursor.h b/src/mongo/db/pipeline/document_source_cursor.h index 38a2ab183ae..29f724da0a1 100644 --- a/src/mongo/db/pipeline/document_source_cursor.h +++ b/src/mongo/db/pipeline/document_source_cursor.h @@ -72,7 +72,7 @@ public: const char* getSourceName() const override; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { StageConstraints constraints(StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_densify.cpp b/src/mongo/db/pipeline/document_source_densify.cpp index 63f3253f57a..b7899f078a2 100644 --- a/src/mongo/db/pipeline/document_source_densify.cpp +++ b/src/mongo/db/pipeline/document_source_densify.cpp @@ -645,17 +645,16 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceInternalDensify::createFromBs return results.front(); } -Value DocumentSourceInternalDensify::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalDensify::serialize(const SerializationOptions& opts) const { MutableDocument spec; - spec[kFieldFieldName] = Value(_field.fullPath()); + spec[kFieldFieldName] = Value(opts.serializeFieldPath(_field)); std::vector<Value> serializedPartitionByFields(_partitions.size()); std::transform(_partitions.begin(), _partitions.end(), serializedPartitionByFields.begin(), - [&](FieldPath field) -> Value { return Value(field.fullPath()); }); + [&](FieldPath field) -> Value { return Value(opts.serializeFieldPath(field)); }); spec[kPartitionByFieldsFieldName] = Value(serializedPartitionByFields); - spec[kRangeFieldName] = _range.serialize(); + spec[kRangeFieldName] = _range.serialize(opts); MutableDocument out; out[getSourceName()] = Value(spec.freeze()); diff --git a/src/mongo/db/pipeline/document_source_densify.h b/src/mongo/db/pipeline/document_source_densify.h index 6415c5425d8..1214c0bac53 100644 --- a/src/mongo/db/pipeline/document_source_densify.h +++ b/src/mongo/db/pipeline/document_source_densify.h @@ -241,15 +241,16 @@ public: static RangeStatement parse(RangeSpec spec); - Value serialize() const { + Value serialize(const SerializationOptions& opts) const { MutableDocument spec; - spec[kArgStep] = _step; + spec[kArgStep] = opts.serializeLiteral(_step); spec[kArgBounds] = stdx::visit( visit_helper::Overloaded{[&](Full) { return Value(kValFull); }, [&](Partition) { return Value(kValPartition); }, [&](ExplicitBounds bounds) { return Value(std::vector<Value>( - {bounds.first.toValue(), bounds.second.toValue()})); + {opts.serializeLiteral(bounds.first.toValue()), + opts.serializeLiteral(bounds.second.toValue())})); }}, _bounds); if (_unit) @@ -380,7 +381,7 @@ public: return kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DepsTracker::State getDependencies(DepsTracker* deps) const final { deps->fields.insert(_field.fullPath()); diff --git a/src/mongo/db/pipeline/document_source_densify_test.cpp b/src/mongo/db/pipeline/document_source_densify_test.cpp index 4be40700323..f83101b7736 100644 --- a/src/mongo/db/pipeline/document_source_densify_test.cpp +++ b/src/mongo/db/pipeline/document_source_densify_test.cpp @@ -49,6 +49,7 @@ using DensifyExplicitNumericTest = AggregationContextFixture; using DensifyPartitionNumericTest = AggregationContextFixture; using DensifyCloneTest = AggregationContextFixture; using DensifyStepTest = AggregationContextFixture; +using DensifyRedactionTest = AggregationContextFixture; Date_t makeDate(std::string dateStr) { auto statusDate = dateFromISOString(dateStr); @@ -1438,5 +1439,174 @@ TEST(DensifyStepTest, InternalDensifyIsOffStepForDaysWithLargeDateStep) { ASSERT_FALSE(val.isOnStepRelativeTo(base, range)); } + +TEST_F(DensifyRedactionTest, RedactionDateBounds) { + auto spec = fromjson(R"({ + $densify: { + field: "a", + range: { + step: 1, + unit: "hour", + bounds: [ + {$date: "2023-04-23T00:00:00.000Z"}, + {$date: "2023-04-23T08:00:00.000Z"} + ] + } + } + })"); + + auto docSource = + DocumentSourceInternalDensify::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalDensify": { + "field": "HASH<a>", + "partitionByFields": [], + "range": { + "step": "?number", + "bounds": [ + "?date", + "?date" + ], + "unit": "hour" + } + } + })", + redact(*docSource)); +} + +TEST_F(DensifyRedactionTest, RedactionFullBoundsWithPartitionFields) { + auto spec = fromjson(R"({ + $densify: { + field: "foo", + partitionByFields: ["a", "b", "c.d"], + range: { + bounds: "full", + step: 100 + } + } + })"); + auto docSource = + DocumentSourceInternalDensify::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalDensify": { + "field": "HASH<foo>", + "partitionByFields": [ + "HASH<a>", + "HASH<b>", + "HASH<c>.HASH<d>" + ], + "range": { + "step": "?number", + "bounds": "full" + } + } + })", + redact(*docSource)); +} + +TEST_F(DensifyRedactionTest, RedactionPartitionBounds) { + auto spec = fromjson(R"({ + $densify: { + field: "x", + partitionByFields: ["foo"], + range: { + bounds: "partition", + step: 50, + unit: "second" + } + } + })"); + auto docSource = + DocumentSourceInternalDensify::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalDensify": { + "field": "HASH<x>", + "partitionByFields": [ + "HASH<foo>" + ], + "range": { + "step": "?number", + "bounds": "partition", + "unit": "second" + } + } + })", + redact(*docSource)); +} + +void assertRangeTimeUnitSerialization(const boost::intrusive_ptr<ExpressionContext>& expCtx, + BSONObj inputStage, + BSONObj expectedStage, + const SerializationOptions& opts) { + auto parsedStage = + DocumentSourceInternalDensify::createFromBson(inputStage.firstElement(), expCtx); + std::vector<Value> serialization; + parsedStage->serializeToArray(serialization, opts); + + auto serializedStage = serialization[0].getDocument().toBson(); + ASSERT_BSONOBJ_EQ(expectedStage, serializedStage); +} + +TEST_F(DensifyRedactionTest, RangeTimeUnitSerializationRepresentative) { + assertRangeTimeUnitSerialization( + getExpCtx(), + fromjson(R"({ + $densify: { + field: "x", + partitionByFields: ["foo"], + range: { + bounds: "partition", + step: 50, + unit: "second" + } + } + })"), + fromjson(R"({ + $_internalDensify: { + field: "x", + partitionByFields: [ + "foo" + ], + range: { + step: 1, + bounds: "partition", + unit: "second" + } + } + })"), + SerializationOptions::kRepresentativeQueryShapeSerializeOptions); +} + +TEST_F(DensifyRedactionTest, RangeTimeUnitSerializationDebug) { + assertRangeTimeUnitSerialization(getExpCtx(), + fromjson( + R"({ + $densify: { + field: "x", + partitionByFields: ["foo"], + range: { + bounds: "partition", + step: 50, + unit: "second" + } + } + })"), + fromjson( + R"({ + $_internalDensify: { + field: "x", + partitionByFields: ["foo"], + range: { + step: "?number", + bounds: "partition", + unit: "second" + } + } + })"), + SerializationOptions::kDebugQueryShapeSerializeOptions); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_documents.cpp b/src/mongo/db/pipeline/document_source_documents.cpp index 934116a7c27..23f5b8c96f8 100644 --- a/src/mongo/db/pipeline/document_source_documents.cpp +++ b/src/mongo/db/pipeline/document_source_documents.cpp @@ -50,26 +50,26 @@ REGISTER_DOCUMENT_SOURCE(documents, std::list<intrusive_ptr<DocumentSource>> DocumentSourceDocuments::createFromBson( BSONElement elem, const intrusive_ptr<ExpressionContext>& expCtx) { - // genField is a temporary field to hold docs to wire $project, - // $unwind, and $replaceRoot together. - auto genField = UUID::gen().toString(); - auto projectContent = BSON(genField << elem); + // kGenFieldName is a temporary field to hold docs to wire $project, $unwind, and $replaceRoot + // together. This may show up in explain, but it will not possibly overlap with user data, since + // the user data is nested a level below, within an array or object. + auto projectContent = BSON(kGenFieldName << elem); auto queue = DocumentSourceQueue::create(expCtx, DocumentSourceDocuments::kStageName); queue->emplace_back(Document{}); /* Create the following pipeline from $documents: [...] - * => [ queue([{}]), - * project: {tempDocumentsField: [...]}, - * unwind: "$tempDocumentsField", - * replaceWith: "$tempDocumentsField" ] + * => [ {$queue: [{}] }, + * {$project: {[kGenFieldName]: [...]}}, + * {$unwind: "$" + kGenFieldName}, + * {$replaceWith: "$" + kGenFieldName} ] */ - return { - queue, - DocumentSourceProject::create(projectContent, expCtx, elem.fieldNameStringData()), - DocumentSourceUnwind::create(expCtx, genField, false, {}, true), - DocumentSourceReplaceRoot::create(expCtx, - ExpressionFieldPath::createPathFromString( - expCtx.get(), genField, expCtx->variablesParseState), - "elements within the array passed to $documents")}; + return {queue, + DocumentSourceProject::create(projectContent, expCtx, elem.fieldNameStringData()), + DocumentSourceUnwind::create(expCtx, kGenFieldName, false, {}, true), + DocumentSourceReplaceRoot::create( + expCtx, + ExpressionFieldPath::createPathFromString( + expCtx.get(), kGenFieldName, expCtx->variablesParseState), + "elements within the array passed to $documents")}; } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_documents.h b/src/mongo/db/pipeline/document_source_documents.h index d0c2b660d87..992b6c6cc4e 100644 --- a/src/mongo/db/pipeline/document_source_documents.h +++ b/src/mongo/db/pipeline/document_source_documents.h @@ -64,9 +64,10 @@ public: } }; +static const std::string kGenFieldName = "_tempDocumentsField"s; static constexpr StringData kStageName = "$documents"_sd; -static std::list<boost::intrusive_ptr<DocumentSource>> createFromBson( +std::list<boost::intrusive_ptr<DocumentSource>> createFromBson( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); }; // namespace DocumentSourceDocuments diff --git a/src/mongo/db/pipeline/document_source_documents_test.cpp b/src/mongo/db/pipeline/document_source_documents_test.cpp new file mode 100644 index 00000000000..3067df16abc --- /dev/null +++ b/src/mongo/db/pipeline/document_source_documents_test.cpp @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_documents.h" +#include "mongo/db/pipeline/document_source_unwind.h" + +namespace mongo { + +using DocumentSourceDocumentsTest = AggregationContextFixture; + +TEST_F(DocumentSourceDocumentsTest, DocumentsStageRedactsCorrectly) { + auto spec = fromjson(R"({ + $documents: [ + { x: 10 }, { x: 2 }, { x: 5 } + ] + })"); + auto docSourcesList = DocumentSourceDocuments::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_EQ(4, docSourcesList.size()); + + // We must retrieve the internally-generated field name shared across these stages in order to + // make sure they're serialized properly. + std::vector<boost::intrusive_ptr<DocumentSource>> docSourcesVec(docSourcesList.begin(), + docSourcesList.end()); + auto unwindStage = static_cast<DocumentSourceUnwind*>(docSourcesVec[2].get()); + ASSERT(unwindStage); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$queue":"?array<?object>"})", + redact(*docSourcesVec[0])); + + + ASSERT_BSONOBJ_EQ( // NOLINT + fromjson(R"({ + "$project": { + "HASH<_id>": true, + "HASH<)" + + // SERVER-87666 ensure that the generated field is consistent. + DocumentSourceDocuments::kGenFieldName + + R"(>": "?array<?object>" + } + })"), + redact(*docSourcesVec[1])); + + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + "{'$unwind': {'path' : '$HASH<" + DocumentSourceDocuments::kGenFieldName + ">' } }", + redact(*docSourcesVec[2])); + + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + "{$replaceRoot: {newRoot: '$HASH<" + DocumentSourceDocuments::kGenFieldName + ">'}}", + redact(*docSourcesVec[3])); +} + +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_exchange.cpp b/src/mongo/db/pipeline/document_source_exchange.cpp index 931b78f316e..441779021e9 100644 --- a/src/mongo/db/pipeline/document_source_exchange.cpp +++ b/src/mongo/db/pipeline/document_source_exchange.cpp @@ -91,8 +91,8 @@ const char* DocumentSourceExchange::getSourceName() const { return kStageName.rawData(); } -Value DocumentSourceExchange::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << _exchange->getSpec().toBSON())); +Value DocumentSourceExchange::serialize(const SerializationOptions& opts) const { + return Value(DOC(getSourceName() << _exchange->getSpec().toBSON(opts))); } DocumentSourceExchange::DocumentSourceExchange( diff --git a/src/mongo/db/pipeline/document_source_exchange.h b/src/mongo/db/pipeline/document_source_exchange.h index dd2c00030de..544f6fb924f 100644 --- a/src/mongo/db/pipeline/document_source_exchange.h +++ b/src/mongo/db/pipeline/document_source_exchange.h @@ -221,7 +221,7 @@ public: const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; /** * DocumentSourceExchange does not have a direct source (it is reading through the shared diff --git a/src/mongo/db/pipeline/document_source_exchange_test.cpp b/src/mongo/db/pipeline/document_source_exchange_test.cpp index ac234357b17..4ed642c49c7 100644 --- a/src/mongo/db/pipeline/document_source_exchange_test.cpp +++ b/src/mongo/db/pipeline/document_source_exchange_test.cpp @@ -747,4 +747,30 @@ TEST_F(DocumentSourceExchangeTest, RejectInvalidMissingKeys) { Exchange(parseSpec(spec), Pipeline::create({}, getExpCtx())), AssertionException, 50967); } +TEST_F(DocumentSourceExchangeTest, QueryShape) { + const size_t nDocs = 500; + + auto source = getMockSource(nDocs); + + ExchangeSpec spec; + spec.setPolicy(ExchangePolicyEnum::kRoundRobin); + spec.setConsumers(1); + spec.setBufferSize(1024); + boost::intrusive_ptr<Exchange> ex = new Exchange(spec, Pipeline::create({source}, getExpCtx())); + boost::intrusive_ptr<DocumentSourceExchange> stage = + new DocumentSourceExchange(getExpCtx(), ex, 0, nullptr); + + ASSERT_BSONOBJ_EQ_AUTO( // + R"({ + "$_internalExchange": { + "policy": "roundrobin", + "consumers": "?number", + "orderPreserving": false, + "bufferSize": "?number", + "key": "?object" + } + })", + redact(*stage)); +} + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_facet.cpp b/src/mongo/db/pipeline/document_source_facet.cpp index bc02ae2b88b..6dfe8fd0a68 100644 --- a/src/mongo/db/pipeline/document_source_facet.cpp +++ b/src/mongo/db/pipeline/document_source_facet.cpp @@ -185,11 +185,12 @@ DocumentSource::GetNextResult DocumentSourceFacet::doGetNext() { return resultDoc.freeze(); } -Value DocumentSourceFacet::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceFacet::serialize(const SerializationOptions& opts) const { MutableDocument serialized; for (auto&& facet : _facets) { - serialized[facet.name] = Value(explain ? facet.pipeline->writeExplainOps(*explain) - : facet.pipeline->serialize()); + serialized[opts.serializeFieldPathFromString(facet.name)] = + Value(opts.verbosity ? facet.pipeline->writeExplainOps(opts) + : facet.pipeline->serialize(opts)); } return Value(Document{{"$facet", serialized.freezeToValue()}}); } @@ -336,20 +337,25 @@ intrusive_ptr<DocumentSource> DocumentSourceFacet::createFromBson( }); }); - // Validate that none of the facet pipelines have any conflicting HostTypeRequirements. This - // verifies both that all stages within each pipeline are consistent, and that the pipelines - // are consistent with one another. - if (!needsShard && pipeline->needsShard()) { - needsShard.emplace(facetName); - } - if (!needsMongoS && pipeline->needsMongosMerger()) { - needsMongoS.emplace(facetName); + // These checks potentially require that we check the catalog to determine where our data + // lives. In circumstances where we aren't actually running the query, we don't need to do + // this (and it can erroneously error - SERVER-83912). + if (expCtx->mongoProcessInterface->isExpectedToExecuteQueries()) { + // Validate that none of the facet pipelines have any conflicting HostTypeRequirements. + // This verifies both that all stages within each pipeline are consistent, and that the + // pipelines are consistent with one another. + if (!needsShard && pipeline->needsShard()) { + needsShard.emplace(facetName); + } + if (!needsMongoS && pipeline->needsMongosMerger()) { + needsMongoS.emplace(facetName); + } + uassert(ErrorCodes::IllegalOperation, + str::stream() << "$facet pipeline '" << *needsMongoS + << "' must run on mongoS, but '" << *needsShard + << "' requires a shard", + !(needsShard && needsMongoS)); } - uassert(ErrorCodes::IllegalOperation, - str::stream() << "$facet pipeline '" << *needsMongoS - << "' must run on mongoS, but '" << *needsShard - << "' requires a shard", - !(needsShard && needsMongoS)); facetPipelines.emplace_back(facetName, std::move(pipeline)); } diff --git a/src/mongo/db/pipeline/document_source_facet.h b/src/mongo/db/pipeline/document_source_facet.h index 193b9e12dd1..e3efb9c37cd 100644 --- a/src/mongo/db/pipeline/document_source_facet.h +++ b/src/mongo/db/pipeline/document_source_facet.h @@ -160,7 +160,7 @@ private: size_t bufferSizeBytes, size_t maxOutputDocBytes); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<TeeBuffer> _teeBuffer; std::vector<FacetPipeline> _facets; diff --git a/src/mongo/db/pipeline/document_source_facet_test.cpp b/src/mongo/db/pipeline/document_source_facet_test.cpp index 44e3d9668e9..78367b24b02 100644 --- a/src/mongo/db/pipeline/document_source_facet_test.cpp +++ b/src/mongo/db/pipeline/document_source_facet_test.cpp @@ -186,9 +186,19 @@ TEST_F(DocumentSourceFacetTest, ShouldAcceptLegalSpecification) { ASSERT_TRUE(facetStage.get()); } +/* + * Override the stub interface to allow full execution in these tests. + */ +class ExecutableStubMongoProcessInterface : public StubMongoProcessInterface { + bool isExpectedToExecuteQueries() override { + return true; + } +}; + TEST_F(DocumentSourceFacetTest, ShouldRejectConflictingHostTypeRequirementsWithinSinglePipeline) { auto ctx = getExpCtx(); ctx->inMongos = true; + ctx->mongoProcessInterface = std::make_unique<ExecutableStubMongoProcessInterface>(); auto spec = fromjson( "{$facet: {badPipe: [{$_internalSplitPipeline: {mergeType: 'anyShard'}}, " @@ -202,6 +212,7 @@ TEST_F(DocumentSourceFacetTest, ShouldRejectConflictingHostTypeRequirementsWithi TEST_F(DocumentSourceFacetTest, ShouldRejectConflictingHostTypeRequirementsAcrossPipelines) { auto ctx = getExpCtx(); ctx->inMongos = true; + ctx->mongoProcessInterface = std::make_unique<ExecutableStubMongoProcessInterface>(); auto spec = fromjson( "{$facet: {shardPipe: [{$_internalSplitPipeline: {mergeType: 'anyShard'}}], mongosPipe: " @@ -904,5 +915,165 @@ TEST_F(DocumentSourceFacetTest, ShouldSurfaceStrictestRequirementsOfEachConstrai ASSERT_FALSE( facetStage->constraints(Pipeline::SplitState::kUnsplit).isAllowedInLookupPipeline()); } + +TEST_F(DocumentSourceFacetTest, RedactsCorrectly) { + auto spec = fromjson(R"({ + $facet: { + a: [ + { $unwind: "$foo" }, + { $sortByCount: "$foo" } + ], + b: [ + { + $match: { + bar: { $exists: 1 } + } + }, + { + $bucket: { + groupBy: "$bar.foo", + boundaries: [0, 50, 100, 200], + output: { + z: { $sum : 1 } + } + } + } + ], + c: [ + { + $bucketAuto: { + groupBy: "$bar.baz", + buckets: 4 + } + } + ] + } + })"); + auto docSource = DocumentSourceFacet::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$facet": { + "HASH<a>": [ + { + "$unwind": { + "path": "$HASH<foo>" + } + }, + { + "$group": { + "_id": "$HASH<foo>", + "HASH<count>": { + "$sum": "?number" + } + } + }, + { + "$sort": { + "HASH<count>": -1 + } + } + ], + "HASH<b>": [ + { + "$match": { + "HASH<bar>": { + "$exists": "?bool" + } + } + }, + { + "$group": { + "_id": { + "$switch": { + "branches": [ + { + "case": { + "$and": [ + { + "$gte": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + }, + { + "$lt": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + } + ] + }, + "then": "?number" + }, + { + "case": { + "$and": [ + { + "$gte": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + }, + { + "$lt": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + } + ] + }, + "then": "?number" + }, + { + "case": { + "$and": [ + { + "$gte": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + }, + { + "$lt": [ + "$HASH<bar>.HASH<foo>", + "?number" + ] + } + ] + }, + "then": "?number" + } + ] + } + }, + "HASH<z>": { + "$sum": "?number" + } + } + }, + { + "$sort": { + "HASH<_id>": 1 + } + } + ], + "HASH<c>": [ + { + "$bucketAuto": { + "groupBy": "$HASH<bar>.HASH<baz>", + "buckets": "?number", + "output": { + "HASH<count>": { + "$sum": "?number" + } + } + } + } + ] + } + })", + redact(*docSource)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp b/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp index 13229ba6d3e..d98aed38f9b 100644 --- a/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.cpp @@ -177,12 +177,12 @@ StageConstraints DocumentSourceFindAndModifyImageLookup::constraints( ChangeStreamRequirement::kDenylist); } -Value DocumentSourceFindAndModifyImageLookup::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceFindAndModifyImageLookup::serialize(const SerializationOptions& opts) const { return Value( Document{{kStageName, Value(Document{{kIncludeCommitTransactionTimestampFieldName, - _includeCommitTransactionTimestamp ? Value(true) : Value()}})}}); + _includeCommitTransactionTimestamp ? opts.serializeLiteral(true) + : Value()}})}}); } DepsTracker::State DocumentSourceFindAndModifyImageLookup::getDependencies( diff --git a/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.h b/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.h index 9fac2c429c4..9e554afcc2e 100644 --- a/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.h +++ b/src/mongo/db/pipeline/document_source_find_and_modify_image_lookup.h @@ -60,7 +60,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/pipeline/document_source_geo_near.cpp b/src/mongo/db/pipeline/document_source_geo_near.cpp index 1ee840df794..dc4aa5f8be6 100644 --- a/src/mongo/db/pipeline/document_source_geo_near.cpp +++ b/src/mongo/db/pipeline/document_source_geo_near.cpp @@ -57,40 +57,45 @@ REGISTER_DOCUMENT_SOURCE(geoNear, DocumentSourceGeoNear::createFromBson, AllowedWithApiStrict::kAlways); -Value DocumentSourceGeoNear::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceGeoNear::serialize(const SerializationOptions& opts) const { MutableDocument result; if (keyFieldPath) { - result.setField(kKeyFieldName, Value(keyFieldPath->fullPath())); + result.setField(kKeyFieldName, Value(opts.serializeFieldPath(*keyFieldPath))); } auto nearValue = [&]() -> Value { if (auto constGeometry = dynamic_cast<ExpressionConstant*>(_nearGeometry.get()); constGeometry) { - return constGeometry->getValue(); + return opts.serializeLiteral(constGeometry->getValue()); } else { - return _nearGeometry->serialize(static_cast<bool>(explain)); + return _nearGeometry->serialize(opts); } }(); result.setField("near", nearValue); - result.setField("distanceField", Value(distanceField->fullPath())); + result.setField("distanceField", Value(opts.serializeFieldPath(*distanceField))); if (maxDistance) { - result.setField("maxDistance", Value(*maxDistance)); + result.setField("maxDistance", opts.serializeLiteral(*maxDistance)); } if (minDistance) { - result.setField("minDistance", Value(*minDistance)); + result.setField("minDistance", opts.serializeLiteral(*minDistance)); } - result.setField("query", Value(query)); - result.setField("spherical", Value(spherical)); + if (opts.transformIdentifiers || opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + auto matchExpr = uassertStatusOK(MatchExpressionParser::parse(query, pExpCtx)); + result.setField("query", Value(matchExpr->serialize(opts))); + } else { + result.setField("query", Value(query)); + } + result.setField("spherical", opts.serializeLiteral(spherical)); if (distanceMultiplier) { - result.setField("distanceMultiplier", Value(*distanceMultiplier)); + result.setField("distanceMultiplier", opts.serializeLiteral(*distanceMultiplier)); } if (includeLocs) - result.setField("includeLocs", Value(includeLocs->fullPath())); + result.setField("includeLocs", Value(opts.serializeFieldPath(*includeLocs))); return Value(DOC(getSourceName() << result.freeze())); } diff --git a/src/mongo/db/pipeline/document_source_geo_near.h b/src/mongo/db/pipeline/document_source_geo_near.h index 2b012285560..e9fd04e5f2d 100644 --- a/src/mongo/db/pipeline/document_source_geo_near.h +++ b/src/mongo/db/pipeline/document_source_geo_near.h @@ -79,7 +79,7 @@ public: MONGO_UNREACHABLE; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<DocumentSource> optimize() final; diff --git a/src/mongo/db/pipeline/document_source_geo_near_test.cpp b/src/mongo/db/pipeline/document_source_geo_near_test.cpp index cc3a510b84e..3ef5920c99e 100644 --- a/src/mongo/db/pipeline/document_source_geo_near_test.cpp +++ b/src/mongo/db/pipeline/document_source_geo_near_test.cpp @@ -111,5 +111,98 @@ TEST_F(DocumentSourceGeoNearTest, CanParseAndSerializeKeyField) { {"spherical", false}}}}}}; ASSERT_VALUE_EQ(expectedSerialization, serialized[0]); } + +TEST_F(DocumentSourceGeoNearTest, RedactionWithGeoJSONPoint) { + auto spec = fromjson(R"({ + $geoNear: { + distanceField: "a", + maxDistance: 2, + minDistance: 1, + near: { + type: "Point", + coordinates: [ -23.484, 28.3913 ] + }, + query: { foo : "bar" }, + spherical: true + } + })"); + auto docSource = DocumentSourceGeoNear::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$geoNear": { + "near": "?object", + "distanceField": "HASH<a>", + "maxDistance": "?number", + "minDistance": "?number", + "query": { + "HASH<foo>": { + "$eq": "?string" + } + }, + "spherical": "?bool" + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGeoNearTest, RedactionWithGeoJSONLineString) { + auto spec = fromjson(R"({ + $geoNear: { + distanceField: "a", + near: { + type: "LineString", + coordinates: [[0,0], [-1,-1]] + }, + minDistance: 0.5 + } + })"); + auto docSource = DocumentSourceGeoNear::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$geoNear": { + "near": "?object", + "distanceField": "HASH<a>", + "minDistance": "?number", + "query": {}, + "spherical": "?bool" + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGeoNearTest, RedactionWithLegacyCoordinates) { + auto spec = fromjson(R"({ + $geoNear: { + distanceField: "foo", + distanceMultiplier: 3.14, + includeLocs: "bar.baz", + near: [10, 10], + key: "z", + query: { + a : { $gt: 10 } + }, + spherical: false + } + })"); + auto docSource = DocumentSourceGeoNear::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$geoNear": { + "key": "HASH<z>", + "near": "?array<?number>", + "distanceField": "HASH<foo>", + "query": { + "HASH<a>": { + "$gt": "?number" + } + }, + "spherical": "?bool", + "distanceMultiplier": "?number", + "includeLocs": "HASH<bar>.HASH<baz>" + } + })", + redact(*docSource)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_graph_lookup.cpp b/src/mongo/db/pipeline/document_source_graph_lookup.cpp index 044875f737f..62a35123173 100644 --- a/src/mongo/db/pipeline/document_source_graph_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_graph_lookup.cpp @@ -544,46 +544,56 @@ void DocumentSourceGraphLookUp::checkMemoryUsage() { _cache.evictDownTo(_maxMemoryUsageBytes - _frontierUsageBytes - _visitedUsageBytes); } -void DocumentSourceGraphLookUp::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { +void DocumentSourceGraphLookUp::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { auto fromValue = (pExpCtx->ns.db() == _from.db()) - ? Value(_from.coll()) - : Value(Document{{"db", _from.db()}, {"coll", _from.coll()}}); + ? Value(opts.serializeIdentifier(_from.coll())) + : Value(Document{{"db", opts.serializeIdentifier(_from.db())}, + {"coll", opts.serializeIdentifier(_from.coll())}}); // Serialize default options. - MutableDocument spec(DOC("from" << fromValue << "as" << _as.fullPath() << "connectToField" - << _connectToField.fullPath() << "connectFromField" - << _connectFromField.fullPath() << "startWith" - << _startWith->serialize(false))); + MutableDocument spec(DOC("from" << fromValue << "as" << opts.serializeFieldPath(_as) + << "connectToField" << opts.serializeFieldPath(_connectToField) + << "connectFromField" + << opts.serializeFieldPath(_connectFromField) << "startWith" + << _startWith->serialize(opts))); // depthField is optional; serialize it if it was specified. if (_depthField) { - spec["depthField"] = Value(_depthField->fullPath()); + spec["depthField"] = Value(opts.serializeFieldPath(*_depthField)); } if (_maxDepth) { - spec["maxDepth"] = Value(*_maxDepth); + spec["maxDepth"] = Value(opts.serializeLiteral(*_maxDepth)); } if (_additionalFilter) { - spec["restrictSearchWithMatch"] = Value(*_additionalFilter); + if (opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + auto matchExpr = + uassertStatusOK(MatchExpressionParser::parse(*_additionalFilter, pExpCtx)); + spec["restrictSearchWithMatch"] = Value(matchExpr->serialize(opts)); + } else { + spec["restrictSearchWithMatch"] = Value(*_additionalFilter); + } } // If we are explaining, include an absorbed $unwind inside the $graphLookup specification. - if (_unwind && explain) { + if (_unwind && opts.verbosity) { const boost::optional<FieldPath> indexPath = (*_unwind)->indexPath(); spec["unwinding"] = Value(DOC("preserveNullAndEmptyArrays" - << (*_unwind)->preserveNullAndEmptyArrays() << "includeArrayIndex" - << (indexPath ? Value((*indexPath).fullPath()) : Value()))); + << opts.serializeLiteral((*_unwind)->preserveNullAndEmptyArrays()) + << "includeArrayIndex" + << (indexPath ? Value(opts.serializeFieldPath(*indexPath)) : Value()))); } array.push_back(Value(DOC(getSourceName() << spec.freeze()))); // If we are not explaining, the output of this method must be parseable, so serialize our // $unwind into a separate stage. - if (_unwind && !explain) { - (*_unwind)->serializeToArray(array); + if (_unwind && !opts.verbosity) { + (*_unwind)->serializeToArray(array, opts); } } diff --git a/src/mongo/db/pipeline/document_source_graph_lookup.h b/src/mongo/db/pipeline/document_source_graph_lookup.h index a2b6ef6a952..52134b61efe 100644 --- a/src/mongo/db/pipeline/document_source_graph_lookup.h +++ b/src/mongo/db/pipeline/document_source_graph_lookup.h @@ -97,7 +97,7 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Returns the 'as' path, and possibly the fields modified by an absorbed $unwind. @@ -188,9 +188,10 @@ private: boost::optional<long long> maxDepth, boost::optional<boost::intrusive_ptr<DocumentSourceUnwind>> unwindSrc); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { // Should not be called; use serializeToArray instead. - MONGO_UNREACHABLE; + MONGO_UNREACHABLE_TASSERT(7484306); } /** diff --git a/src/mongo/db/pipeline/document_source_graph_lookup_test.cpp b/src/mongo/db/pipeline/document_source_graph_lookup_test.cpp index 26345aa16ab..c5dd404a04e 100644 --- a/src/mongo/db/pipeline/document_source_graph_lookup_test.cpp +++ b/src/mongo/db/pipeline/document_source_graph_lookup_test.cpp @@ -713,5 +713,126 @@ TEST_F(DocumentSourceGraphLookUpTest, ShouldNotExpandArraysWithinArraysAtEndOfCo ASSERT(graphLookupStage->getNext().isEOF()); } +TEST_F(DocumentSourceGraphLookUpTest, RedactionStartWithSingleField) { + NamespaceString graphLookupNs(getExpCtx()->ns.db(), "coll"); + getExpCtx()->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {graphLookupNs.coll().toString(), {graphLookupNs, std::vector<BSONObj>()}}}); + + auto spec = fromjson(R"({ + "$graphLookup": { + "from": "coll", + "startWith": "$a.b", + "connectFromField": "c.d", + "connectToField": "e.f", + "as": "x", + "depthField": "y", + "maxDepth": 5, + "restrictSearchWithMatch": { + "foo": "abc", + "bar.baz": { "$gt": 5 } + } + } + })"); + auto docSource = DocumentSourceGraphLookUp::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$graphLookup": { + "from": "HASH<coll>", + "as": "HASH<x>", + "connectToField": "HASH<e>.HASH<f>", + "connectFromField": "HASH<c>.HASH<d>", + "startWith": "$HASH<a>.HASH<b>", + "depthField": "HASH<y>", + "maxDepth": "?number", + "restrictSearchWithMatch": { + "$and": [ + { + "HASH<foo>": { + "$eq": "?string" + } + }, + { + "HASH<bar>.HASH<baz>": { + "$gt": "?number" + } + } + ] + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGraphLookUpTest, RedactionStartWithArrayOfFields) { + NamespaceString graphLookupNs(getExpCtx()->ns.db(), "coll"); + getExpCtx()->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {graphLookupNs.coll().toString(), {graphLookupNs, std::vector<BSONObj>()}}}); + + auto spec = fromjson(R"({ + $graphLookup: { + from: "coll", + startWith: ["$a.b", "$bar.baz"], + connectFromField: "x", + connectToField: "y", + as: "z" + } + })"); + auto docSource = DocumentSourceGraphLookUp::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$graphLookup": { + "from": "HASH<coll>", + "as": "HASH<z>", + "connectToField": "HASH<y>", + "connectFromField": "HASH<x>", + "startWith": ["$HASH<a>.HASH<b>", "$HASH<bar>.HASH<baz>"] + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGraphLookUpTest, RedactionWithAbsorbedUnwind) { + auto expCtx = getExpCtx(); + + NamespaceString graphLookupNs(getExpCtx()->ns.db(), "coll"); + expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {graphLookupNs.coll().toString(), {graphLookupNs, std::vector<BSONObj>()}}}); + + auto unwindStage = DocumentSourceUnwind::create(expCtx, "results", false, boost::none); + auto graphLookupStage = DocumentSourceGraphLookUp::create( + getExpCtx(), + graphLookupNs, + "results", + "from", + "to", + ExpressionFieldPath::deprecatedCreate(expCtx.get(), "startPoint"), + boost::none, + boost::none, + boost::none, + unwindStage); + + auto serialized = redactToArray(*graphLookupStage); + ASSERT_EQ(2, serialized.size()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$graphLookup": { + "from": "HASH<coll>", + "as": "HASH<results>", + "connectToField": "HASH<to>", + "connectFromField": "HASH<from>", + "startWith": "$HASH<startPoint>" + } + })", + serialized[0].getDocument().toBson()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$unwind": { + path: "$HASH<results>" + } + })", + serialized[1].getDocument().toBson()); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_group_base.cpp b/src/mongo/db/pipeline/document_source_group_base.cpp index 0268cd57127..2c4e8c2dd3b 100644 --- a/src/mongo/db/pipeline/document_source_group_base.cpp +++ b/src/mongo/db/pipeline/document_source_group_base.cpp @@ -69,19 +69,20 @@ using std::pair; using std::shared_ptr; using std::vector; -Value DocumentSourceGroupBase::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceGroupBase::serialize(const SerializationOptions& opts) const { MutableDocument insides; // Add the _id. if (_idFieldNames.empty()) { invariant(_idExpressions.size() == 1); - insides["_id"] = _idExpressions[0]->serialize(static_cast<bool>(explain)); + insides["_id"] = _idExpressions[0]->serialize(opts); } else { // Decomposed document case. invariant(_idExpressions.size() == _idFieldNames.size()); MutableDocument md; for (size_t i = 0; i < _idExpressions.size(); i++) { - md[_idFieldNames[i]] = _idExpressions[i]->serialize(static_cast<bool>(explain)); + md[opts.serializeFieldPathFromString(_idFieldNames[i])] = + _idExpressions[i]->serialize(opts); } insides["_id"] = md.freezeToValue(); } @@ -89,34 +90,34 @@ Value DocumentSourceGroupBase::serialize(boost::optional<ExplainOptions::Verbosi // Add the remaining fields. for (auto&& accumulatedField : _accumulatedFields) { intrusive_ptr<AccumulatorState> accum = accumulatedField.makeAccumulator(); - insides[accumulatedField.fieldName] = - Value(accum->serialize(accumulatedField.expr.initializer, - accumulatedField.expr.argument, - static_cast<bool>(explain))); + insides[opts.serializeFieldPathFromString(accumulatedField.fieldName)] = + Value(accum->serialize( + accumulatedField.expr.initializer, accumulatedField.expr.argument, opts)); } if (_doingMerge) { - insides["$doingMerge"] = Value(true); + insides["$doingMerge"] = opts.serializeLiteral(true); } - serializeAdditionalFields(insides, explain); + serializeAdditionalFields(insides, opts); MutableDocument out; out[getSourceName()] = insides.freezeToValue(); - if (explain && *explain >= ExplainOptions::Verbosity::kExecStats) { + if (opts.verbosity && *opts.verbosity >= ExplainOptions::Verbosity::kExecStats) { MutableDocument md; for (size_t i = 0; i < _accumulatedFields.size(); i++) { - md[_accumulatedFields[i].fieldName] = Value(static_cast<long long>( - _memoryTracker[_accumulatedFields[i].fieldName].maxMemoryBytes())); + md[opts.serializeFieldPathFromString(_accumulatedFields[i].fieldName)] = + opts.serializeLiteral(static_cast<long long>( + _memoryTracker[_accumulatedFields[i].fieldName].maxMemoryBytes())); } out["maxAccumulatorMemoryUsageBytes"] = Value(md.freezeToValue()); out["totalOutputDataSizeBytes"] = - Value(static_cast<long long>(_stats.totalOutputDataSizeBytes)); - out["usedDisk"] = Value(_stats.spills > 0); - out["spills"] = Value(static_cast<long long>(_stats.spills)); + opts.serializeLiteral(static_cast<long long>(_stats.totalOutputDataSizeBytes)); + out["usedDisk"] = opts.serializeLiteral(_stats.spills > 0); + out["spills"] = opts.serializeLiteral(static_cast<long long>(_stats.spills)); } return out.freezeToValue(); diff --git a/src/mongo/db/pipeline/document_source_group_base.h b/src/mongo/db/pipeline/document_source_group_base.h index 3c20f1bd420..c7a296c2583 100644 --- a/src/mongo/db/pipeline/document_source_group_base.h +++ b/src/mongo/db/pipeline/document_source_group_base.h @@ -54,7 +54,7 @@ public: using Accumulators = std::vector<boost::intrusive_ptr<AccumulatorState>>; using GroupsMap = ValueUnorderedMap<Accumulators>; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<DocumentSource> optimize() final; DepsTracker::State getDependencies(DepsTracker* deps) const final; GetModPathsReturn getModifiedPaths() const final; @@ -200,7 +200,7 @@ protected: } virtual void serializeAdditionalFields( - MutableDocument& out, boost::optional<ExplainOptions::Verbosity> explain) const {}; + MutableDocument& out, const SerializationOptions& opts = SerializationOptions{}) const {}; // If the expression for the '_id' field represents a non-empty object, we track its fields' // names in '_idFieldNames'. diff --git a/src/mongo/db/pipeline/document_source_group_test.cpp b/src/mongo/db/pipeline/document_source_group_test.cpp index 9a621cffdcd..f3bb0863adf 100644 --- a/src/mongo/db/pipeline/document_source_group_test.cpp +++ b/src/mongo/db/pipeline/document_source_group_test.cpp @@ -246,6 +246,103 @@ TEST_F(DocumentSourceGroupTest, ShouldNotReportDottedGroupKeyAsARename) { ASSERT_EQ(modifiedPathsRet.renames.size(), 0UL); } +TEST_F(DocumentSourceGroupTest, GroupRedactsCorrectWithIdNull) { + auto spec = fromjson(R"({ + $group: { + _id: null, + foo: { $count: {} } + } + })"); + auto docSource = DocumentSourceGroup::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$group":{"_id":"?null","HASH<foo>":{"$sum":"?number"}}})", + redact(*docSource)); +} + +TEST_F(DocumentSourceGroupTest, GroupRedactsCorrectWithIdSingleField) { + auto spec = fromjson(R"({ + $group: { + _id: '$foo' + } + })"); + auto docSource = DocumentSourceGroup::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$group": { + "_id": "$HASH<foo>" + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGroupTest, GroupRedactsCorrectWithIdDocument) { + auto spec = fromjson(R"({ + $group: { + _id: { + x: '$x', + y: '$z' + }, + foo: { + $sum: { + $multiply: ['$a.b', '$c', '$d'] + } + }, + bar: { + $first: '$baz' + } + } + })"); + auto docSource = DocumentSourceGroup::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$group": { + "_id": { + "HASH<x>": "$HASH<x>", + "HASH<y>": "$HASH<z>" + }, + "HASH<foo>": { + "$sum": { + "$multiply": ["$HASH<a>.HASH<b>", "$HASH<c>", "$HASH<d>"] + } + }, + "HASH<bar>": { + "$first": "$HASH<baz>" + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceGroupTest, StreamingGroupRedactsCorrectly) { + auto spec = fromjson(R"({ + $_internalStreamingGroup: { + _id: { + a: "$a", + b: "$b" + }, + a: { + $first: '$b' + }, + $monotonicIdFields: [ "a", "b" ] + } + })"); + auto docSource = DocumentSourceStreamingGroup::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalStreamingGroup": { + "_id": { + "HASH<a>": "$HASH<a>", + "HASH<b>": "$HASH<b>" + }, + "HASH<a>": { + "$first": "$HASH<b>" + }, + "$monotonicIdFields": [ "HASH<a>", "HASH<b>" ] + } + })", + redact(*docSource)); +} + BSONObj toBson(const intrusive_ptr<DocumentSource>& source) { vector<Value> arr; source->serializeToArray(arr); diff --git a/src/mongo/db/pipeline/document_source_index_stats.cpp b/src/mongo/db/pipeline/document_source_index_stats.cpp index 139fdde3bca..bd4ee6bf760 100644 --- a/src/mongo/db/pipeline/document_source_index_stats.cpp +++ b/src/mongo/db/pipeline/document_source_index_stats.cpp @@ -78,8 +78,7 @@ intrusive_ptr<DocumentSource> DocumentSourceIndexStats::createFromBson( return new DocumentSourceIndexStats(pExpCtx); } -Value DocumentSourceIndexStats::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceIndexStats::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_index_stats.h b/src/mongo/db/pipeline/document_source_index_stats.h index bd5b7577365..f56e5e22059 100644 --- a/src/mongo/db/pipeline/document_source_index_stats.h +++ b/src/mongo/db/pipeline/document_source_index_stats.h @@ -75,7 +75,7 @@ public: // virtuals from DocumentSource const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { StageConstraints constraints(StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.cpp b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.cpp index 6c9693e0443..e24bf112bcf 100644 --- a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.cpp +++ b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.cpp @@ -118,17 +118,18 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalAllCollectionStats::do } void DocumentSourceInternalAllCollectionStats::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { + std::vector<Value>& array, const SerializationOptions& opts) const { + auto explain = opts.verbosity; if (explain) { BSONObjBuilder bob; - _internalAllCollectionStatsSpec.serialize(&bob); + _internalAllCollectionStatsSpec.serialize(&bob, opts); if (_absorbedMatch) { bob.append("match", _absorbedMatch->getQuery()); } auto doc = Document{{getSourceName(), bob.obj()}}; array.push_back(Value(doc)); } else { - array.push_back(serialize(explain)); + array.push_back(serialize(opts)); if (_absorbedMatch) { _absorbedMatch->serializeToArray(array); } @@ -156,8 +157,7 @@ const char* DocumentSourceInternalAllCollectionStats::getSourceName() const { return kStageNameInternal.rawData(); } -Value DocumentSourceInternalAllCollectionStats::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{getSourceName(), _internalAllCollectionStatsSpec.toBSON()}}); +Value DocumentSourceInternalAllCollectionStats::serialize(const SerializationOptions& opts) const { + return Value(Document{{getSourceName(), _internalAllCollectionStatsSpec.toBSON(opts)}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.h b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.h index 38324fb51ad..1648939ddd1 100644 --- a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.h +++ b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.h @@ -81,7 +81,7 @@ public: const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { StageConstraints constraints(StreamType::kStreaming, @@ -110,7 +110,7 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; private: GetNextResult doGetNext() final; diff --git a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.idl b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.idl index 61fc5d02f03..c473a5e5ba3 100644 --- a/src/mongo/db/pipeline/document_source_internal_all_collection_stats.idl +++ b/src/mongo/db/pipeline/document_source_internal_all_collection_stats.idl @@ -40,8 +40,10 @@ structs: DocumentSourceInternalAllCollectionStatsSpec: description: Specification for an $_internalAllCollectionStats stage. strict: true + query_shape_component: true fields: stats: description: Specification for a $collStats stage. type: DocumentSourceCollStatsSpec optional: true + query_shape: literal diff --git a/src/mongo/db/pipeline/document_source_internal_all_collection_stats_test.cpp b/src/mongo/db/pipeline/document_source_internal_all_collection_stats_test.cpp new file mode 100644 index 00000000000..e303febfacb --- /dev/null +++ b/src/mongo/db/pipeline/document_source_internal_all_collection_stats_test.cpp @@ -0,0 +1,143 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/document_source_internal_all_collection_stats.h" +#include "mongo/db/pipeline/document_source_internal_all_collection_stats_gen.h" +#include <boost/cstdint.hpp> +#include <cstdint> + +#include <boost/move/utility_core.hpp> +#include <boost/smart_ptr/intrusive_ptr.hpp> + +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_coll_stats.h" +#include "mongo/db/pipeline/document_source_coll_stats_gen.h" +#include "mongo/db/pipeline/storage_stats_spec_gen.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/intrusive_counter.h" + +namespace mongo { +namespace { +using DocumentSourceInternalAllCollectionStatsTest = AggregationContextFixture; + +auto representativeShape(const DocumentSourceInternalAllCollectionStats& allCollStatsStage) { + SerializationOptions opts{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + return allCollStatsStage.serialize(opts).getDocument().toBson(); +} + +TEST_F(DocumentSourceInternalAllCollectionStatsTest, QueryShape) { + auto innerSpec = DocumentSourceCollStatsSpec(); + auto spec = DocumentSourceInternalAllCollectionStatsSpec(); + spec.setStats(innerSpec); + + auto stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalAllCollectionStats":{"stats":{}}})", + redact(*stage)); + + innerSpec.setCount(BSONObj()); + innerSpec.setQueryExecStats(BSONObj()); + spec.setStats(innerSpec); + stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalAllCollectionStats":{"stats":{"count":{},"queryExecStats":{}}}})", + redact(*stage)); + + auto latencyStats = LatencyStatsSpec(); + latencyStats.setHistograms(true); + innerSpec.setLatencyStats(latencyStats); + spec.setStats(innerSpec); + stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalAllCollectionStats": { + "stats": { + "latencyStats": { + "histograms": true + }, + "count": {}, + "queryExecStats": {} + } + } + })", + redact(*stage)); + + auto storageStats = StorageStatsSpec(); + storageStats.setScale(2); + storageStats.setVerbose(true); + innerSpec.setStorageStats(storageStats); + spec.setStats(innerSpec); + stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalAllCollectionStats": { + "stats": { + "latencyStats": { + "histograms": true + }, + "storageStats": { + "scale": "?number", + "verbose": true, + "waitForLock": true, + "numericOnly": false + }, + "count": {}, + "queryExecStats": {} + } + } + })", + redact(*stage)); + + storageStats.setWaitForLock(false); + storageStats.setNumericOnly(false); + innerSpec.setStorageStats(storageStats); + spec.setStats(innerSpec); + stage = make_intrusive<DocumentSourceInternalAllCollectionStats>(getExpCtx(), spec); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalAllCollectionStats": { + "stats": { + "latencyStats": { + "histograms": true + }, + "storageStats": { + "scale": "?number", + "verbose": true, + "waitForLock": false, + "numericOnly": false + }, + "count": {}, + "queryExecStats": {} + } + } + })", + redact(*stage)); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.cpp b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.cpp index 51b0dee8ac1..2d499e0d88c 100644 --- a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.cpp +++ b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.cpp @@ -95,9 +95,9 @@ DocumentSource::GetNextResult DocumentSourceInternalApplyOplogUpdate::doGetNext( return Document(doc.getObject()); } -Value DocumentSourceInternalApplyOplogUpdate::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{kStageName, Document{{kOplogUpdateFieldName, _oplogUpdate}}}}); +Value DocumentSourceInternalApplyOplogUpdate::serialize(const SerializationOptions& opts) const { + return Value(Document{ + {kStageName, Document{{kOplogUpdateFieldName, opts.serializeLiteral(_oplogUpdate)}}}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.h b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.h index ada146b79ea..89860c90efd 100644 --- a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.h +++ b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update.h @@ -81,8 +81,7 @@ public: } private: - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; GetNextResult doGetNext() override; diff --git a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update_test.cpp b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update_test.cpp index fc95d45e6fa..47123e55af2 100644 --- a/src/mongo/db/pipeline/document_source_internal_apply_oplog_update_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_apply_oplog_update_test.cpp @@ -217,6 +217,7 @@ TEST_F(DocumentSourceInternalApplyOplogUpdateTest, ShouldErrorOnInvalidDiffs) { ASSERT_THROWS_CODE(stage->getNext(), DBException, 4770507); } +#if 0 // TODO(SERVER-95699): Disabling test which creates fieldnames with embedded NULs { auto spec = BSON("$_internalApplyOplogUpdate" << BSON("oplogUpdate" << BSON("$v" << 2 << "diff" @@ -238,6 +239,27 @@ TEST_F(DocumentSourceInternalApplyOplogUpdateTest, ShouldErrorOnInvalidDiffs) { stage->setSource(mock.get()); ASSERT_THROWS_CODE(stage->getNext(), DBException, 4770505); } +#endif +} + +TEST_F(DocumentSourceInternalApplyOplogUpdateTest, RedactsCorrectly) { + auto spec = fromjson(R"({ + $_internalApplyOplogUpdate: { + oplogUpdate: { + $v: 2, + diff: { sa: [] } + } + } + })"); + auto docSource = + DocumentSourceInternalApplyOplogUpdate::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalApplyOplogUpdate": { + "oplogUpdate":"?object" + } + })", + redact(*docSource)); } } // namespace diff --git a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.cpp b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.cpp index f643bcb32f3..b44d2bb52cd 100644 --- a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.cpp +++ b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.cpp @@ -139,15 +139,16 @@ DocumentSource::GetNextResult DocumentSourceInternalGeoNearDistance::doGetNext() return next; } -Value DocumentSourceInternalGeoNearDistance::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalGeoNearDistance::serialize(const SerializationOptions& opts) const { MutableDocument out; - out.setField(DocumentSourceInternalGeoNearDistance::kNearFieldName, Value(_coords)); - out.setField(DocumentSourceInternalGeoNearDistance::kKeyFieldName, Value(_key)); + out.setField(DocumentSourceInternalGeoNearDistance::kNearFieldName, + opts.serializeLiteral(_coords)); + out.setField(DocumentSourceInternalGeoNearDistance::kKeyFieldName, + Value(opts.serializeFieldPathFromString(_key))); out.setField(DocumentSourceInternalGeoNearDistance::kDistanceFieldFieldName, - Value(_distanceField.fullPath())); + Value(opts.serializeFieldPath(_distanceField))); out.setField(DocumentSourceInternalGeoNearDistance::kDistanceMultiplierFieldName, - Value(_distanceMultiplier)); + opts.serializeLiteral(_distanceMultiplier)); return Value(DOC(getSourceName() << out.freeze())); } diff --git a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h index b1a24787224..5dc81ea7401 100644 --- a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h +++ b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h @@ -84,8 +84,7 @@ public: } private: - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; GetNextResult doGetNext() override; diff --git a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance_test.cpp b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance_test.cpp index 687fe4f9e7a..67f7300fd2b 100644 --- a/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_compute_geo_near_distance_test.cpp @@ -32,6 +32,7 @@ #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/exec/document_value/document_value_test_util.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" #include "mongo/db/pipeline/document_source_internal_compute_geo_near_distance.h" #include "mongo/db/pipeline/document_source_mock.h" @@ -145,5 +146,31 @@ TEST_F(DocumentSourceInternalGeoNearDistanceTest, DistanceBetweenTwoMixedPointsS ASSERT_APPROX_EQUAL(doc["dist"].coerceToDouble(), meterToLatDegree, 300); } +TEST_F(DocumentSourceInternalGeoNearDistanceTest, RedactsCorrectly) { + BSONObj computeGeoSpec = fromjson(R"( + { $_internalComputeGeoNearDistance: { + near: { + type: "Point", + coordinates: [0, 1] + }, + key: "loc", + distanceMultiplier: 1, + distanceField: "dist" + }})"); + auto geoDist = DocumentSourceInternalGeoNearDistance::createFromBson( + computeGeoSpec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalComputeGeoNearDistance": { + "near": "?object", + "key": "HASH<loc>", + "distanceField": "HASH<dist>", + "distanceMultiplier": "?number" + } + })", + redact(*geoDist, true)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp index be9784f5346..9d52a0f7124 100644 --- a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp +++ b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.cpp @@ -33,6 +33,7 @@ #include "mongo/bson/bsonobj.h" #include "mongo/db/exec/document_value/document.h" +#include "mongo/db/index/index_descriptor.h" #include "mongo/db/list_indexes_gen.h" #include "mongo/db/pipeline/expression_context.h" #include "mongo/db/pipeline/lite_parsed_document_source.h" @@ -146,11 +147,13 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceInternalConvertBucketIndexSta } Value DocumentSourceInternalConvertBucketIndexStats::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { + const SerializationOptions& opts) const { MutableDocument out; - out.addField(timeseries::kTimeFieldName, Value{_timeseriesOptions.timeField}); + out.addField(timeseries::kTimeFieldName, + Value{opts.serializeFieldPathFromString(_timeseriesOptions.timeField)}); if (_timeseriesOptions.metaField) { - out.addField(timeseries::kMetaFieldName, Value{*_timeseriesOptions.metaField}); + out.addField(timeseries::kMetaFieldName, + Value{opts.serializeFieldPathFromString(*_timeseriesOptions.metaField)}); } return Value(DOC(getSourceName() << out.freeze())); } diff --git a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h index 498518ee951..5e7b4c09959 100644 --- a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h +++ b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h @@ -77,7 +77,7 @@ public: ChangeStreamRequirement::kDenylist}; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::optional<DistributedPlanLogic> distributedPlanLogic() final { return boost::none; diff --git a/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats_test.cpp b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats_test.cpp new file mode 100644 index 00000000000..d0b1bddbe0b --- /dev/null +++ b/src/mongo/db/pipeline/document_source_internal_convert_bucket_index_stats_test.cpp @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/exec/document_value/document_value_test_util.h" +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_internal_convert_bucket_index_stats.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/unittest/bson_test_util.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { +using InternalConvertBucketIndexStatsTest = AggregationContextFixture; + +TEST_F(InternalConvertBucketIndexStatsTest, QueryShapeAndRedaction) { + auto expCtx = make_intrusive<ExpressionContextForTest>(); + + auto stage = std::make_unique<DocumentSourceInternalConvertBucketIndexStats>( + expCtx, TimeseriesConversionOptions{"timefield"}); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$_internalConvertBucketIndexStats":{"timeField":"HASH<timefield>"}})", + redact(*stage)); + + std::string metaField = "metafield"; + stage = std::make_unique<DocumentSourceInternalConvertBucketIndexStats>( + expCtx, TimeseriesConversionOptions{"timefield", metaField}); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalConvertBucketIndexStats": { + "timeField": "HASH<timefield>", + "metaField": "HASH<metafield>" + } + })", + redact(*stage)); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.cpp b/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.cpp index af9d4c297b1..6cd368fde1a 100644 --- a/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.cpp +++ b/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.cpp @@ -60,8 +60,7 @@ DocumentSource::GetNextResult DocumentSourceInternalInhibitOptimization::doGetNe return pSource->getNext(); } -Value DocumentSourceInternalInhibitOptimization::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalInhibitOptimization::serialize(const SerializationOptions& opts) const { return Value(Document{{getSourceName(), Value{Document{}}}}); } diff --git a/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.h b/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.h index 79e39f865d9..80d74219af0 100644 --- a/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.h +++ b/src/mongo/db/pipeline/document_source_internal_inhibit_optimization.h @@ -71,7 +71,7 @@ public: private: GetNextResult doGetNext() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_shard_filter.cpp b/src/mongo/db/pipeline/document_source_internal_shard_filter.cpp index a8b112ae834..f63125ff9b6 100644 --- a/src/mongo/db/pipeline/document_source_internal_shard_filter.cpp +++ b/src/mongo/db/pipeline/document_source_internal_shard_filter.cpp @@ -95,8 +95,7 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalShardFilter::doOptimiz return ret; } -Value DocumentSourceInternalShardFilter::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalShardFilter::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } diff --git a/src/mongo/db/pipeline/document_source_internal_shard_filter.h b/src/mongo/db/pipeline/document_source_internal_shard_filter.h index e223af734a9..a8b88869321 100644 --- a/src/mongo/db/pipeline/document_source_internal_shard_filter.h +++ b/src/mongo/db/pipeline/document_source_internal_shard_filter.h @@ -64,8 +64,8 @@ public: ChangeStreamRequirement::kDenylist); } - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::optional<DistributedPlanLogic> distributedPlanLogic() override { return boost::none; diff --git a/src/mongo/db/pipeline/document_source_internal_shardserver_info.cpp b/src/mongo/db/pipeline/document_source_internal_shardserver_info.cpp new file mode 100644 index 00000000000..6e3fe14c578 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_internal_shardserver_info.cpp @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/pipeline/document_source_internal_shardserver_info.h" +#include "mongo/db/pipeline/pipeline.h" + +namespace mongo { + +REGISTER_DOCUMENT_SOURCE(_internalShardServerInfo, + DocumentSourceInternalShardServerInfo::LiteParsed::parse, + DocumentSourceInternalShardServerInfo::createFromBson, + AllowedWithApiStrict::kNeverInVersion1); + +boost::intrusive_ptr<DocumentSource> DocumentSourceInternalShardServerInfo::createFromBson( + BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx) { + uassert(ErrorCodes::TypeMismatch, + str::stream() << "$_internalShardServerInfo must take an empty object but found: " + << elem, + elem.type() == BSONType::Object && elem.Obj().isEmpty()); + + return new DocumentSourceInternalShardServerInfo(expCtx); +} + +DocumentSource::GetNextResult DocumentSourceInternalShardServerInfo::doGetNext() { + if (!_didEmit) { + auto shardName = pExpCtx->mongoProcessInterface->getShardName(pExpCtx->opCtx); + auto hostAndPort = pExpCtx->mongoProcessInterface->getHostAndPort(pExpCtx->opCtx); + _didEmit = true; + return DocumentSource::GetNextResult(DOC("shard" << shardName << "host" << hostAndPort)); + } + + return DocumentSource::GetNextResult::makeEOF(); +} + +Value DocumentSourceInternalShardServerInfo::serialize(const SerializationOptions& opts) const { + return Value(Document{{getSourceName(), Value{Document{{}}}}}); +} + +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_shardserver_info.h b/src/mongo/db/pipeline/document_source_internal_shardserver_info.h new file mode 100644 index 00000000000..5a1cbaa8f63 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_internal_shardserver_info.h @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/pipeline/document_source.h" + +namespace mongo { + +/** + * An internal stage available for testing. Gets the host and shard name for every shard server in + * the cluster. + */ +class DocumentSourceInternalShardServerInfo final : public DocumentSource { +public: + class LiteParsed : public LiteParsedDocumentSource { + public: + static std::unique_ptr<LiteParsed> parse(const NamespaceString& nss, + const BSONElement& spec) { + return std::make_unique<LiteParsed>(spec.fieldName()); + } + + LiteParsed(std::string parseTimeName) + : LiteParsedDocumentSource(std::move(parseTimeName)) {} + + stdx::unordered_set<NamespaceString> getInvolvedNamespaces() const final { + return stdx::unordered_set<NamespaceString>(); + } + + PrivilegeVector requiredPrivileges(bool isMongos, + bool bypassDocumentValidation) const final { + return {}; + } + + bool allowedToPassthroughFromMongos() const final { + return false; + } + }; + + static constexpr StringData kStageName = "$_internalShardServerInfo"_sd; + + static boost::intrusive_ptr<DocumentSource> createFromBson( + BSONElement, const boost::intrusive_ptr<ExpressionContext>&); + + static boost::intrusive_ptr<DocumentSource> create( + const boost::intrusive_ptr<ExpressionContext>& expCtx) { + return new DocumentSourceInternalShardServerInfo(expCtx); + } + + const char* getSourceName() const final { + return kStageName.rawData(); + } + + boost::optional<DistributedPlanLogic> distributedPlanLogic() final { + return boost::none; + } + + StageConstraints constraints(Pipeline::SplitState pipeState) const final { + StageConstraints constraints{StreamType::kStreaming, + PositionRequirement::kFirst, + HostTypeRequirement::kAllShardServers, + DiskUseRequirement::kNoDiskUse, + FacetRequirement::kNotAllowed, + TransactionRequirement::kNotAllowed, + LookupRequirement::kNotAllowed, + UnionRequirement::kAllowed}; + constraints.isIndependentOfAnyCollection = true; + constraints.requiresInputDocSource = false; + return constraints; + } + +private: + DocumentSourceInternalShardServerInfo(const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSource(kStageName, expCtx) {} + + GetNextResult doGetNext() final; + + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + + bool _didEmit = false; +}; + +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_shred_documents.cpp b/src/mongo/db/pipeline/document_source_internal_shred_documents.cpp index 8b51bb2a1ca..3bb636c5004 100644 --- a/src/mongo/db/pipeline/document_source_internal_shred_documents.cpp +++ b/src/mongo/db/pipeline/document_source_internal_shred_documents.cpp @@ -55,8 +55,7 @@ DocumentSource::GetNextResult DocumentSourceInternalShredDocuments::doGetNext() return next; } -Value DocumentSourceInternalShredDocuments::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalShredDocuments::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } diff --git a/src/mongo/db/pipeline/document_source_internal_shred_documents.h b/src/mongo/db/pipeline/document_source_internal_shred_documents.h index 22f1585f557..ebb095d404c 100644 --- a/src/mongo/db/pipeline/document_source_internal_shred_documents.h +++ b/src/mongo/db/pipeline/document_source_internal_shred_documents.h @@ -66,8 +66,7 @@ public: return constraints; } - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::optional<DistributedPlanLogic> distributedPlanLogic() override { return boost::none; diff --git a/src/mongo/db/pipeline/document_source_internal_split_pipeline.cpp b/src/mongo/db/pipeline/document_source_internal_split_pipeline.cpp index 8f5f9927ada..73ad3e7ffd2 100644 --- a/src/mongo/db/pipeline/document_source_internal_split_pipeline.cpp +++ b/src/mongo/db/pipeline/document_source_internal_split_pipeline.cpp @@ -86,8 +86,7 @@ DocumentSource::GetNextResult DocumentSourceInternalSplitPipeline::doGetNext() { return pSource->getNext(); } -Value DocumentSourceInternalSplitPipeline::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalSplitPipeline::serialize(const SerializationOptions& opts) const { std::string mergeTypeString; switch (_mergeType) { diff --git a/src/mongo/db/pipeline/document_source_internal_split_pipeline.h b/src/mongo/db/pipeline/document_source_internal_split_pipeline.h index 967440775d9..ff752d1e22d 100644 --- a/src/mongo/db/pipeline/document_source_internal_split_pipeline.h +++ b/src/mongo/db/pipeline/document_source_internal_split_pipeline.h @@ -85,8 +85,7 @@ private: : DocumentSource(kStageName, expCtx), _mergeType(mergeType) {} GetNextResult doGetNext() final; - - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; HostTypeRequirement _mergeType = HostTypeRequirement::kNone; }; diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp index f352b734e01..d38f5992eaf 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.cpp @@ -232,6 +232,7 @@ boost::intrusive_ptr<DocumentSourceGroup> createBucketGroupForReorder( void optimizePrefix(Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { auto prefix = Pipeline::SourceContainer(container->begin(), itr); Pipeline::optimizeContainer(&prefix); + Pipeline::optimizeEachStage(&prefix); container->erase(container->begin(), itr); container->splice(itr, prefix); } @@ -474,15 +475,17 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceInternalUnpackBucket::createF expCtx, BucketUnpacker{std::move(bucketSpec)}, 3600, assumeClean); } -void DocumentSourceInternalUnpackBucket::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { +void DocumentSourceInternalUnpackBucket::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { + auto explain = opts.verbosity; + MutableDocument out; auto behavior = _bucketUnpacker.behavior() == BucketSpec::Behavior::kInclude ? kInclude : kExclude; const auto& spec = _bucketUnpacker.bucketSpec(); std::vector<Value> fields; for (auto&& field : spec.fieldSet()) { - fields.emplace_back(field); + fields.emplace_back(opts.serializeFieldPathFromString(field)); } if (((_bucketUnpacker.includeMetaField() && _bucketUnpacker.behavior() == BucketSpec::Behavior::kInclude) || @@ -491,23 +494,26 @@ void DocumentSourceInternalUnpackBucket::serializeToArray( std::find(spec.computedMetaProjFields().cbegin(), spec.computedMetaProjFields().cend(), *spec.metaField()) == spec.computedMetaProjFields().cend()) - fields.emplace_back(*spec.metaField()); + fields.emplace_back(opts.serializeFieldPathFromString(*spec.metaField())); out.addField(behavior, Value{std::move(fields)}); - out.addField(timeseries::kTimeFieldName, Value{spec.timeField()}); + out.addField(timeseries::kTimeFieldName, + Value{opts.serializeFieldPathFromString(spec.timeField())}); if (spec.metaField()) { - out.addField(timeseries::kMetaFieldName, Value{*spec.metaField()}); + out.addField(timeseries::kMetaFieldName, + Value{opts.serializeFieldPathFromString(*spec.metaField())}); } - out.addField(kBucketMaxSpanSeconds, Value{_bucketMaxSpanSeconds}); + out.addField(kBucketMaxSpanSeconds, opts.serializeLiteral(Value{_bucketMaxSpanSeconds})); if (_assumeNoMixedSchemaData) - out.addField(kAssumeNoMixedSchemaData, Value(_assumeNoMixedSchemaData)); + out.addField(kAssumeNoMixedSchemaData, + opts.serializeLiteral(Value(_assumeNoMixedSchemaData))); if (spec.usesExtendedRange()) { // Include this flag so that 'explain' is more helpful. // But this is not so useful for communicating from one process to another, // because mongos and/or the primary shard don't know whether any other shard // has extended-range data. - out.addField(kUsesExtendedRange, Value{true}); + out.addField(kUsesExtendedRange, opts.serializeLiteral(Value{true})); } if (!spec.computedMetaProjFields().empty()) @@ -516,34 +522,40 @@ void DocumentSourceInternalUnpackBucket::serializeToArray( std::transform(spec.computedMetaProjFields().cbegin(), spec.computedMetaProjFields().cend(), std::back_inserter(compFields), - [](auto&& projString) { return Value{projString}; }); + [opts](auto&& projString) { + return Value{ + opts.serializeFieldPathFromString(projString)}; + }); return compFields; }()}); if (_bucketUnpacker.includeMinTimeAsMetadata()) { - out.addField(kIncludeMinTimeAsMetadata, Value{_bucketUnpacker.includeMinTimeAsMetadata()}); + out.addField(kIncludeMinTimeAsMetadata, + opts.serializeLiteral(Value{_bucketUnpacker.includeMinTimeAsMetadata()})); } if (_bucketUnpacker.includeMaxTimeAsMetadata()) { - out.addField(kIncludeMaxTimeAsMetadata, Value{_bucketUnpacker.includeMaxTimeAsMetadata()}); + out.addField(kIncludeMaxTimeAsMetadata, + opts.serializeLiteral(Value{_bucketUnpacker.includeMaxTimeAsMetadata()})); } if (_wholeBucketFilter) { - out.addField(kWholeBucketFilter, Value{_wholeBucketFilter->serialize()}); + out.addField(kWholeBucketFilter, Value{_wholeBucketFilter->serialize(opts)}); } if (_eventFilter) { - out.addField(kEventFilter, Value{_eventFilter->serialize()}); + out.addField(kEventFilter, Value{_eventFilter->serialize(opts)}); } if (!explain) { array.push_back(Value(DOC(getSourceName() << out.freeze()))); if (_sampleSize) { auto sampleSrc = DocumentSourceSample::create(pExpCtx, *_sampleSize); - sampleSrc->serializeToArray(array); + sampleSrc->serializeToArray(array, opts); } } else { if (_sampleSize) { - out.addField("sample", Value{static_cast<long long>(*_sampleSize)}); - out.addField("bucketMaxCount", Value{_bucketMaxCount}); + out.addField("sample", + opts.serializeLiteral(Value{static_cast<long long>(*_sampleSize)})); + out.addField("bucketMaxCount", opts.serializeLiteral(Value{_bucketMaxCount})); } array.push_back(Value(DOC(getSourceName() << out.freeze()))); } @@ -1205,6 +1217,34 @@ bool findSequentialDocumentCache(Pipeline::SourceContainer::iterator start, return start != end; } +Pipeline::SourceContainer::iterator DocumentSourceInternalUnpackBucket::optimizeAtRestOfPipeline( + Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { + if (itr == container->end()) { + return itr; + } + + invariant(*itr == this); + Pipeline::SourceContainer::iterator unpackBucket = itr; + + itr = std::next(itr); + + try { + while (itr != container->end()) { + if (itr == unpackBucket) { + itr = std::next(itr); + if (itr == container->end()) + break; + } + itr = (*itr).get()->optimizeAt(itr, container); + } + } catch (DBException& ex) { + ex.addContext("Failed to optimize pipeline"); + throw; + } + + return itr; +} + DepsTracker DocumentSourceInternalUnpackBucket::getRestPipelineDependencies( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container, @@ -1297,31 +1337,31 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalUnpackBucket::doOptimi } } - // Optimize the pipeline after this stage to merge $match stages and push them forward, and to - // take advantage of $expr rewrite optimizations. + // OptimizeAt the pipeline after this stage to merge $match stages and push them forward. if (!_optimizedEndOfPipeline) { _optimizedEndOfPipeline = true; if (std::next(itr) == container->end()) { return container->end(); } - if (auto nextStage = dynamic_cast<DocumentSourceGeoNear*>(std::next(itr)->get())) { - // If the end of the pipeline starts with a $geoNear stage, make sure it gets optimized - // in a context where it knows there are other stages before it. It will split itself - // up into separate $match and $sort stages. But it doesn't split itself up when it's - // the first stage, because it expects to use a special DocumentSouceGeoNearCursor plan. - nextStage->optimizeAt(std::next(itr), container); - } + auto cacheFound = findSequentialDocumentCache(itr, container->end()); if (cacheFound) { - // optimizeAt() is responsible for reordering stages, and optimize() is responsible for - // simplifying individual stages. $sequentialCache's optimizeAt() places the stage where - // it can cache as big a prefix of the pipeline as possible. To do so correctly, it - // needs to look at dependencies: a stage that depends on a let-variable cannot be - // cached. But optimize() can inline variables. Therefore, we want to avoid calling - // optimize() before $sequentialCache has a chance to run optimizeAt(). - return Pipeline::optimizeAtEndOfPipeline(itr, container); + // We want to call optimizeAt() on the rest of the pipeline first, and exit this + // function since any calls to optimize() will interfere with the + // sequentialDocumentCache's ability to properly place itself or abandon. + return DocumentSourceInternalUnpackBucket::optimizeAtRestOfPipeline(itr, container); } else { + if (auto nextStage = dynamic_cast<DocumentSourceGeoNear*>(std::next(itr)->get())) { + // If the end of the pipeline starts with a $geoNear stage, make sure it gets + // optimized in a context where it knows there are other stages before it. It will + // split itself up into separate $match and $sort stages. But it doesn't split + // itself up when it's the first stage, because it expects to use a special + // DocumentSouceGeoNearCursor plan. + nextStage->optimizeAt(std::next(itr), container); + } + // We want to optimize the rest of the pipeline to ensure the stages are in their + // optimal position and expressions have been optimized to allow for certain rewrites. Pipeline::optimizeEndOfPipeline(itr, container); } @@ -1418,9 +1458,8 @@ Pipeline::SourceContainer::iterator DocumentSourceInternalUnpackBucket::doOptimi // Create a loose bucket predicate and push it before the unpacking stage. if (predicates.loosePredicate) { - BSONObjBuilder bob; - predicates.loosePredicate->serialize(&bob); - container->insert(itr, DocumentSourceMatch::create(bob.obj(), pExpCtx)); + container->insert( + itr, DocumentSourceMatch::create(predicates.loosePredicate->serialize(), pExpCtx)); // Give other stages a chance to optimize with the new $match. return std::prev(itr) == container->begin() ? std::prev(itr) diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h index 4d20a22f107..8d28e5e75cd 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket.h @@ -77,13 +77,14 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Use 'serializeToArray' above. */ - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { - MONGO_UNREACHABLE; + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { + MONGO_UNREACHABLE_TASSERT(7484305); } bool includeMetaField() const { @@ -270,6 +271,17 @@ private: bool haveComputedMetaField() const; + /** + * Applies optimizeAt() to all stages in the given pipeline after the stage that 'itr' points + * to, which is the bucket unpack stage. + * + * Due to the manipulation of 'itr' through the optimizations, it may be possible that + * preceeding stages will be optimized. However, optimization of the bucket unpack stage will be + * skipped. + */ + Pipeline::SourceContainer::iterator optimizeAtRestOfPipeline( + Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container); + // If buckets contained a mixed type schema along some path, we have to push down special // predicates in order to ensure correctness. bool _assumeNoMixedSchemaData = false; diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp index 39721256948..d51e8a5a90e 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/create_predicates_on_bucket_level_field_test.cpp @@ -55,7 +55,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.max.a': {$_internalExprGt: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -77,7 +77,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.max.a': {$_internalExprGte: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -99,7 +99,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.min.a': {$_internalExprLt: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -121,7 +121,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.min.a': {$_internalExprLte: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -143,7 +143,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {$and:[{'control.min.a': {$_internalExprLte: 1}}," "{'control.max.a': {$_internalExprGte: 1}}]}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," @@ -190,7 +190,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, " ]}}" " ]}" "]}"); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), expected); + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), expected); ASSERT_FALSE(predicate.tightPredicate); } @@ -214,7 +214,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.max.a': {$_internalExprGt: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -241,7 +241,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {'control.max.a': {$_internalExprGte: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," "{$type: [ \"$control.max.a\" ]} ]}} ]}")); @@ -316,7 +316,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [ {$and:[{'control.min.a': {$_internalExprLte: 1}}," "{'control.max.a': {$_internalExprGte: 1}}]}," "{$expr: {$ne: [ {$type: [ \"$control.min.a\" ]}," @@ -339,7 +339,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$and: [ {$or: [ {'control.max.b': {$_internalExprGt: 1}}," "{$expr: {$ne: [ {$type: [ \"$control.min.b\" ]}," "{$type: [ \"$control.max.b\" ]} ]}} ]}," @@ -383,7 +383,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [" " {'control.max.b': {$_internalExprGt: 1}}," " {$expr: {$ne: [" @@ -410,7 +410,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$and: [ {$or: [ {'control.max.b': {$_internalExprGte: 2}}," "{$expr: {$ne: [ {$type: [ \"$control.min.b\" ]}," "{$type: [ \"$control.max.b\" ]} ]}} ]}," @@ -439,7 +439,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, ->createPredicatesOnBucketLevelField(original->getMatchExpression()); ASSERT(predicate.loosePredicate); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [" " {$or: [" " {'control.max.b': {$_internalExprGt: 1}}," @@ -516,7 +516,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$or: [" " {$or: [" " {'control.max.b': {$_internalExprGte: 2}}," @@ -557,7 +557,8 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, ASSERT_EQ(pipeline->getSources().size(), 2U); // To get the optimized $match from the pipeline, we have to serialize with explain. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps( + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); ASSERT_EQ(stages.size(), 2U); ASSERT_BSONOBJ_EQ(stages[0].getDocument().toBson(), @@ -677,8 +678,8 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, ->createPredicatesOnBucketLevelField(original->getMatchExpression()); // Meta predicates are mapped to the meta field, not the control min/max fields. - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), fromjson("{meta: {$gt: 5}}")); - ASSERT_BSONOBJ_EQ(predicate.tightPredicate->serialize(true), fromjson("{meta: {$gt: 5}}")); + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{meta: {$gt: 5}}")); + ASSERT_BSONOBJ_EQ(predicate.tightPredicate->serialize(), fromjson("{meta: {$gt: 5}}")); } TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, @@ -697,10 +698,8 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, ->createPredicatesOnBucketLevelField(original->getMatchExpression()); // Meta predicates are mapped to the meta field, not the control min/max fields. - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), - fromjson("{'meta.foo': {$gt: 5}}")); - ASSERT_BSONOBJ_EQ(predicate.tightPredicate->serialize(true), - fromjson("{'meta.foo': {$gt: 5}}")); + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{'meta.foo': {$gt: 5}}")); + ASSERT_BSONOBJ_EQ(predicate.tightPredicate->serialize(), fromjson("{'meta.foo': {$gt: 5}}")); } TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, @@ -718,7 +717,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$and: [" " {$or: [" " {'control.max.a': {$_internalExprGt: 1}}," @@ -770,10 +769,10 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 3); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.min.time" << BSON("$_internalExprLt" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.max.time" << BSON("$_internalExprLt" << datePlusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[2].get()); @@ -827,10 +826,10 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 3); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.min.time" << BSON("$_internalExprLte" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.max.time" << BSON("$_internalExprLte" << datePlusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[2].get()); @@ -876,15 +875,15 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 6); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.min.time" << BSON("$_internalExprLte" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.min.time" << BSON("$_internalExprGte" << dateMinusBucketSpan))); - ASSERT_BSONOBJ_EQ((*children)[2]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[2]->serialize(), BSON("control.max.time" << BSON("$_internalExprGte" << date))); ASSERT_BSONOBJ_EQ( - (*children)[3]->serialize(true), + (*children)[3]->serialize(), BSON("control.max.time" << BSON("$_internalExprLte" << datePlusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[4].get()); @@ -938,10 +937,10 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 3); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.max.time" << BSON("$_internalExprGt" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.min.time" << BSON("$_internalExprGt" << dateMinusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[2].get()); @@ -987,10 +986,10 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, OptimizeMapsTimePre auto children = andExpr->getChildVector(); ASSERT_EQ(children->size(), 3); - ASSERT_BSONOBJ_EQ((*children)[0]->serialize(true), + ASSERT_BSONOBJ_EQ((*children)[0]->serialize(), BSON("control.max.time" << BSON("$_internalExprGte" << date))); ASSERT_BSONOBJ_EQ( - (*children)[1]->serialize(true), + (*children)[1]->serialize(), BSON("control.min.time" << BSON("$_internalExprGte" << dateMinusBucketSpan))); auto idPred = dynamic_cast<ComparisonMatchExpressionBase*>((*children)[2].get()); @@ -1138,7 +1137,7 @@ TEST_F(InternalUnpackBucketPredicateMappingOptimizationTest, auto predicate = dynamic_cast<DocumentSourceInternalUnpackBucket*>(container.front().get()) ->createPredicatesOnBucketLevelField(original->getMatchExpression()); - ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(true), + ASSERT_BSONOBJ_EQ(predicate.loosePredicate->serialize(), fromjson("{$_internalBucketGeoWithin: { withinRegion: { $geometry: { type : " "\"Polygon\" ,coordinates: [ [ [ 0, 0 ], [ 3, 6 ], [ 6, 1 ], [ 0, 0 " "] ] ]}},field: \"loc\"}}")); diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/optimize_pipeline_test.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/optimize_pipeline_test.cpp index 0f9a60dbf2a..5b87a64a258 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/optimize_pipeline_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/optimize_pipeline_test.cpp @@ -40,7 +40,8 @@ namespace mongo { namespace { using OptimizePipeline = AggregationContextFixture; - +const auto kExplain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; TEST_F(OptimizePipeline, MixedMatchPushedDown) { auto unpack = fromjson( "{$_internalUnpackBucket: { exclude: [], timeField: 'time', metaField: 'myMeta', " @@ -53,7 +54,7 @@ TEST_F(OptimizePipeline, MixedMatchPushedDown) { pipeline->optimizePipeline(); // To get the optimized $match from the pipeline, we have to serialize with explain. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(2u, stages.size()); // We should push down the $match on the metaField and the predicates on the control field. @@ -104,7 +105,7 @@ TEST_F(OptimizePipeline, MixedMatchOr) { pipeline->optimizePipeline(); - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(2u, stages.size()); auto expected = fromjson( "{$match: {$and: [" @@ -170,7 +171,7 @@ TEST_F(OptimizePipeline, MultipleMatchesPushedDown) { // We should push down both the $match on the metaField and the predicates on the control field. // The created $match stages should be added before $_internalUnpackBucket and merged. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(2u, stages.size()); ASSERT_BSONOBJ_EQ(fromjson("{$match: {$and: [ {meta: {$gte: 0}}," "{meta: {$lte: 5}}," @@ -199,7 +200,7 @@ TEST_F(OptimizePipeline, MultipleMatchesPushedDownWithSort) { // We should push down both the $match on the metaField and the predicates on the control field. // The created $match stages should be added before $_internalUnpackBucket and merged. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(3u, stages.size()); ASSERT_BSONOBJ_EQ(fromjson("{$match: {$and: [ { meta: { $gte: 0 } }," "{meta: { $lte: 5 } }," @@ -343,7 +344,7 @@ TEST_F(OptimizePipeline, MixedMatchThenProjectPushedDown) { pipeline->optimizePipeline(); // We can push down part of the $match and use dependency analysis on the end of the pipeline. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(3u, stages.size()); ASSERT_BSONOBJ_EQ(fromjson("{$match: {$and: [{meta: {$eq: 'abc'}}," "{$or: [ {'control.min.a': { $_internalExprLte: 4 } }," @@ -392,7 +393,7 @@ TEST_F(OptimizePipeline, ProjectThenMixedMatchPushedDown) { pipeline->optimizePipeline(); // We should push down part of the $match and do dependency analysis on the rest. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(3u, stages.size()); ASSERT_BSONOBJ_EQ(fromjson("{$match: {$and: [{meta: {$eq: \"abc\"}}," "{$or: [ {'control.min.a': {$_internalExprLte: 4}}," @@ -423,7 +424,7 @@ TEST_F(OptimizePipeline, ProjectWithRenameThenMixedMatchPushedDown) { pipeline->optimizePipeline(); // We should push down part of the $match and do dependency analysis on the end of the pipeline. - auto stages = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto stages = pipeline->writeExplainOps(kExplain); ASSERT_EQ(3u, stages.size()); ASSERT_BSONOBJ_EQ( fromjson("{$match: {$and: [{$or: [ {'control.max.y': {$_internalExprGte: \"abc\"}}," @@ -1021,5 +1022,52 @@ TEST_F(OptimizePipeline, StreamingGroupIsNotEnabledWhenTimeFieldIsModified) { ASSERT_EQ(serialized.size(), 4U); ASSERT_BSONOBJ_EQ(groupSpecObj, serialized.back()); } + +TEST_F(OptimizePipeline, ComputedMetaProjFieldsAreNotInInclusionProjection) { + auto pipeline = Pipeline::parse( + makeVector( + fromjson( + "{$_internalUnpackBucket: { exclude: [], timeField: 'time', metaField: " + "'myMeta', bucketMaxSpanSeconds: 3600, computedMetaProjFields: ['time', 'y']}}"), + fromjson("{$project: {time: 1, x: 1}}")), + getExpCtx()); + ASSERT_EQ(2u, pipeline->getSources().size()); + + pipeline->optimizePipeline(); + + // The fields in 'computedMetaProjFields' that are not in the project should be removed. + auto serialized = pipeline->serializeToBson(); + ASSERT_EQ(1u, serialized.size()); + ASSERT_BSONOBJ_EQ( + fromjson("{$_internalUnpackBucket: { include: ['_id', 'time', 'x'], timeField: 'time', " + "metaField: " + "'myMeta', bucketMaxSpanSeconds: 3600, computedMetaProjFields: ['time']}}"), + serialized[0]); +} + +TEST_F(OptimizePipeline, ComputedMetaProjectFieldsAfterInclusionGetsAddedToIncludes) { + + auto pipeline = Pipeline::parse( + makeVector(fromjson("{$_internalUnpackBucket: { exclude: [], timeField: 'time', metaField: " + "'myMeta', bucketMaxSpanSeconds: 3600, computedMetaProjFields: []}}"), + fromjson("{$project: {myMeta: 1}}"), + fromjson("{$addFields: {newMeta: {$toUpper : '$myMeta'}}}")), + getExpCtx()); + ASSERT_EQ(3u, pipeline->getSources().size()); + + pipeline->optimizePipeline(); + + auto serialized = pipeline->serializeToBson(); + ASSERT_EQ(2u, serialized.size()); + + ASSERT_BSONOBJ_EQ(fromjson("{$addFields: {newMeta: {$toUpper: ['$meta']}}}"), serialized[0]); + + // 'newMeta' field gets added to 'computedMetaProjFields' and to 'include'. + auto expectedSpecObj = fromjson( + "{$_internalUnpackBucket: { include: ['_id','newMeta', 'myMeta'], timeField: 'time', " + "metaField: 'myMeta', " + "bucketMaxSpanSeconds: 3600, computedMetaProjFields: ['newMeta']}}"); + ASSERT_BSONOBJ_EQ(expectedSpecObj, serialized[1]); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/unpack_bucket_exec_test.cpp b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/unpack_bucket_exec_test.cpp index 06217344dad..3a02d4d0331 100644 --- a/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/unpack_bucket_exec_test.cpp +++ b/src/mongo/db/pipeline/document_source_internal_unpack_bucket_test/unpack_bucket_exec_test.cpp @@ -56,37 +56,38 @@ TEST_F(InternalUnpackBucketExecTest, UnpackBasicIncludeAllMeasurementFields) { // This source will produce two buckets. auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {'m1': 9, 'm2': 9, 'm3': 9}, data: {_id: {'0':3, '1':4}, " - "time: {'0':3, '1':4}, " + "time: {'0':Date(3), '1':Date(4)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); // The first result exists and is as expected. auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -105,36 +106,38 @@ TEST_F(InternalUnpackBucketExecTest, UnpackExcludeASingleField) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}, " + "{'0':Date(3), '1':Date(4)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); // The first result exists and is as expected. auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -152,10 +155,10 @@ TEST_F(InternalUnpackBucketExecTest, UnpackEmptyInclude) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}, " + "{'0':Date(3), '1':Date(4)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); @@ -189,37 +192,38 @@ TEST_F(InternalUnpackBucketExecTest, UnpackEmptyExclude) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}, " + "{'0':Date(3), '1':Date(4)}, " "a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -242,7 +246,7 @@ TEST_F(InternalUnpackBucketExecTest, UnpackNeitherIncludeNorExcludeDefaultsToEmp meta: {'m1': 999, 'm2': 9999}, data: { _id: {'0':1, '1':2}, - time: {'0':1, '1':2}, + time: {'0':Date(1), '1':Date(2)}, a:{'0':1, '1':2}, b:{'1':1} } @@ -252,7 +256,7 @@ TEST_F(InternalUnpackBucketExecTest, UnpackNeitherIncludeNorExcludeDefaultsToEmp meta: {m1: 9, m2: 9, m3: 9}, data: { _id: {'0':3, '1':4}, - time: {'0':3, '1':4}, + time: {'0':Date(3), '1':Date(4)}, a:{'0':1, '1':2}, b:{'1':1} } @@ -262,27 +266,28 @@ TEST_F(InternalUnpackBucketExecTest, UnpackNeitherIncludeNorExcludeDefaultsToEmp auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, a: 2, b: 1}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, a: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -300,20 +305,22 @@ TEST_F(InternalUnpackBucketExecTest, SparseColumnsWhereOneColumnIsExhaustedBefor auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "a:{'0':1}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, b: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -334,37 +341,38 @@ TEST_F(InternalUnpackBucketExecTest, UnpackBasicIncludeWithDollarPrefix) { // This source will produce two buckets. auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}, " + "time: {'0':Date(1), '1':Date(2)}, " "$a:{'0':1, '1':2}, b:{'1':1}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}, " + "{'0':Date(3), '1':Date(4)}, " "$a:{'0':1, '1':2}, b:{'1':1}}}"}, expCtx); unpack->setSource(source.get()); // The first result exists and is as expected. auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1, $a: 1}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1, $a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2, $a: 2, b: 1}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2, $a: 2, b: 1}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, $a: 1}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3, $a: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ( next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, $a: 2, b: 1}"))); + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4, $a: 2, b: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -382,32 +390,34 @@ TEST_F(InternalUnpackBucketExecTest, UnpackMetadataOnly) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'0':1, '1':2}, " - "time: {'0':1, '1':2}}}", + "time: {'0':Date(1), '1':Date(2)}}}", "{control: {'version': 1}, meta: {m1: 9, m2: 9, m3: 9}, data: {_id: {'0':3, '1':4}, time: " - "{'0':3, '1':4}}}"}, + "{'0':Date(3), '1':Date(4)}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}"))); + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(3), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 3}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -425,42 +435,45 @@ TEST_F(InternalUnpackBucketExecTest, UnpackWithStrangeTimestampOrdering) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", "{control: {'version': 1}, meta: {'m1': 9, 'm2': 9, 'm3': 9}, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}"))); + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 999, m2: 9999}, _id: 3}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 999, m2: 9999}, _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 4, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(4), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 5, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 5}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(5), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 6, myMeta: {m1: 9, m2: 9, m3: 9}, _id: 6}"))); + ASSERT_DOCUMENT_EQ( + next.getDocument(), + Document(fromjson("{time: Date(6), myMeta: {m1: 9, m2: 9, m3: 9}, _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -474,39 +487,39 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesMissingMetadataWhenMet << DocumentSourceInternalUnpackBucket::kBucketMaxSpanSeconds << 3600)); auto unpack = DocumentSourceInternalUnpackBucket::createFromBsonInternal(spec.firstElement(), expCtx); - auto source = - DocumentSourceMock::createForTest({"{control: {'version': 1}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", - "{control: {'version': 1}, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, - expCtx); + auto source = DocumentSourceMock::createForTest( + {"{control: {'version': 1}, data: {_id: {'1':1, " + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", + "{control: {'version': 1}, data: {_id: {'1':4, " + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, + expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 1, _id: 1}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(1), _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 2, _id: 2}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(2), _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 3, _id: 3}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(3), _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 4, _id: 4}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(4), _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 5, _id: 5}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(5), _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 6, _id: 6}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(6), _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -524,37 +537,37 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesExcludedMetadataWhenBu DocumentSourceInternalUnpackBucket::createFromBsonInternal(spec.firstElement(), expCtx); auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", "{control: {'version': 1}, meta: {'m1': 9, 'm2': 9, 'm3': 9}, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 1, _id: 1}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(1), _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 2, _id: 2}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(2), _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 3, _id: 3}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(3), _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 4, _id: 4}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(4), _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 5, _id: 5}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(5), _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 6, _id: 6}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(6), _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -572,7 +585,7 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerThrowsOnUndefinedMetadata) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: undefined, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}"}, + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}"}, expCtx); unpack->setSource(source.get()); ASSERT_THROWS_CODE(unpack->getNext(), AssertionException, 5369600); @@ -589,9 +602,9 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerThrowsWhenMetadataIsPresentUn auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", "{control: {'version': 1}, meta: null, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, expCtx); unpack->setSource(source.get()); @@ -610,39 +623,42 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesNullMetadata) { auto source = DocumentSourceMock::createForTest( {"{control: {'version': 1}, meta: {'m1': 999, 'm2': 9999}, data: {_id: {'1':1, " - "'0':2, '2': 3}, time: {'1':1, '0': 2, '2': 3}}}", + "'0':2, '2': 3}, time: {'1':Date(1), '0': Date(2), '2': Date(3)}}}", "{control: {'version': 1}, meta: null, data: {_id: {'1':4, " - "'0':5, '2':6}, time: {'1':4, '0': 5, '2': 6}}}"}, + "'0':5, '2':6}, time: {'1':Date(4), '0': Date(5), '2': Date(6)}}}"}, expCtx); unpack->setSource(source.get()); auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}"))); + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 999, m2: 9999}, _id: 3}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 999, m2: 9999}, _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 4, myMeta: null, _id: 4}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), + Document(fromjson("{time: Date(4), myMeta: null, _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 5, myMeta: null, _id: 5}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), + Document(fromjson("{time: Date(5), myMeta: null, _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 6, myMeta: null, _id: 6}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), + Document(fromjson("{time: Date(6), myMeta: null, _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -668,7 +684,7 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesMissingMetadata) { }, data: { _id: {'1':1, '0':2, '2': 3}, - time: {'1':1, '0': 2, '2': 3} + time: {'1':Date(1), '0': Date(2), '2': Date(3)} } })", R"( @@ -676,7 +692,7 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesMissingMetadata) { control: {'version': 1}, data: { _id: {'1':4, '0':5, '2':6}, - time: {'1':4, '0': 5, '2': 6} + time: {'1':Date(4), '0': Date(5), '2': Date(6)} } })"}, expCtx); @@ -685,30 +701,30 @@ TEST_F(InternalUnpackBucketExecTest, BucketUnpackerHandlesMissingMetadata) { auto next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 1, myMeta: {m1: 999, m2: 9999}, _id: 1}"))); + Document(fromjson("{time: Date(1), myMeta: {m1: 999, m2: 9999}, _id: 1}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 2, myMeta: {m1: 999, m2: 9999}, _id: 2}"))); + Document(fromjson("{time: Date(2), myMeta: {m1: 999, m2: 9999}, _id: 2}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); ASSERT_DOCUMENT_EQ(next.getDocument(), - Document(fromjson("{time: 3, myMeta: {m1: 999, m2: 9999}, _id: 3}"))); + Document(fromjson("{time: Date(3), myMeta: {m1: 999, m2: 9999}, _id: 3}"))); // Second bucket next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 4, _id: 4}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(4), _id: 4}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 5, _id: 5}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(5), _id: 5}"))); next = unpack->getNext(); ASSERT_TRUE(next.isAdvanced()); - ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: 6, _id: 6}"))); + ASSERT_DOCUMENT_EQ(next.getDocument(), Document(fromjson("{time: Date(6), _id: 6}"))); next = unpack->getNext(); ASSERT_TRUE(next.isEOF()); @@ -890,7 +906,13 @@ TEST_F(InternalUnpackBucketExecTest, ParserRoundtripsComputedMetaProjFieldsInclu auto array = std::vector<Value>{}; DocumentSourceInternalUnpackBucket::createFromBsonInternal(bson.firstElement(), getExpCtx()) ->serializeToArray(array); - ASSERT_BSONOBJ_EQ(array[0].getDocument().toBson(), bson); + + // Since fields in 'computedMetaProjFields' are not in 'include' they should be removed. + auto expectedBson = fromjson( + "{$_internalUnpackBucket: {include: [], timeField: 'time', metaField: 'meta', " + "bucketMaxSpanSeconds: 3600}}"); + + ASSERT_BSONOBJ_EQ(array[0].getDocument().toBson(), expectedBson); } TEST_F(InternalUnpackBucketExecTest, ParserRoundtripsComputedMetaProjFieldsIncludeWithCompute) { @@ -925,5 +947,18 @@ TEST_F(InternalUnpackBucketExecTest, ParserRoundtripsComputedMetaProjFieldOverri unpackBucket->serializeToArray(array); ASSERT_BSONOBJ_EQ(array[0].getDocument().toBson(), bson); } + +TEST_F(InternalUnpackBucketExecTest, RedactsCorrectly) { + auto bson = fromjson( + "{$_internalUnpackBucket: {include: ['a', 'b', 'c'], timeField: 'time', metaField: 'meta', " + "bucketMaxSpanSeconds: 3600, computedMetaProjFields: ['a', 'b', 'c']}}"); + auto docSource = DocumentSourceInternalUnpackBucket::createFromBsonInternal(bson.firstElement(), + getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + "{$_internalUnpackBucket: {include: [\"HASH<a>\", \"HASH<b>\", \"HASH<c>\"], timeField: " + "\"HASH<time>\", metaField: \"HASH<meta>\", bucketMaxSpanSeconds: \"?number\", " + "computedMetaProjFields: [\"HASH<a>\", \"HASH<b>\", \"HASH<c>\"]}}", + redact(*docSource)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_limit.cpp b/src/mongo/db/pipeline/document_source_limit.cpp index f50e795897d..2347e185a55 100644 --- a/src/mongo/db/pipeline/document_source_limit.cpp +++ b/src/mongo/db/pipeline/document_source_limit.cpp @@ -88,8 +88,8 @@ DocumentSource::GetNextResult DocumentSourceLimit::doGetNext() { return nextInput; } -Value DocumentSourceLimit::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{getSourceName(), _limit}}); +Value DocumentSourceLimit::serialize(const SerializationOptions& opts) const { + return Value(Document{{getSourceName(), opts.serializeLiteral(_limit)}}); } intrusive_ptr<DocumentSourceLimit> DocumentSourceLimit::create( diff --git a/src/mongo/db/pipeline/document_source_limit.h b/src/mongo/db/pipeline/document_source_limit.h index 6fe5370b0c2..82de8094431 100644 --- a/src/mongo/db/pipeline/document_source_limit.h +++ b/src/mongo/db/pipeline/document_source_limit.h @@ -69,7 +69,7 @@ public: */ Pipeline::SourceContainer::iterator doOptimizeAt(Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DepsTracker::State getDependencies(DepsTracker* deps) const final { return DepsTracker::State::SEE_NEXT; // This doesn't affect needed fields diff --git a/src/mongo/db/pipeline/document_source_limit_test.cpp b/src/mongo/db/pipeline/document_source_limit_test.cpp index 899dbe4de42..5465cf1bdf2 100644 --- a/src/mongo/db/pipeline/document_source_limit_test.cpp +++ b/src/mongo/db/pipeline/document_source_limit_test.cpp @@ -161,5 +161,12 @@ TEST_F(DocumentSourceLimitTest, ShouldPropagatePauses) { ASSERT_TRUE(limit->getNext().isEOF()); } +TEST_F(DocumentSourceLimitTest, RedactsCorrectly) { + auto limit = DocumentSourceLimit::create(getExpCtx(), 2); + ASSERT_VALUE_EQ_AUTO( // NOLINT + "{ $limit: \"?number\" }", + redact(*limit)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_list_cached_and_active_users.h b/src/mongo/db/pipeline/document_source_list_cached_and_active_users.h index f1f1e25e55e..ce8a4f9883a 100644 --- a/src/mongo/db/pipeline/document_source_list_cached_and_active_users.h +++ b/src/mongo/db/pipeline/document_source_list_cached_and_active_users.h @@ -87,7 +87,8 @@ public: return kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { return Value(Document{{getSourceName(), Document{}}}); } diff --git a/src/mongo/db/pipeline/document_source_list_catalog.cpp b/src/mongo/db/pipeline/document_source_list_catalog.cpp index 7e7920bd99e..1675f931d52 100644 --- a/src/mongo/db/pipeline/document_source_list_catalog.cpp +++ b/src/mongo/db/pipeline/document_source_list_catalog.cpp @@ -133,8 +133,7 @@ intrusive_ptr<DocumentSource> DocumentSourceListCatalog::createFromBson( return new DocumentSourceListCatalog(pExpCtx); } -Value DocumentSourceListCatalog::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceListCatalog::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_list_catalog.h b/src/mongo/db/pipeline/document_source_list_catalog.h index c3bcb52fe77..89398ccb519 100644 --- a/src/mongo/db/pipeline/document_source_list_catalog.h +++ b/src/mongo/db/pipeline/document_source_list_catalog.h @@ -75,7 +75,7 @@ public: // virtuals from DocumentSource const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { StageConstraints constraints(StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_list_local_sessions.h b/src/mongo/db/pipeline/document_source_list_local_sessions.h index 5306fc1fb7e..22d7657ea3d 100644 --- a/src/mongo/db/pipeline/document_source_list_local_sessions.h +++ b/src/mongo/db/pipeline/document_source_list_local_sessions.h @@ -101,8 +101,9 @@ public: return DocumentSourceListLocalSessions::kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { - return Value(Document{{getSourceName(), _spec.toBSON()}}); + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { + return Value(Document{{getSourceName(), _spec.toBSON(opts)}}); } StageConstraints constraints(Pipeline::SplitState pipeState) const final { diff --git a/src/mongo/db/pipeline/document_source_list_sessions.cpp b/src/mongo/db/pipeline/document_source_list_sessions.cpp index 3c19beb4d5f..53744d3aa12 100644 --- a/src/mongo/db/pipeline/document_source_list_sessions.cpp +++ b/src/mongo/db/pipeline/document_source_list_sessions.cpp @@ -74,13 +74,12 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceListSessions::createFromBson( return new DocumentSourceListSessions(query, pExpCtx, spec.getAllUsers(), spec.getUsers()); } -Value DocumentSourceListSessions::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceListSessions::serialize(const SerializationOptions& opts) const { ListSessionsSpec spec; spec.setAllUsers(_allUsers); spec.setUsers(_users); spec.setPredicate(_predicate); - return Value(Document{{getSourceName(), spec.toBSON()}}); + return Value(Document{{getSourceName(), spec.toBSON(opts)}}); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_list_sessions.h b/src/mongo/db/pipeline/document_source_list_sessions.h index 9c3e4d10d68..ec47f6c1f89 100644 --- a/src/mongo/db/pipeline/document_source_list_sessions.h +++ b/src/mongo/db/pipeline/document_source_list_sessions.h @@ -98,7 +98,7 @@ public: return DocumentSourceListSessions::kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { return {StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_list_sessions.idl b/src/mongo/db/pipeline/document_source_list_sessions.idl index 0db282831dc..c293e85b589 100644 --- a/src/mongo/db/pipeline/document_source_list_sessions.idl +++ b/src/mongo/db/pipeline/document_source_list_sessions.idl @@ -39,21 +39,31 @@ structs: ListSessionsUser: description: "A struct representing a $listSessions/$listLocalSessions User" strict: true + query_shape_component: true fields: - user: string - db: string + user: + type: string + query_shape: anonymize + db: + type: string + query_shape: anonymize ListSessionsSpec: description: "$listSessions and $listLocalSessions pipeline spec" strict: true + query_shape_component: true fields: allUsers: type: bool default: false + # This boolean parameterizes the stage rather than representing user input, so do not abstract the literal. + query_shape: parameter users: type: array<ListSessionsUser> optional: true + query_shape: literal $_internalPredicate: cpp_name: predicate type: object optional: true + query_shape: literal # This is a MatchExpression predicate and could be shape-ified rather than completely abstracted. diff --git a/src/mongo/db/pipeline/document_source_lookup.cpp b/src/mongo/db/pipeline/document_source_lookup.cpp index 5e3d067f21c..0b623cb899d 100644 --- a/src/mongo/db/pipeline/document_source_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_lookup.cpp @@ -40,6 +40,7 @@ #include "mongo/db/pipeline/document_path_support.h" #include "mongo/db/pipeline/document_source_documents.h" #include "mongo/db/pipeline/document_source_merge_gen.h" +#include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/document_source_sort.h" #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/expression_context.h" @@ -143,11 +144,9 @@ bool checkModifiedPathsSortReorder(const SortPattern& sortPattern, } // namespace -DocumentSourceLookUp::DocumentSourceLookUp( - NamespaceString fromNs, - std::string as, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, - const boost::intrusive_ptr<ExpressionContext>& expCtx) +DocumentSourceLookUp::DocumentSourceLookUp(NamespaceString fromNs, + std::string as, + const boost::intrusive_ptr<ExpressionContext>& expCtx) : DocumentSource(kStageName, expCtx), _fromNs(std::move(fromNs)), _as(std::move(as)), @@ -159,35 +158,34 @@ DocumentSourceLookUp::DocumentSourceLookUp( _fromExpCtx = expCtx->copyForSubPipeline(resolvedNamespace.ns, resolvedNamespace.uuid); _fromExpCtx->inLookup = true; - if (fromCollator) { - _fromExpCtx->setCollator(std::move(fromCollator.get())); - _hasExplicitCollation = true; - } } -DocumentSourceLookUp::DocumentSourceLookUp( - NamespaceString fromNs, - std::string as, - std::string localField, - std::string foreignField, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, - const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSourceLookUp(fromNs, as, std::move(fromCollator), expCtx) { +DocumentSourceLookUp::DocumentSourceLookUp(NamespaceString fromNs, + std::string as, + std::string localField, + std::string foreignField, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSourceLookUp(fromNs, as, expCtx) { _localField = std::move(localField); _foreignField = std::move(foreignField); // We append an additional BSONObj to '_resolvedPipeline' as a placeholder for the $match stage // we'll eventually construct from the input document. _resolvedPipeline.reserve(_resolvedPipeline.size() + 1); - _resolvedPipeline.push_back(BSON("$match" << BSONObj())); - _fieldMatchPipelineIdx = _resolvedPipeline.size() - 1; + // Initialize the introspection pipeline before we insert the $match. This is okay because we do + // not use the introspection pipeline during/after query execution, which is when the $match is + // necessary. initializeResolvedIntrospectionPipeline(); + + _resolvedPipeline.push_back(BSON("$match" << BSONObj())); + _fieldMatchPipelineIdx = _resolvedPipeline.size() - 1; } std::vector<BSONObj> extractSourceStage(const std::vector<BSONObj>& pipeline) { if (!pipeline.empty() && (pipeline[0].hasField(DocumentSourceDocuments::kStageName) || + pipeline[0].hasField(DocumentSourceQueue::kStageName) || pipeline[0].hasField("$search"_sd))) { return {pipeline[0]}; } @@ -199,10 +197,9 @@ DocumentSourceLookUp::DocumentSourceLookUp( std::string as, std::vector<BSONObj> pipeline, BSONObj letVariables, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, boost::optional<std::pair<std::string, std::string>> localForeignFields, const boost::intrusive_ptr<ExpressionContext>& expCtx) - : DocumentSourceLookUp(fromNs, as, std::move(fromCollator), expCtx) { + : DocumentSourceLookUp(fromNs, as, expCtx) { // '_resolvedPipeline' will first be initialized by the constructor delegated to within this // constructor's initializer list. It will be populated with view pipeline prefix if 'fromNs' // represents a view. We will then append stages to ensure any view prefix is not overwritten. @@ -214,8 +211,9 @@ DocumentSourceLookUp::DocumentSourceLookUp( // the local/foreignField $match. It must next after $documents if present. auto sourceStages = extractSourceStage(pipeline); _resolvedPipeline.insert(_resolvedPipeline.end(), sourceStages.begin(), sourceStages.end()); - _resolvedPipeline.push_back(BSON("$match" << BSONObj())); - _fieldMatchPipelineIdx = _resolvedPipeline.size() - 1; + // Save the correct position of the $match, but wait to insert it until we have finished + // constructing the pipeline and created the introspection pipeline below. + _fieldMatchPipelineIdx = _resolvedPipeline.size(); // Add the user pipeline to '_resolvedPipeline' after any potential view prefix and $match _resolvedPipeline.insert( _resolvedPipeline.end(), pipeline.begin() + sourceStages.size(), pipeline.end()); @@ -240,7 +238,20 @@ DocumentSourceLookUp::DocumentSourceLookUp( _variablesParseState.defineVariable(varName)); } + // Initialize the introspection pipeline before we insert the $match (if applicable). This is + // okay because we only use the introspection pipeline for reference while doing query analysis + // and analyzing involved dependencies/variables/collections/constraints. We do not use the + // introspection pipeline during/after query execution, which is when the $match is necessary. + // It wouldn't hurt anything to include the $match in this pipeline, but we also use the + // introspection pipeline in serialization, so it would be a bit odd to include an extra empty + // $match. initializeResolvedIntrospectionPipeline(); + + // Finally, insert the $match placeholder if we need it. + if (_fieldMatchPipelineIdx) { + _resolvedPipeline.insert(_resolvedPipeline.begin() + *_fieldMatchPipelineIdx, + BSON("$match" << BSONObj())); + } } DocumentSourceLookUp::DocumentSourceLookUp(const DocumentSourceLookUp& original, @@ -256,10 +267,9 @@ DocumentSourceLookUp::DocumentSourceLookUp(const DocumentSourceLookUp& original, _variables(original._variables), _variablesParseState(original._variablesParseState.copyWith(_variables.useIdGenerator())), _fromExpCtx(original._fromExpCtx->copyWith(_resolvedNs, original._fromExpCtx->uuid)), - _hasExplicitCollation(original._hasExplicitCollation), _resolvedPipeline(original._resolvedPipeline), _userPipeline(original._userPipeline), - _resolvedIntrospectionPipeline(original._resolvedIntrospectionPipeline->clone()), + _resolvedIntrospectionPipeline(original._resolvedIntrospectionPipeline->clone(_fromExpCtx)), _letVariables(original._letVariables) { if (!_localField && !_foreignField) { _cache.emplace(internalDocumentSourceCursorBatchSizeBytes.load()); @@ -321,10 +331,8 @@ std::unique_ptr<DocumentSourceLookUp::LiteParsed> DocumentSourceLookUp::LitePars liteParsedPipeline = LiteParsedPipeline(fromNss, pipeline); } - bool hasInternalCollation = static_cast<bool>(specObj["_internalCollation"]); - return std::make_unique<DocumentSourceLookUp::LiteParsed>( - spec.fieldName(), std::move(fromNss), std::move(liteParsedPipeline), hasInternalCollation); + spec.fieldName(), std::move(fromNss), std::move(liteParsedPipeline)); } PrivilegeVector DocumentSourceLookUp::LiteParsed::requiredPrivileges( @@ -380,8 +388,6 @@ void DocumentSourceLookUp::determineSbeCompatibility() { // than indexes into arrays, which is compatible with SBE.) && !FieldRef(_localField->fullPath()).hasNumericPathComponents() && !FieldRef(_foreignField->fullPath()).hasNumericPathComponents() - // Setting a collator on an individual $lookup stage with _internalCollation isn't supported - && !_hasExplicitCollation // We currently don't lower $lookup against views ('_fromNs' does not correspond to a // view). && pExpCtx->getResolvedNamespace(_fromNs).pipeline.empty(); @@ -558,6 +564,7 @@ std::unique_ptr<Pipeline, PipelineDeleter> DocumentSourceLookUp::buildPipeline( // If we don't have a cache, build and return the pipeline immediately. if (!_cache || _cache->isAbandoned()) { MakePipelineOptions pipelineOpts; + pipelineOpts.alreadyOptimized = false; pipelineOpts.optimize = true; pipelineOpts.attachCursorSource = true; pipelineOpts.validator = lookupPipeValidator; @@ -592,6 +599,7 @@ std::unique_ptr<Pipeline, PipelineDeleter> DocumentSourceLookUp::buildPipeline( // Construct the basic pipeline without a cache stage. Avoid optimizing here since we need to // add the cache first, as detailed below. MakePipelineOptions pipelineOpts; + pipelineOpts.alreadyOptimized = false; pipelineOpts.optimize = false; pipelineOpts.attachCursorSource = false; pipelineOpts.validator = lookupPipeValidator; @@ -647,21 +655,55 @@ std::unique_ptr<Pipeline, PipelineDeleter> DocumentSourceLookUp::buildPipeline( return pipeline; } +/** + * Method that looks for a DocumentSourceSequentialDocumentCache stage and calls optimizeAt() on + * it if it has yet to be optimized. + */ +void findAndOptimizeSequentialDocumentCache(Pipeline& pipeline) { + auto& container = pipeline.getSources(); + auto itr = (&container)->begin(); + while (itr != (&container)->end()) { + if (dynamic_cast<DocumentSourceSequentialDocumentCache*>(itr->get())) { + auto sequentialCache = dynamic_cast<DocumentSourceSequentialDocumentCache*>(itr->get()); + if (!sequentialCache->hasOptimizedPos()) { + sequentialCache->optimizeAt(itr, &container); + } + } + itr = std::next(itr); + } +} + void DocumentSourceLookUp::addCacheStageAndOptimize(Pipeline& pipeline) { - // Add the cache stage at the end and optimize. During the optimization process, the cache will - // either move itself to the correct position in the pipeline, or will abandon itself if no - // suitable cache position exists. Do it only if pipeline optimization is enabled, otherwise - // Pipeline::optimizePipeline() will exit early and correct placement of the cache will not - // occur. + // Adds the cache to the end of the pipeline and calls optimizeContainer which will ensure the + // stages of the pipeline are in the correct and optimal order, before the cache runs + // doOptimizeAt. During the optimization process, the cache will either move itself to the + // correct position in the pipeline, or abandon itself if no suitable cache position exists. + // Once the cache is finished optimizing, the entire pipeline is optimized. + // + // When pipeline optimization is disabled, 'Pipeline::optimizePipeline()' exits early and so the + // cache would not be placed correctly. So we only add the cache when pipeline optimization is + // enabled. if (auto fp = globalFailPointRegistry().find("disablePipelineOptimization"); fp && fp->shouldFail()) { _cache->abandon(); } else { + // The cache needs to see the full pipeline in its correct order in order to properly place + // itself, therefore we are adding it to the end of the pipeline, and calling + // optimizeContainer on the pipeline to ensure the rest of the pipeline is in its correct + // order before optimizing the cache. pipeline.addFinalSource( DocumentSourceSequentialDocumentCache::create(_fromExpCtx, _cache.get_ptr())); - } - pipeline.optimizePipeline(); + auto& container = pipeline.getSources(); + + Pipeline::optimizeContainer(&container); + + // We want to ensure the cache has been optimized prior to any calls to optimize(). + findAndOptimizeSequentialDocumentCache(pipeline); + + // Optimize the pipeline, with the cache in its correct position if it exists. + Pipeline::optimizeEachStage(&container); + } } DocumentSource::GetModPathsReturn DocumentSourceLookUp::getModifiedPaths() const { @@ -766,12 +808,14 @@ Pipeline::SourceContainer::iterator DocumentSourceLookUp::doOptimizeAt( bool isMatchOnlyOnAs = true; auto computeWhetherMatchOnAs = [&isMatchOnlyOnAs, &outputPath](MatchExpression* expression, std::string path) -> void { - // If 'expression' is the child of a $elemMatch, we cannot internalize the $match. For - // example, {b: {$elemMatch: {$gt: 1, $lt: 4}}}, where "b" is our "_as" field. This is - // because there's no way to modify the expression to be a match just on 'b'--we cannot - // change the path to an empty string, or remove the node entirely. - if (expression->matchType() == MatchExpression::ELEM_MATCH_VALUE || - expression->matchType() == MatchExpression::ELEM_MATCH_OBJECT) { + // There are certain situations where this rewrite would not be correct. For example, + // if 'expression' is the child of a value $elemMatch, we cannot internalize the $match. + // Consider {b: {$elemMatch: {$gt: 1, $lt: 4}}}, where "b" is our "_as" field. This rewrite + // is not supported because there's no way to modify the expression to be a match just on + // 'b'--we cannot change the path to an empty string, or remove the node entirely. + // For other internal nodes with paths, we don't support the rewrite to keep the + // descendMatchOnPath implementation simple. + if (MatchExpression::isInternalNodeWithPath(expression->matchType())) { isMatchOnlyOnAs = false; } if (expression->numChildren() == 0) { @@ -1023,44 +1067,65 @@ void DocumentSourceLookUp::appendSpecificExecStats(MutableDocument& doc) const { doc["indexesUsed"] = Value{std::move(indexesUsedVec)}; } -void DocumentSourceLookUp::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { - +void DocumentSourceLookUp::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { // Support alternative $lookup from config.cache.chunks* namespaces. auto fromValue = (pExpCtx->ns.db() == _fromNs.db()) - ? Value(_fromNs.coll()) - : Value(Document{{"db", _fromNs.db()}, {"coll", _fromNs.coll()}}); + ? Value(opts.serializeIdentifier(_fromNs.coll())) + : Value(Document{{"db", opts.serializeIdentifier(_fromNs.db())}, + {"coll", opts.serializeIdentifier(_fromNs.coll())}}); - MutableDocument output( - Document{{getSourceName(), Document{{"from", fromValue}, {"as", _as.fullPath()}}}}); + MutableDocument output(Document{ + {getSourceName(), Document{{"from", fromValue}, {"as", opts.serializeFieldPath(_as)}}}}); if (hasLocalFieldForeignFieldJoin()) { - output[getSourceName()]["localField"] = Value(_localField->fullPath()); - output[getSourceName()]["foreignField"] = Value(_foreignField->fullPath()); + output[getSourceName()]["localField"] = Value(opts.serializeFieldPath(_localField.value())); + output[getSourceName()]["foreignField"] = + Value(opts.serializeFieldPath(_foreignField.value())); } // Add a pipeline field if only-pipeline syntax was used (to ensure the output is valid $lookup // syntax) or if a $match was absorbed. - auto pipeline = _userPipeline.get_value_or(std::vector<BSONObj>()); + auto serializedPipeline = [&]() -> std::vector<BSONObj> { + if (!_userPipeline) { + return std::vector<BSONObj>{}; + } + if (opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + return Pipeline::parse(*_userPipeline, _fromExpCtx, lookupPipeValidator) + ->serializeToBson(opts); + } + if (opts.serializeForQueryAnalysis) { + // If we are in query analysis, encrypted fields will have been marked in the + // introspection pipeline, so we need to serialize that here. + return _resolvedIntrospectionPipeline->serializeToBson(opts); + } + return *_userPipeline; + }(); if (_additionalFilter) { - pipeline.emplace_back(BSON("$match" << *_additionalFilter)); + auto serializedFilter = [&]() -> BSONObj { + if (opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + auto filter = + uassertStatusOK(MatchExpressionParser::parse(*_additionalFilter, pExpCtx)); + return filter->serialize(opts); + } + return *_additionalFilter; + }(); + serializedPipeline.emplace_back(BSON("$match" << serializedFilter)); } - if (!hasLocalFieldForeignFieldJoin() || pipeline.size() > 0) { + if (!hasLocalFieldForeignFieldJoin() || serializedPipeline.size() > 0) { MutableDocument exprList; for (auto letVar : _letVariables) { - exprList.addField(letVar.name, - letVar.expression->serialize(static_cast<bool>(explain))); + exprList.addField(opts.serializeFieldPathFromString(letVar.name), + letVar.expression->serialize(opts)); } output[getSourceName()]["let"] = Value(exprList.freeze()); - output[getSourceName()]["pipeline"] = Value(pipeline); - } - - if (_hasExplicitCollation) { - output[getSourceName()]["_internalCollation"] = Value(_fromExpCtx->getCollatorBSON()); + output[getSourceName()]["pipeline"] = Value(serializedPipeline); } - if (explain) { + if (opts.verbosity) { if (_unwindSrc) { const boost::optional<FieldPath> indexPath = _unwindSrc->indexPath(); output[getSourceName()]["unwinding"] = @@ -1068,8 +1133,11 @@ void DocumentSourceLookUp::serializeToArray( << _unwindSrc->preserveNullAndEmptyArrays() << "includeArrayIndex" << (indexPath ? Value(indexPath->fullPath()) : Value()))); } - - if (explain.get() >= ExplainOptions::Verbosity::kExecStats) { + // Conflict is .get() vs .value(), changed in 6.1 during + // https://jira.mongodb.org/browse/SERVER-68246. Going to stick with 7.0's version of + // '.value()', I think there is no meaningful difference here since line 1080 check that the + // optional is set. + if (opts.verbosity.value() >= ExplainOptions::Verbosity::kExecStats) { appendSpecificExecStats(output); } @@ -1203,7 +1271,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( std::vector<BSONObj> pipeline; bool hasPipeline = false; bool hasLet = false; - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator; for (auto&& argument : elem.Obj()) { const auto argName = argument.fieldNameStringData(); @@ -1229,16 +1296,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( continue; } - if (argName == "_internalCollation"_sd) { - const auto& collationSpec = argument.Obj(); - if (!collationSpec.isEmpty()) { - fromCollator.emplace(uassertStatusOK( - CollatorFactoryInterface::get(pExpCtx->opCtx->getServiceContext()) - ->makeFromBSON(collationSpec))); - } - continue; - } - uassert(ErrorCodes::FailedToParse, str::stream() << "$lookup argument '" << argName << "' must be a string, found " << argument << ": " << argument.type(), @@ -1270,7 +1327,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( std::move(as), std::move(pipeline), std::move(letVariables), - std::move(fromCollator), boost::none, pExpCtx); } else { @@ -1285,7 +1341,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( std::move(as), std::move(pipeline), std::move(letVariables), - std::move(fromCollator), std::pair(std::move(localField), std::move(foreignField)), pExpCtx); } @@ -1303,7 +1358,6 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceLookUp::createFromBson( std::move(as), std::move(localField), std::move(foreignField), - std::move(fromCollator), pExpCtx); } lookupStage->determineSbeCompatibility(); diff --git a/src/mongo/db/pipeline/document_source_lookup.h b/src/mongo/db/pipeline/document_source_lookup.h index 7985601c02e..61298c298f5 100644 --- a/src/mongo/db/pipeline/document_source_lookup.h +++ b/src/mongo/db/pipeline/document_source_lookup.h @@ -71,11 +71,9 @@ public: LiteParsed(std::string parseTimeName, NamespaceString foreignNss, - boost::optional<LiteParsedPipeline> pipeline, - bool hasInternalCollation) + boost::optional<LiteParsedPipeline> pipeline) : LiteParsedDocumentSourceNestedPipelines( - std::move(parseTimeName), std::move(foreignNss), std::move(pipeline)), - _hasInternalCollation(hasInternalCollation) {} + std::move(parseTimeName), std::move(foreignNss), std::move(pipeline)) {} /** * Lookup from a sharded collection may not be allowed. @@ -91,16 +89,6 @@ public: return (involvedNss.find(nss) == involvedNss.end()); } - void assertPermittedInAPIVersion(const APIParameters& apiParameters) const final { - if (apiParameters.getAPIVersion() && *apiParameters.getAPIVersion() == "1" && - apiParameters.getAPIStrict().value_or(false)) { - uassert( - ErrorCodes::APIStrictError, - "The _internalCollation argument to $lookup is not supported in API Version 1", - !_hasInternalCollation); - } - } - void getForeignExecutionNamespaces( stdx::unordered_set<NamespaceString>& nssSet) const final { // We do not recurse on, nor insert '_foreignNss' in the event that this $lookup has @@ -112,10 +100,7 @@ public: } PrivilegeVector requiredPrivileges(bool isMongos, - bool bypassDocumentValidation) const override final; - - private: - bool _hasInternalCollation = false; + bool bypassDocumentValidation) const final; }; /** @@ -127,7 +112,7 @@ public: const char* getSourceName() const final; void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Returns the 'as' path, and possibly fields modified by an absorbed $unwind. @@ -276,7 +261,6 @@ private: */ DocumentSourceLookUp(NamespaceString fromNs, std::string as, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, const boost::intrusive_ptr<ExpressionContext>& expCtx); /** * Constructor used for a $lookup stage specified using the {from: ..., localField: ..., @@ -286,7 +270,6 @@ private: std::string as, std::string localField, std::string foreignField, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, const boost::intrusive_ptr<ExpressionContext>& expCtx); /** @@ -298,15 +281,15 @@ private: std::string as, std::vector<BSONObj> pipeline, BSONObj letVariables, - boost::optional<std::unique_ptr<CollatorInterface>> fromCollator, boost::optional<std::pair<std::string, std::string>> localForeignFields, const boost::intrusive_ptr<ExpressionContext>& expCtx); /** * Should not be called; use serializeToArray instead. */ - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { - MONGO_UNREACHABLE; + Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { + MONGO_UNREACHABLE_TASSERT(7484304); } GetNextResult unwindResult(); @@ -400,12 +383,6 @@ private: // namespace. boost::intrusive_ptr<ExpressionContext> _fromExpCtx; - // When a `_internalCollation` has been specified on a $lookup stage, we will set that collation - // on `_fromExpCtx`. An explicit simple collation however is represented in the same way as the - // default binary collation. We need to differentiate between the two to avoid serializing the - // collation when not set explicitly. - bool _hasExplicitCollation = false; - // Can this $lookup be pushed down into SBE? bool _sbeCompatible = false; diff --git a/src/mongo/db/pipeline/document_source_lookup_test.cpp b/src/mongo/db/pipeline/document_source_lookup_test.cpp index a565fb60c26..30f764808db 100644 --- a/src/mongo/db/pipeline/document_source_lookup_test.cpp +++ b/src/mongo/db/pipeline/document_source_lookup_test.cpp @@ -61,7 +61,8 @@ using std::vector; using DocumentSourceLookUpTest = AggregationContextFixture; const long long kDefaultMaxCacheSize = internalDocumentSourceLookupCacheSizeBytes.load(); -const auto kExplain = ExplainOptions::Verbosity::kQueryPlanner; +const auto kExplain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; // For tests which need to run in a replica set context. class ReplDocumentSourceLookUpTest : public DocumentSourceLookUpTest { @@ -608,49 +609,46 @@ TEST_F(DocumentSourceLookUpTest, LookupWithLetReParseSerializedStageWithFromDBAn ASSERT_VALUE_EQ(newSerialization[0], serialization[0]); } - -// Tests that $lookup with 'collation' can be round tripped. -TEST_F(DocumentSourceLookUpTest, LookupReParseSerializedStageWithCollation) { +// Tests that $lookup with '$documents' can be round tripped. +TEST_F(DocumentSourceLookUpTest, LookupReParseSerializedStageWithDocumentsPipelineStage) { auto expCtx = getExpCtx(); - NamespaceString fromNs("test", "coll"); + NamespaceString fromNs = NamespaceString("unittest", "$cmd.aggregate"); expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ {fromNs.coll().toString(), {fromNs, std::vector<BSONObj>()}}}); - - auto originalBSON = BSON( - "$lookup" << BSON("from" - << "coll" - << "let" - << BSON("local_x" - << "$x") - << "pipeline" << BSON_ARRAY(BSON("$match" << BSON("x" << 1))) << "as" - << "as" - << "_internalCollation" - << BSON("locale" - << "en_US" - << "caseLevel" << false << "caseFirst" - << "off" - << "strength" << 1 << "numericOrdering" << false << "alternate" - << "non-ignorable" - << "maxVariable" - << "punct" - << "normalization" << false << "backwards" << false << "version" - << "57.1"))); + auto originalBSON = + BSON("$lookup" << BSON("localField" + << "y" + << "foreignField" + << "x" + << "pipeline" + << BSON_ARRAY(BSON("$documents" + << BSON_ARRAY(BSON("x" << 5) << BSON("y" << 15)))) + << "as" + << "as")); auto lookupStage = DocumentSourceLookUp::createFromBson(originalBSON.firstElement(), expCtx); // // Serialize the $lookup stage and confirm contents. // vector<Value> serialization; - static const UnorderedFieldsBSONObjComparator kComparator; - lookupStage->serializeToArray(serialization); - auto serializedBSON = serialization[0].getDocument().toBson(); - std::cout << "serializedBSON: " << serializedBSON << std::endl; - ASSERT_EQ(kComparator.compare(serializedBSON, originalBSON), 0); + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + lookupStage->serializeToArray(serialization, opts); + auto serializedDoc = serialization[0].getDocument(); + ASSERT_EQ(serializedDoc["$lookup"].getType(), BSONType::Object); - auto roundTripped = DocumentSourceLookUp::createFromBson(serializedBSON.firstElement(), expCtx); + // Ensure the $documents desugared to $queue properly. + auto serializedStage = serializedDoc["$lookup"].getDocument(); + ASSERT_EQ(serializedStage["pipeline"].getType(), BSONType::Array); + ASSERT_EQ(serializedStage["pipeline"].getArrayLength(), 4UL); + + ASSERT_EQ(serializedStage["pipeline"][0].getType(), BSONType::Object); + ASSERT_EQ(serializedStage["pipeline"][0]["$queue"].getType(), BSONType::Array); + + auto roundTripped = + DocumentSourceLookUp::createFromBson(serializedDoc.toBson().firstElement(), expCtx); vector<Value> newSerialization; - roundTripped->serializeToArray(newSerialization); + roundTripped->serializeToArray(newSerialization, opts); ASSERT_EQ(newSerialization.size(), 1UL); ASSERT_VALUE_EQ(newSerialization[0], serialization[0]); @@ -1106,13 +1104,10 @@ TEST_F(DocumentSourceLookUpTest, ExprEmbeddedInMatchExpressionShouldBeOptimized) auto& matchSource = dynamic_cast<const DocumentSourceMatch&>(*secondSource); // Ensure that the '$$var' in the embedded expression got optimized to ExpressionConstant. - BSONObjBuilder builder; - matchSource.getMatchExpression()->serialize(&builder); - auto serializedMatch = builder.obj(); auto expectedMatch = fromjson("{$and: [{_id: {$_internalExprEq: 5}}, {$expr: {$eq: ['$_id', {$const: 5}]}}]}"); - ASSERT_VALUE_EQ(Value(serializedMatch), Value(expectedMatch)); + ASSERT_VALUE_EQ(Value(matchSource.getMatchExpression()->serialize()), Value(expectedMatch)); } TEST_F(DocumentSourceLookUpTest, @@ -1426,5 +1421,57 @@ TEST_F(DocumentSourceLookUpTest, ShouldNotCacheIfCorrelatedStageIsAbsorbedIntoPl ASSERT_VALUE_EQ(Value(subPipeline->writeExplainOps(kExplain)), Value(BSONArray(expectedPipe))); } +TEST_F(DocumentSourceLookUpTest, RedactsCorrectlyWithPipeline) { + auto expCtx = getExpCtx(); + NamespaceString fromNs("test", "coll"); + expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {fromNs.coll().toString(), {fromNs, std::vector<BSONObj>()}}}); + + BSONArrayBuilder pipeline; + pipeline << BSON("$match" << BSON("a" + << "myStr")); + pipeline << BSON("$project" << BSON("_id" << 0 << "a" << 1)); + auto docSource = DocumentSourceLookUp::createFromBson( + BSON("$lookup" << BSON("from" << fromNs.coll() << "localField" + << "foo" + << "foreignField" + << "bar" + << "let" + << BSON("var1" + << "$x") + << "pipeline" << pipeline.arr() << "as" + << "out")) + .firstElement(), + expCtx); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$lookup": { + "from": "HASH<coll>", + "as": "HASH<out>", + "localField": "HASH<foo>", + "foreignField": "HASH<bar>", + "let": { + "HASH<var1>": "$HASH<x>" + }, + "pipeline": [ + { + "$match": { + "HASH<a>": { + "$eq": "?string" + } + } + }, + { + "$project": { + "HASH<a>": true, + "HASH<_id>": false + } + } + ] + } + })", + redact(*docSource)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_match.cpp b/src/mongo/db/pipeline/document_source_match.cpp index 418f1821b4b..3690b603ab7 100644 --- a/src/mongo/db/pipeline/document_source_match.cpp +++ b/src/mongo/db/pipeline/document_source_match.cpp @@ -47,6 +47,7 @@ #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/lite_parsed_document_source.h" #include "mongo/db/pipeline/semantic_analysis.h" +#include "mongo/logv2/redaction.h" #include "mongo/util/ctype.h" #include "mongo/util/str.h" @@ -67,11 +68,10 @@ const char* DocumentSourceMatch::getSourceName() const { return kStageName.rawData(); } -Value DocumentSourceMatch::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { - BSONObjBuilder builder; - _expression->serialize(&builder); - return Value(DOC(getSourceName() << Document(builder.obj()))); +Value DocumentSourceMatch::serialize(const SerializationOptions& opts) const { + if (opts.verbosity || opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + return Value(DOC(getSourceName() << Document(_expression->serialize(opts)))); } return Value(DOC(getSourceName() << Document(getQuery()))); } @@ -449,15 +449,11 @@ DocumentSourceMatch::splitSourceByFunc(const OrderedPathSet& fields, // the corresponding BSONObj may not exist. Therefore, we take each of these expressions, // serialize them, and then re-parse them, constructing new BSON that is owned by the // DocumentSourceMatch. - BSONObjBuilder firstBob; - newExpr.first->serialize(&firstBob); - auto firstMatch = DocumentSourceMatch::create(firstBob.obj(), pExpCtx); + auto firstMatch = DocumentSourceMatch::create(newExpr.first->serialize(), pExpCtx); intrusive_ptr<DocumentSourceMatch> secondMatch; if (newExpr.second) { - BSONObjBuilder secondBob; - newExpr.second->serialize(&secondBob); - secondMatch = DocumentSourceMatch::create(secondBob.obj(), pExpCtx); + secondMatch = DocumentSourceMatch::create(newExpr.second->serialize(), pExpCtx); } return {std::move(firstMatch), std::move(secondMatch)}; @@ -469,8 +465,9 @@ boost::intrusive_ptr<DocumentSourceMatch> DocumentSourceMatch::descendMatchOnPat const intrusive_ptr<ExpressionContext>& expCtx) { expression::mapOver(matchExpr, [&descendOn](MatchExpression* node, std::string path) -> void { // Cannot call this method on a $match including a $elemMatch. - invariant(node->matchType() != MatchExpression::ELEM_MATCH_OBJECT && - node->matchType() != MatchExpression::ELEM_MATCH_VALUE); + tassert(9224700, + "The given match expression has a node that represents a partial path.", + !MatchExpression::isInternalNodeWithPath(node->matchType())); // Only leaf and array match expressions have a path. if (node->getCategory() != MatchExpression::MatchCategory::kLeaf && node->getCategory() != MatchExpression::MatchCategory::kArrayMatching) { @@ -478,7 +475,10 @@ boost::intrusive_ptr<DocumentSourceMatch> DocumentSourceMatch::descendMatchOnPat } auto leafPath = node->path(); - invariant(expression::isPathPrefixOf(descendOn, leafPath)); + tassert(9224701, + str::stream() << "Expected '" << redact(descendOn) << "' to be a prefix of '" + << redact(leafPath) << "', but it is not.", + expression::isPathPrefixOf(descendOn, leafPath)); auto newPath = leafPath.substr(descendOn.size() + 1); if (node->getCategory() == MatchExpression::MatchCategory::kLeaf) { @@ -490,9 +490,7 @@ boost::intrusive_ptr<DocumentSourceMatch> DocumentSourceMatch::descendMatchOnPat } }); - BSONObjBuilder query; - matchExpr->serialize(&query); - return new DocumentSourceMatch(query.obj(), expCtx); + return new DocumentSourceMatch(matchExpr->serialize(), expCtx); } std::pair<boost::intrusive_ptr<DocumentSourceMatch>, boost::intrusive_ptr<DocumentSourceMatch>> @@ -579,4 +577,22 @@ void DocumentSourceMatch::rebuild(BSONObj filter) { getDependencies(&_dependencies); } +Value DocumentSourceInternalChangeStreamMatch::serialize(const SerializationOptions& opts) const { + if (opts.literalPolicy != LiteralSerializationPolicy::kUnchanged || opts.transformIdentifiers) { + // Stages made internally by 'DocumentSourceChangeStream' should not be serialized for + // query stats. For query stats we will serialize only the user specified $changeStream + // stage. + return Value(); + } + return doSerialize(opts); +} + +intrusive_ptr<DocumentSourceInternalChangeStreamMatch> +DocumentSourceInternalChangeStreamMatch::create(BSONObj filter, + const intrusive_ptr<ExpressionContext>& expCtx) { + intrusive_ptr<DocumentSourceInternalChangeStreamMatch> internalMatch( + new DocumentSourceInternalChangeStreamMatch(filter, expCtx)); + return internalMatch; +} + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_match.h b/src/mongo/db/pipeline/document_source_match.h index 0f55c2c9dba..53c336a322d 100644 --- a/src/mongo/db/pipeline/document_source_match.h +++ b/src/mongo/db/pipeline/document_source_match.h @@ -126,8 +126,7 @@ public: ChangeStreamRequirement::kAllowlist}; } - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const override; /** * Attempts to combine with any subsequent $match stages, joining the query objects with a @@ -232,4 +231,42 @@ private: DepsTracker _dependencies; }; +/** + * A DocumentSource class for all internal change stream stages that are also match stages. This + * currently handles parsing for query stats. + */ +class DocumentSourceInternalChangeStreamMatch : public DocumentSourceMatch { +public: + DocumentSourceInternalChangeStreamMatch(std::unique_ptr<MatchExpression> expr, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSourceMatch(std::move(expr), expCtx) {} + + static boost::intrusive_ptr<DocumentSourceInternalChangeStreamMatch> create( + BSONObj filter, const boost::intrusive_ptr<ExpressionContext>& expCtx); + + /** + * Must override the serialize method, since internal change stream stages are serialized + * differently than match stages. This function mirrors + * DocumentSourceInternalChangeStreamStage::serialize and was added because this class cannot + * inherit from both DocumentSourceInternalChangeStreamStage and DocumentSourceMatch. + */ + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + + virtual Value doSerialize(const SerializationOptions& opts) const { + return DocumentSourceMatch::serialize(opts); + }; + +protected: + DocumentSourceInternalChangeStreamMatch(const BSONObj& query, + const boost::intrusive_ptr<ExpressionContext>& expCtx) + : DocumentSourceMatch(query, expCtx) {} + + DocumentSourceInternalChangeStreamMatch( + const DocumentSourceInternalChangeStreamMatch& other, + const boost::intrusive_ptr<ExpressionContext>& newExpCtx) + : DocumentSourceMatch( + other.serialize().getDocument().toBson().firstElement().embeddedObject(), + newExpCtx ? newExpCtx : other.pExpCtx) {} +}; + } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_match_test.cpp b/src/mongo/db/pipeline/document_source_match_test.cpp index f5b53a393e8..4017720e667 100644 --- a/src/mongo/db/pipeline/document_source_match_test.cpp +++ b/src/mongo/db/pipeline/document_source_match_test.cpp @@ -54,7 +54,6 @@ using std::string; // This provides access to getExpCtx(), but we'll use a different name for this test suite. using DocumentSourceMatchTest = AggregationContextFixture; -constexpr auto kExplain = ExplainOptions::Verbosity::kQueryPlanner; TEST_F(DocumentSourceMatchTest, RedactSafePortion) { auto expCtx = getExpCtx(); @@ -589,7 +588,7 @@ TEST_F(DocumentSourceMatchTest, RepeatedJoinWithShouldNotNestAnds) { DEATH_TEST_REGEX_F(DocumentSourceMatchTest, ShouldFailToDescendExpressionOnPathThatIsNotACommonPrefix, - "Invariant failure.*expression::isPathPrefixOf") { + "Tripwire assertion.*Expected 'a' to be a prefix of 'b.c', but it is not.") { const auto expCtx = getExpCtx(); const auto matchSpec = BSON("a.b" << 1 << "b.c" << 1); const auto matchExpression = @@ -597,25 +596,23 @@ DEATH_TEST_REGEX_F(DocumentSourceMatchTest, DocumentSourceMatch::descendMatchOnPath(matchExpression.get(), "a", expCtx); } -DEATH_TEST_REGEX_F(DocumentSourceMatchTest, - ShouldFailToDescendExpressionOnPathThatContainsElemMatchWithObject, - R"#(Invariant failure.*node->matchType\(\))#") { +DEATH_TEST_REGEX_F( + DocumentSourceMatchTest, + ShouldFailToDescendExpressionOnPathThatContainsElemMatchWithObject, + "Tripwire assertion.*The given match expression has a node that represents a partial path.") { const auto expCtx = getExpCtx(); const auto matchSpec = BSON("a" << BSON("$elemMatch" << BSON("a.b" << 1))); const auto matchExpression = unittest::assertGet(MatchExpressionParser::parse(matchSpec, expCtx)); - BSONObjBuilder out; - matchExpression->serialize(&out); DocumentSourceMatch::descendMatchOnPath(matchExpression.get(), "a", expCtx); } -// Due to the order of traversal of the MatchExpression tree, this test may actually trigger the -// invariant failure that the path being descended is not a prefix of the path of the -// MatchExpression node corresponding to the '$gt' expression, which will report an empty path. -DEATH_TEST_F(DocumentSourceMatchTest, - ShouldFailToDescendExpressionOnPathThatContainsElemMatchWithValue, - "Invariant failure") { +DEATH_TEST_REGEX_F(DocumentSourceMatchTest, + ShouldFailToDescendExpressionOnPathThatContainsElemMatchWithValue, + "Tripwire assertion.") { const auto expCtx = getExpCtx(); + // We will either hit the assertion that $elemMatch is not allowed to be descended on or the + // assertion that the path of the '$gt' expression (empty path) is not prefixed by 'a' const auto matchSpec = BSON("a" << BSON("$elemMatch" << BSON("$gt" << 0))); const auto matchExpression = unittest::assertGet(MatchExpressionParser::parse(matchSpec, expCtx)); @@ -698,9 +695,71 @@ TEST_F(DocumentSourceMatchTest, ShouldShowOptimizationsInExplainOutputWhenOptimi auto expectedMatch = fromjson("{$match: {a:{$eq: 1}}}"); - ASSERT_VALUE_EQ( - Value((static_cast<DocumentSourceMatch*>(optimizedMatch.get()))->serialize(kExplain)), - Value(expectedMatch)); + ASSERT_VALUE_EQ(Value((static_cast<DocumentSourceMatch*>(optimizedMatch.get())) + ->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})), + Value(expectedMatch)); +} + +TEST_F(DocumentSourceMatchTest, RedactionWithAnd) { + auto spec = fromjson(R"({ + $match: { + $and: [ + { + "a.c": "abc" + }, + { + "b": { + $gt: 10 + } + } + ] + }})"); + auto docSource = DocumentSourceMatch::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$match": { + "$and": [ + { + "HASH<a>.HASH<c>": { + "$eq": "?string" + } + }, + { + "HASH<b>": { + "$gt": "?number" + } + } + ] + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceMatchTest, RedactionWithExprPipeline) { + auto spec = fromjson(R"({ + $match: { + $expr: { + $eq: [ + '$foo', + '$bar' + ] + } + } + })"); + auto docSource = DocumentSourceMatch::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$match": { + "$expr": { + "$eq": [ + "$HASH<foo>", + "$HASH<bar>" + ] + } + } + })", + redact(*docSource)); } } // namespace diff --git a/src/mongo/db/pipeline/document_source_merge.cpp b/src/mongo/db/pipeline/document_source_merge.cpp index e2746296f01..c46cfa8c641 100644 --- a/src/mongo/db/pipeline/document_source_merge.cpp +++ b/src/mongo/db/pipeline/document_source_merge.cpp @@ -605,7 +605,7 @@ boost::optional<DocumentSource::DistributedPlanLogic> DocumentSourceMerge::distr return DocumentSourceWriter::distributedPlanLogic(); } -Value DocumentSourceMerge::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceMerge::serialize(const SerializationOptions& opts) const { DocumentSourceMergeSpec spec; spec.setTargetNss(_outputNs); spec.setLet([&]() -> boost::optional<BSONObj> { @@ -615,11 +615,27 @@ Value DocumentSourceMerge::serialize(boost::optional<ExplainOptions::Verbosity> BSONObjBuilder bob; for (auto&& [name, expr] : *_letVariables) { - bob << name << expr->serialize(static_cast<bool>(explain)); + bob << opts.serializeFieldPathFromString(name) << expr->serialize(opts); } return bob.obj(); }()); - spec.setWhenMatched(MergeWhenMatchedPolicy{_descriptor.mode.first, _pipeline}); + spec.setWhenMatched(MergeWhenMatchedPolicy{ + _descriptor.mode.first, [&]() -> boost::optional<std::vector<BSONObj>> { + if (!_pipeline.has_value()) { + return boost::none; + } + auto expCtxWithLetVariables = pExpCtx->copyWith(getOutputNs()); + if (spec.getLet()) { + BSONObjBuilder cleanLetSpecBuilder; + for (auto&& [name, expr] : *_letVariables) { + cleanLetSpecBuilder.append(name, BSONObj{}); + } + expCtxWithLetVariables->variables.seedVariablesWithLetParameters( + expCtxWithLetVariables.get(), cleanLetSpecBuilder.obj()); + } + return Pipeline::parse(_pipeline.value(), expCtxWithLetVariables) + ->serializeToBson(opts); + }()}); spec.setWhenNotMatched(_descriptor.mode.second); spec.setOn([&]() { std::vector<std::string> mergeOnFields; @@ -629,7 +645,7 @@ Value DocumentSourceMerge::serialize(boost::optional<ExplainOptions::Verbosity> return mergeOnFields; }()); spec.setTargetCollectionVersion(_targetCollectionVersion); - return Value(Document{{getSourceName(), spec.toBSON()}}); + return Value(Document{{getSourceName(), spec.toBSON(opts)}}); } std::pair<DocumentSourceMerge::BatchObject, int> DocumentSourceMerge::makeBatchObject( diff --git a/src/mongo/db/pipeline/document_source_merge.h b/src/mongo/db/pipeline/document_source_merge.h index 9388add24ba..fe5bec997e2 100644 --- a/src/mongo/db/pipeline/document_source_merge.h +++ b/src/mongo/db/pipeline/document_source_merge.h @@ -135,7 +135,7 @@ public: boost::optional<DistributedPlanLogic> distributedPlanLogic() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Creates a new $merge stage from the given arguments. diff --git a/src/mongo/db/pipeline/document_source_merge.idl b/src/mongo/db/pipeline/document_source_merge.idl index 7948517e773..6aa54ec327d 100644 --- a/src/mongo/db/pipeline/document_source_merge.idl +++ b/src/mongo/db/pipeline/document_source_merge.idl @@ -81,16 +81,19 @@ structs: DocumentSourceMergeSpec: description: A document used to specify the $merge stage of an aggregation pipeline. strict: true + query_shape_component: true fields: into: cpp_name: targetNss type: MergeTargetNss description: Target namespace to merge documents from an aggregation pipeline into. + query_shape: custom on: type: MergeOnFields optional: true description: A single field or array of fields that uniquely identify a document. + query_shape: custom let: type: object @@ -98,18 +101,21 @@ structs: description: Specifies variables to use in the update pipeline defined in MergeWhenMatchedPolicy when the 'whenMatched' mode is a custom pipeline. + query_shape: custom whenMatched: type: MergeWhenMatchedPolicy optional: true description: The merge mode for the merge operation when source and target elements match. + query_shape: parameter whenNotMatched: type: MergeWhenNotMatchedMode optional: true description: The merge mode for the merge operation when source and target elements do not match. + query_shape: parameter targetCollectionVersion: type: ChunkVersionArrayWronglyEncodedAsBSONObjFormat @@ -119,3 +125,4 @@ structs: in which case the shard key may have changed, or had its shard key refined. This also can be used to detect if the collection has gone from unsharded to sharded, and thus now has a shard key. + query_shape: parameter diff --git a/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp b/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp index 61b2aef0c97..5f349309ac7 100644 --- a/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp +++ b/src/mongo/db/pipeline/document_source_merge_cursors_test.cpp @@ -53,6 +53,7 @@ #include "mongo/s/client/shard_registry.h" #include "mongo/s/sharding_router_test_fixture.h" #include "mongo/stdx/thread.h" +#include "mongo/unittest/bson_test_util.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -380,5 +381,42 @@ TEST_F(DocumentSourceMergeCursorsTest, ShouldEnforceSortSpecifiedViaARMParams) { future.default_timed_get(); } + +using DocumentSourceMergeCursorsShapeTest = AggregationContextFixture; +TEST_F(DocumentSourceMergeCursorsShapeTest, QueryShape) { + auto expCtx = getExpCtx(); + AsyncResultsMergerParams armParams; + armParams.setNss(kTestNss); + std::vector<RemoteCursor> cursors; + cursors.emplace_back( + makeRemoteCursor(kTestShardIds[0], kTestShardHosts[0], CursorResponse(expCtx->ns, 1, {}))); + cursors.emplace_back( + makeRemoteCursor(kTestShardIds[1], kTestShardHosts[1], CursorResponse(expCtx->ns, 2, {}))); + armParams.setRemotes(std::move(cursors)); + auto stage = DocumentSourceMergeCursors::create(expCtx, std::move(armParams)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$mergeCursors": { + "compareWholeSortKey": "?bool", + "remotes": [ + { + "shardId": "HASH<FakeShard1>", + "hostAndPort": "HASH<FakeShard1Host:12345>", + "cursorResponse": "?object" + }, + { + "shardId": "HASH<FakeShard2>", + "hostAndPort": "HASH<FakeShard2Host:12345>", + "cursorResponse": "?object" + } + ], + "nss": "HASH<test.mergeCursors>", + "allowPartialResults": false, + "recordRemoteOpWaitTime": false + } + })", + redact(*stage)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_merge_spec.cpp b/src/mongo/db/pipeline/document_source_merge_spec.cpp index 41926a7cfd9..84184804dd2 100644 --- a/src/mongo/db/pipeline/document_source_merge_spec.cpp +++ b/src/mongo/db/pipeline/document_source_merge_spec.cpp @@ -37,6 +37,7 @@ #include "mongo/db/pipeline/aggregation_request_helper.h" #include "mongo/db/pipeline/document_source_merge.h" #include "mongo/db/pipeline/document_source_merge_gen.h" +#include "mongo/db/query/query_shape/serialization_options.h" namespace mongo { using namespace fmt::literals; @@ -66,8 +67,11 @@ NamespaceString mergeTargetNssParseFromBSON(const BSONElement& elem) { void mergeTargetNssSerializeToBSON(const NamespaceString& targetNss, StringData fieldName, - BSONObjBuilder* bob) { - bob->append(fieldName, BSON("db" << targetNss.db() << "coll" << targetNss.coll())); + BSONObjBuilder* bob, + const SerializationOptions& opts) { + bob->append(fieldName, + BSON("db" << opts.serializeIdentifier(targetNss.db()) << "coll" + << opts.serializeIdentifier(targetNss.coll()))); } std::vector<std::string> mergeOnFieldsParseFromBSON(const BSONElement& elem) { @@ -104,11 +108,12 @@ std::vector<std::string> mergeOnFieldsParseFromBSON(const BSONElement& elem) { void mergeOnFieldsSerializeToBSON(const std::vector<std::string>& fields, StringData fieldName, - BSONObjBuilder* bob) { + BSONObjBuilder* bob, + const SerializationOptions& opts) { if (fields.size() == 1) { - bob->append(fieldName, fields.front()); + bob->append(fieldName, opts.serializeFieldPathFromString(fields.front())); } else { - bob->append(fieldName, fields); + bob->append(fieldName, opts.serializeFieldPathFromString(fields)); } } diff --git a/src/mongo/db/pipeline/document_source_merge_spec.h b/src/mongo/db/pipeline/document_source_merge_spec.h index 35c353bb78a..2ccd4f8548a 100644 --- a/src/mongo/db/pipeline/document_source_merge_spec.h +++ b/src/mongo/db/pipeline/document_source_merge_spec.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/db/query/query_shape/serialization_options.h" #include <boost/optional.hpp> #include <string> #include <vector> @@ -54,7 +55,8 @@ struct MergeWhenMatchedPolicy { */ void mergeTargetNssSerializeToBSON(const NamespaceString& targetNss, StringData fieldName, - BSONObjBuilder* bob); + BSONObjBuilder* bob, + const SerializationOptions& opts = {}); NamespaceString mergeTargetNssParseFromBSON(const BSONElement& elem); /** @@ -63,7 +65,8 @@ NamespaceString mergeTargetNssParseFromBSON(const BSONElement& elem); */ void mergeOnFieldsSerializeToBSON(const std::vector<std::string>& fields, StringData fieldName, - BSONObjBuilder* bob); + BSONObjBuilder* bob, + const SerializationOptions& opts = {}); std::vector<std::string> mergeOnFieldsParseFromBSON(const BSONElement& elem); /** diff --git a/src/mongo/db/pipeline/document_source_merge_test.cpp b/src/mongo/db/pipeline/document_source_merge_test.cpp index e7d87707900..1987d09da07 100644 --- a/src/mongo/db/pipeline/document_source_merge_test.cpp +++ b/src/mongo/db/pipeline/document_source_merge_test.cpp @@ -29,8 +29,10 @@ #include "mongo/platform/basic.h" +#include "mongo/unittest/bson_test_util.h" #include <boost/intrusive_ptr.hpp> +#include "mongo/bson/unordered_fields_bsonobj_comparator.h" #include "mongo/db/exec/document_value/document.h" #include "mongo/db/exec/document_value/document_value_test_util.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" @@ -792,12 +794,12 @@ TEST_F(DocumentSourceMergeTest, SerializeDefaultLetVariable) { // Test the behaviour of 'let' serialization for each whenNotMatched mode. TEST_F(DocumentSourceMergeTest, SerializeLetVariables) { - auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" - << "$$v1" - << "y" - << "$$v2" - << "z" - << "$$v3"))); + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("_id" << true << "x" + << "$$v1" + << "y" + << "$$v2" + << "z" + << "$$v3"))); const auto createAndSerializeMergeStage = [this, &pipeline](StringData whenNotMatched) { auto spec = BSON("$merge" << BSON("into" @@ -843,8 +845,8 @@ TEST_F(DocumentSourceMergeTest, SerializeLetVariables) { TEST_F(DocumentSourceMergeTest, SerializeLetArrayVariable) { for (auto&& whenNotMatched : {"insert", "fail", "discard"}) { - auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" - << "$$v1"))); + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("_id" << true << "x" + << "$$v1"))); auto spec = BSON( "$merge" << BSON("into" << "target_collection" @@ -872,8 +874,9 @@ TEST_F(DocumentSourceMergeTest, SerializeLetArrayVariable) { // SERVER-41272, this test should be updated to accordingly. TEST_F(DocumentSourceMergeTest, SerializeNullLetVariablesAsDefault) { for (auto&& whenNotMatched : {"insert", "fail", "discard"}) { - auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" - << "1"))); + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("_id" << true << "x" + << BSON("$const" + << "1")))); auto spec = BSON("$merge" << BSON("into" << "target_collection" << "let" << BSONNULL << "whenMatched" << pipeline @@ -890,8 +893,9 @@ TEST_F(DocumentSourceMergeTest, SerializeNullLetVariablesAsDefault) { TEST_F(DocumentSourceMergeTest, SerializeEmptyLetVariables) { for (auto&& whenNotMatched : {"insert", "fail", "discard"}) { - auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" - << "1"))); + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("_id" << true << "x" + << BSON("$const" + << "1")))); auto spec = BSON("$merge" << BSON("into" << "target_collection" << "let" << BSONObj() << "whenMatched" << pipeline @@ -907,6 +911,41 @@ TEST_F(DocumentSourceMergeTest, SerializeEmptyLetVariables) { } } +TEST_F(DocumentSourceMergeTest, SerializeEmptyLetVariableMentionNew) { + auto pipeline = BSON_ARRAY(fromjson("{$project: {_id: true, x: '$$new'}}")); + auto spec = + BSON("$merge" << BSON("into" + << "target_collection" + << "let" << BSONObj() << "whenMatched" << pipeline << "whenNotMatched" + << "insert")); + auto mergeStage = createMergeStage(spec); + ASSERT(mergeStage); + auto serialized = mergeStage->serialize().getDocument(); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$merge": { + "into": { + "db": "unittests", + "coll": "target_collection" + }, + "on": "_id", + "let": { + "new": "$$ROOT" + }, + "whenMatched": [ + { + "$project": { + "_id": true, + "x": "$$new" + } + } + ], + "whenNotMatched": "insert" + } + })", + serialized.toBson()); +} + TEST_F(DocumentSourceMergeTest, OnlyObjectCanBeUsedAsLetVariables) { for (auto&& whenNotMatched : {"insert", "fail", "discard"}) { auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" @@ -950,5 +989,49 @@ TEST_F(DocumentSourceMergeTest, FailsToParseIfOnFieldHaveDuplicates) { ASSERT_THROWS_CODE(createMergeStage(spec), AssertionException, 31465); } +TEST_F(DocumentSourceMergeTest, QueryShape) { + auto pipeline = BSON_ARRAY(BSON("$project" << BSON("x" + << "1"))); + auto let = BSON("new" + << "$$ROOT" + << "year" + << "2020"); + auto spec = + BSON("$merge" << BSON("into" + << "target_collection" + << "let" << let << "whenMatched" << pipeline << "whenNotMatched" + << "insert")); + auto mergeStage = createMergeStage(spec); + ASSERT(mergeStage); + auto serialized = mergeStage->serialize().getDocument(); + + auto expectedBson = fromjson(R"({ + "$merge": { + "into": { + "db": "HASH<unittests>", + "coll": "HASH<target_collection>" + }, + "on": "HASH<_id>", + "let": { + "HASH<year>": "?string", + "HASH<new>": "$$ROOT" + }, + "whenMatched": [ + { + "$project": { + "HASH<_id>": true, + "HASH<x>": "?string" + } + } + ], + "whenNotMatched": "insert" + } + })"); + auto result = redact(*mergeStage); + UnorderedFieldsBSONObjComparator comparator; + ASSERT_EQ(0, comparator.compare(redact(*mergeStage), expectedBson)) + << "Expected [" << expectedBson << "] but found [" << result << "]"; +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_mock.h b/src/mongo/db/pipeline/document_source_mock.h index 39b3e17a265..67f3aad8649 100644 --- a/src/mongo/db/pipeline/document_source_mock.h +++ b/src/mongo/db/pipeline/document_source_mock.h @@ -76,7 +76,7 @@ public: DocumentSourceMock(std::deque<GetNextResult>, const boost::intrusive_ptr<ExpressionContext>&); Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override { + const SerializationOptions& opts = SerializationOptions{}) const final override { // Unlike the queue, it's okay to serialize this stage for testing purposes. return Value(Document{{getSourceName(), Document()}}); } diff --git a/src/mongo/db/pipeline/document_source_operation_metrics.cpp b/src/mongo/db/pipeline/document_source_operation_metrics.cpp index a5e523465e4..7a9dd328f9f 100644 --- a/src/mongo/db/pipeline/document_source_operation_metrics.cpp +++ b/src/mongo/db/pipeline/document_source_operation_metrics.cpp @@ -113,8 +113,7 @@ intrusive_ptr<DocumentSource> DocumentSourceOperationMetrics::createFromBson( return new DocumentSourceOperationMetrics(pExpCtx, clearMetrics); } -Value DocumentSourceOperationMetrics::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceOperationMetrics::serialize(const SerializationOptions& opts) const { return Value(DOC(getSourceName() << Document())); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_operation_metrics.h b/src/mongo/db/pipeline/document_source_operation_metrics.h index d4e25d47bc5..95ac5585b42 100644 --- a/src/mongo/db/pipeline/document_source_operation_metrics.h +++ b/src/mongo/db/pipeline/document_source_operation_metrics.h @@ -89,7 +89,7 @@ public: return boost::none; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; static boost::intrusive_ptr<DocumentSource> createFromBson( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); diff --git a/src/mongo/db/pipeline/document_source_out.cpp b/src/mongo/db/pipeline/document_source_out.cpp index 9b826fb3c89..a6188dfd382 100644 --- a/src/mongo/db/pipeline/document_source_out.cpp +++ b/src/mongo/db/pipeline/document_source_out.cpp @@ -210,8 +210,11 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceOut::createFromBson( return create(targetNS, expCtx); } -Value DocumentSourceOut::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(kStageName << DOC("db" << _outputNs.db() << "coll" << _outputNs.coll()))); +Value DocumentSourceOut::serialize(const SerializationOptions& opts) const { + MutableDocument spec; + spec["coll"] = Value(opts.serializeIdentifier(_outputNs.coll())); + spec["db"] = Value(opts.serializeIdentifier(_outputNs.db())); + return Value(Document{{kStageName, spec.freezeToValue()}}); } void DocumentSourceOut::waitWhileFailPointEnabled() { diff --git a/src/mongo/db/pipeline/document_source_out.h b/src/mongo/db/pipeline/document_source_out.h index 0be5153cdf0..4b2e043dd22 100644 --- a/src/mongo/db/pipeline/document_source_out.h +++ b/src/mongo/db/pipeline/document_source_out.h @@ -91,8 +91,7 @@ public: UnionRequirement::kNotAllowed}; } - Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Creates a new $out stage from the given arguments. diff --git a/src/mongo/db/pipeline/document_source_out_test.cpp b/src/mongo/db/pipeline/document_source_out_test.cpp index aef9be321f0..1338addb727 100644 --- a/src/mongo/db/pipeline/document_source_out_test.cpp +++ b/src/mongo/db/pipeline/document_source_out_test.cpp @@ -114,5 +114,24 @@ TEST_F(DocumentSourceOutTest, SerializeToString) { ASSERT_EQ(reSerialized["$out"]["coll"].getStringData(), "some_collection"); } +TEST_F(DocumentSourceOutTest, Redaction) { + auto spec = fromjson(R"({ + $out: { + db: "foo", + coll: "bar" + } + })"); + auto docSource = DocumentSourceOut::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + $out: { + coll: "HASH<bar>", + db: "HASH<foo>" + } + })", + redact(*docSource)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_plan_cache_stats.cpp b/src/mongo/db/pipeline/document_source_plan_cache_stats.cpp index 51f7bf3d650..f49a3cba01f 100644 --- a/src/mongo/db/pipeline/document_source_plan_cache_stats.cpp +++ b/src/mongo/db/pipeline/document_source_plan_cache_stats.cpp @@ -57,9 +57,13 @@ DocumentSourcePlanCacheStats::DocumentSourcePlanCacheStats( const boost::intrusive_ptr<ExpressionContext>& expCtx) : DocumentSource(kStageName, expCtx) {} -void DocumentSourcePlanCacheStats::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +void DocumentSourcePlanCacheStats::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { + if (opts.verbosity) { + tassert(7513100, + "$planCacheStats is not equipped to serialize in explain mode with redaction on", + !opts.transformIdentifiers && + opts.literalPolicy == LiteralSerializationPolicy::kUnchanged); array.push_back(Value{ Document{{kStageName, Document{{"match"_sd, @@ -67,7 +71,7 @@ void DocumentSourcePlanCacheStats::serializeToArray( } else { array.push_back(Value{Document{{kStageName, Document{}}}}); if (_absorbedMatch) { - _absorbedMatch->serializeToArray(array); + _absorbedMatch->serializeToArray(array, opts); } } } diff --git a/src/mongo/db/pipeline/document_source_plan_cache_stats.h b/src/mongo/db/pipeline/document_source_plan_cache_stats.h index d72b0a22ac8..e1db4cf732c 100644 --- a/src/mongo/db/pipeline/document_source_plan_cache_stats.h +++ b/src/mongo/db/pipeline/document_source_plan_cache_stats.h @@ -117,7 +117,7 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override; + const SerializationOptions& opts = SerializationOptions{}) const final override; private: DocumentSourcePlanCacheStats(const boost::intrusive_ptr<ExpressionContext>& expCtx); @@ -125,8 +125,8 @@ private: GetNextResult doGetNext() final; Value serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const override { - MONGO_UNREACHABLE; // Should call serializeToArray instead. + const SerializationOptions& opts = SerializationOptions{}) const final override { + MONGO_UNREACHABLE_TASSERT(7484303); // Should call serializeToArray instead. } // If running through mongos in a sharded cluster, stores the shard name so that it can be diff --git a/src/mongo/db/pipeline/document_source_plan_cache_stats_test.cpp b/src/mongo/db/pipeline/document_source_plan_cache_stats_test.cpp index 4f89e47acdf..262806e0780 100644 --- a/src/mongo/db/pipeline/document_source_plan_cache_stats_test.cpp +++ b/src/mongo/db/pipeline/document_source_plan_cache_stats_test.cpp @@ -41,6 +41,8 @@ namespace mongo { using DocumentSourcePlanCacheStatsTest = AggregationContextFixture; +static const SerializationOptions kExplain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; /** * A MongoProcessInterface used for testing which returns artificial plan cache stats. */ @@ -106,7 +108,9 @@ TEST_F(DocumentSourcePlanCacheStatsTest, CanParseAndSerializeAsExplainSuccessful const auto specObj = fromjson("{$planCacheStats: {}}"); auto stage = DocumentSourcePlanCacheStats::createFromBson(specObj.firstElement(), getExpCtx()); std::vector<Value> serialized; - stage->serializeToArray(serialized, ExplainOptions::Verbosity::kQueryPlanner); + stage->serializeToArray( + serialized, + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}); ASSERT_EQ(1u, serialized.size()); ASSERT_BSONOBJ_EQ(specObj, serialized[0].getDocument().toBson()); } @@ -139,12 +143,31 @@ TEST_F(DocumentSourcePlanCacheStatsTest, SerializesSuccessfullyAfterAbsorbingMat pipeline->optimizePipeline(); ASSERT_EQ(1u, pipeline->getSources().size()); - auto serialized = pipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto serialized = pipeline->writeExplainOps(kExplain); ASSERT_EQ(1u, serialized.size()); ASSERT_BSONOBJ_EQ(fromjson("{$planCacheStats: {match: {foo: 'bar'}}}"), serialized[0].getDocument().toBson()); } +TEST_F(DocumentSourcePlanCacheStatsTest, RedactsSuccessfullyAfterAbsorbingMatch) { + const auto specObj = fromjson("{$planCacheStats: {}}"); + auto planCacheStats = + DocumentSourcePlanCacheStats::createFromBson(specObj.firstElement(), getExpCtx()); + auto match = DocumentSourceMatch::create(fromjson("{foo: 'bar'}"), getExpCtx()); + auto pipeline = Pipeline::create({planCacheStats, match}, getExpCtx()); + ASSERT_EQ(2u, pipeline->getSources().size()); + + pipeline->optimizePipeline(); + ASSERT_EQ(1u, pipeline->getSources().size()); + auto serialized = redactToArray(*pipeline->getSources().front()); + ASSERT_EQ(2u, serialized.size()); + + ASSERT_BSONOBJ_EQ(specObj, serialized[0].getDocument().toBson()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$match":{"HASH<foo>":{"$eq":"?string"}}})", + serialized[1].getDocument().toBson().getOwned()); +} + TEST_F(DocumentSourcePlanCacheStatsTest, ReturnsImmediateEOFWithEmptyPlanCache) { getExpCtx()->mongoProcessInterface = std::make_shared<PlanCacheStatsMongoProcessInterface>(std::vector<BSONObj>{}); diff --git a/src/mongo/db/pipeline/document_source_project_test.cpp b/src/mongo/db/pipeline/document_source_project_test.cpp index ced99ce0200..bd25639ee3e 100644 --- a/src/mongo/db/pipeline/document_source_project_test.cpp +++ b/src/mongo/db/pipeline/document_source_project_test.cpp @@ -338,6 +338,46 @@ TEST_F(ProjectStageTest, CannotAddNestedDocumentExceedingDepthLimit) { ErrorCodes::Overflow); } +/** + * A default redaction strategy that generates easy to check results for testing purposes. + */ +std::string transformIdentifiersForTest(StringData s) { + return str::stream() << "HASH<" << s << ">"; +} + +TEST_F(ProjectStageTest, ShapifyAndRedact) { + auto inclusionProject = DocumentSourceProject::create( + fromjson("{a: true, x: '$b', y: {$and: ['$c','$d']}, z: {$meta: 'textScore'}}"), + getExpCtx(), + "$project"_sd); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$project": { + "HASH<_id>": true, + "HASH<a>": true, + "HASH<x>": "$HASH<b>", + "HASH<y>": { + "$and": [ "$HASH<c>", "$HASH<d>" ] + }, + "HASH<z>": { "$meta": "textScore" } + } + })", + redact(*inclusionProject)); + + auto exclusionProject = DocumentSourceProject::create( + fromjson("{a: false, 'b.c': false}"), getExpCtx(), "$project"_sd); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$project": { + "HASH<a>": false, + "HASH<b>": { + "HASH<c>": false }, + "HASH<_id>": true } + })", + redact(*exclusionProject)); +} + TEST_F(UnsetTest, AcceptsValidUnsetSpecWithArray) { auto spec = BSON("$unset" << BSON_ARRAY("a" << "b" diff --git a/src/mongo/db/pipeline/document_source_query_stats.cpp b/src/mongo/db/pipeline/document_source_query_stats.cpp new file mode 100644 index 00000000000..69170760ace --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats.cpp @@ -0,0 +1,301 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/pipeline/document_source_query_stats.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQueryStats + +#include "mongo/idl/idl_parser.h" +#include "mongo/logv2/log.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/debug_util.h" + +namespace mongo { +namespace { +Counter64 queryStatsHmacApplicationErrors; +ServerStatusMetricField<Counter64> displayWriteErrorsMetric("queryStats.numHmacApplicationErrors", + &queryStatsHmacApplicationErrors); +} // namespace + +REGISTER_DOCUMENT_SOURCE(queryStats, + DocumentSourceQueryStats::LiteParsed::parse, + DocumentSourceQueryStats::createFromBson, + AllowedWithApiStrict::kNeverInVersion1); + +namespace { + +/** + * Parse the spec object calling the `ctor` with the TransformAlgorithm enum algorithm and + * std::string hmacKey arguments. + */ +template <typename Ctor> +auto parseSpec(const BSONElement& spec, const Ctor& ctor) { + uassert(ErrorCodes::FailedToParse, + str::stream() << DocumentSourceQueryStats::kStageName + << " value must be an object. Found: " << typeName(spec.type()), + spec.type() == BSONType::Object); + BSONObj obj = spec.embeddedObject(); + TransformAlgorithmEnum algorithm = TransformAlgorithmEnum::kNone; + std::string hmacKey; + auto parsed = DocumentSourceQueryStatsSpec::parse(DocumentSourceQueryStats::kStageName, obj); + boost::optional<TransformIdentifiersSpec> transformIdentifiers = + parsed.getTransformIdentifiers(); + + if (transformIdentifiers) { + algorithm = transformIdentifiers->getAlgorithm(); + boost::optional<ConstDataRange> hmacKeyContainer = transformIdentifiers->getHmacKey(); + uassert(ErrorCodes::FailedToParse, + str::stream() << "The 'hmacKey' parameter of the $queryStats stage must be " + "specified when applying the hmac-sha-256 algorithm", + algorithm != TransformAlgorithmEnum::kHmacSha256 || + hmacKeyContainer != boost::none); + hmacKey = std::string(hmacKeyContainer->data(), (size_t)hmacKeyContainer->length()); + } + return ctor(algorithm, hmacKey); +} + +} // namespace + +BSONObj DocumentSourceQueryStats::computeQueryStatsKey(std::shared_ptr<const Key> key) const { + static const auto sha256HmacStringDataHasher = [](std::string key, const StringData& sd) { + auto hashed = SHA256Block::computeHmac( + (const uint8_t*)key.data(), key.size(), (const uint8_t*)sd.rawData(), sd.size()); + return hashed.toString(); + }; + + auto opts = SerializationOptions{}; + opts.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + if (_algorithm == TransformAlgorithmEnum::kHmacSha256) { + opts.transformIdentifiers = true; + opts.transformIdentifiersCallback = [&](StringData sd) { + return sha256HmacStringDataHasher(_hmacKey, sd); + }; + } + return key->toBson(pExpCtx->opCtx, opts); +} + +std::unique_ptr<DocumentSourceQueryStats::LiteParsed> DocumentSourceQueryStats::LiteParsed::parse( + const NamespaceString& nss, const BSONElement& spec) { + return parseSpec(spec, [&](TransformAlgorithmEnum algorithm, std::string hmacKey) { + return std::make_unique<DocumentSourceQueryStats::LiteParsed>( + spec.fieldName(), algorithm, hmacKey); + }); +} + +boost::intrusive_ptr<DocumentSource> DocumentSourceQueryStats::createFromBson( + BSONElement spec, const boost::intrusive_ptr<ExpressionContext>& pExpCtx) { + const NamespaceString& nss = pExpCtx->ns; + + uassert(ErrorCodes::InvalidNamespace, + "$queryStats must be run against the 'admin' database with {aggregate: 1}", + nss.db() == NamespaceString::kAdminDb && nss.isCollectionlessAggregateNS()); + + LOGV2_DEBUG_OPTIONS(7808300, + 1, + {logv2::LogTruncation::Disabled}, + "Logging invocation $queryStats", + "commandSpec"_attr = + spec.Obj().redact(BSONObj::RedactLevel::sensitiveOnly)); + return parseSpec(spec, [&](TransformAlgorithmEnum algorithm, std::string hmacKey) { + return new DocumentSourceQueryStats(pExpCtx, algorithm, hmacKey); + }); +} + +Value DocumentSourceQueryStats::serialize(const SerializationOptions& opts) const { + auto hmacKey = opts.serializeLiteral( + BSONBinData(_hmacKey.c_str(), _hmacKey.size(), BinDataType::Sensitive)); + if (opts.literalPolicy == LiteralSerializationPolicy::kToRepresentativeParseableValue) { + // The default shape for a BinData under this policy is empty and has sub-type 0 (general). + // This doesn't quite work for us since we assert when we parse that it is at least 32 bytes + // and also is sub-type 8 (sensitive). + hmacKey = + Value(BSONBinData("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 32, BinDataType::Sensitive)); + } + return Value{ + Document{{kStageName, + _transformIdentifiers + ? Document{{"transformIdentifiers", + Document{{"algorithm", TransformAlgorithm_serializer(_algorithm)}, + {"hmacKey", hmacKey}}}} + : Document{}}}}; +} + +DocumentSource::GetNextResult DocumentSourceQueryStats::doGetNext() { + const auto shouldLog = _algorithm != TransformAlgorithmEnum::kNone; + /** + * When a CopiedPartition is present (loaded) and contains more elements (QueryStatsEntry), we + * can process and return the next element in the _currentCopiedPartition. + * + * When the current CopiedPartition is exhausted (emptied), we move on to the next + * partition. Once we have iterated to the end of the valid partitions, we are done iteratiing + * over all the queryStatsStore entries. + * + * We iterate over a copied container (CopiedParitition) containing the entries in + * the partition to reduce the time under which the partition lock is held. + */ + auto& queryStatsStore = getQueryStatsStore(getContext()->opCtx); + + while (_currentCopiedPartition.isValidPartitionId(queryStatsStore.numPartitions())) { + if (!_currentCopiedPartition.isLoaded()) { + _currentCopiedPartition.load(queryStatsStore); + } + // CopiedPartition::load() will throw if any errors occur. + // Safe to assume _currentCopiedPartition is now loaded. + + // Exhaust all elements in the current copied partition. + // Use a while loop here to handle cases where toDocument() may fail for a specific + // QueryStatsEntry, in which case we suppress the thrown exception and continue + // iterating to the next available entry. + while (!_currentCopiedPartition.empty()) { + auto& statsEntries = _currentCopiedPartition.statsEntries; + const auto& queryStatsEntry = statsEntries.front(); + ON_BLOCK_EXIT([&statsEntries]() { statsEntries.pop_front(); }); + if (auto doc = + toDocument(_currentCopiedPartition.getReadTimestamp(), queryStatsEntry)) { + if (shouldLog) { + LOGV2_DEBUG_OPTIONS(7808301, + 3, + {logv2::LogTruncation::Disabled}, + "Logging all outputs of $queryStats", + "thisOutput"_attr = *doc); + } + return std::move(*doc); + } + } + // Once we have exhausted entries in this partition, move on to the next partition. + _currentCopiedPartition.incrementPartitionId(); + } + + if (shouldLog) { + LOGV2_DEBUG_OPTIONS( + 7808302, 3, {logv2::LogTruncation::Disabled}, "Finished logging output of $queryStats"); + } + return DocumentSource::GetNextResult::makeEOF(); +} + +boost::optional<Document> DocumentSourceQueryStats::toDocument( + const Date_t& partitionReadTime, const QueryStatsEntry& queryStatsEntry) const { + const auto& key = queryStatsEntry.key; + try { + auto queryStatsKey = computeQueryStatsKey(key); + // We use the representative shape to generate the key hash. This avoids returning duplicate + // hashes if we have bugs that cause two different representative shapes to re-parse into + // the same debug shape. + auto representativeShapeKey = key->toBson( + pExpCtx->opCtx, SerializationOptions::kRepresentativeQueryShapeSerializeOptions); + + // This SHA256 version of the hash is output to aid in data analytics use cases. In these + // cases, we often care about comparing hashes from different hosts, potentially on + // different versions and platforms. The thinking here is that the SHA256 algorithm is more + // stable across these different environments than the quicker 'absl::HashOf' + // implementation. + auto hash = SHA256Block::computeHash((const uint8_t*)representativeShapeKey.objdata(), + representativeShapeKey.objsize()) + .toString(); + return Document{{"key", std::move(queryStatsKey)}, + {"keyHash", hash}, + {"metrics", queryStatsEntry.toBSON()}, + {"asOf", partitionReadTime}}; + } catch (const DBException& ex) { + queryStatsHmacApplicationErrors.increment(); + const auto& hash = absl::Hash<query_stats::Key>{}(*key); + const auto queryShape = key->universalComponents()._queryShape->toBson( + pExpCtx->opCtx, SerializationOptions::kRepresentativeQueryShapeSerializeOptions); + LOGV2_DEBUG(7349403, + 2, + "Error encountered when applying hmac to query shape, will not publish " + "queryStats for this entry.", + "status"_attr = ex.toStatus(), + "hash"_attr = hash, + "debugQueryShape"_attr = queryShape); + + if (kDebugBuild || internalQueryStatsErrorsAreCommandFatal.load()) { + auto keyString = std::to_string(hash); + tasserted(7349401, + str::stream() << "Was not able to re-parse queryStats key when " + "reading queryStats.Status " + << ex.toString() << " Hash: " << keyString + << " Query Shape: " << queryShape.toString()); + } + } + return {}; +} + +/** + * Loads the current CopiedPartition with copies of the QueryStatsEntries located in partition of + * cache corresponding to the partitionId of the current CopiedPartition. This ensures that the + * partition mutex is only held for the duration of copying. + */ +void DocumentSourceQueryStats::CopiedPartition::load(QueryStatsStore& queryStatsStore) { + tassert(7932100, + "Attempted to load invalid partition.", + _partitionId < queryStatsStore.numPartitions()); + tassert(7932101, "Partition was already loaded.", !isLoaded()); + // 'statsEntries' should be empty, clear just in case. + statsEntries.clear(); + + // Capture the time at which reading the partition begins. + _readTimestamp = Date_t::now(); + { + // We only keep the partition (which holds a lock) + // for the time needed to collect the metrics (QueryStatsEntry) + const auto partition = queryStatsStore.getPartition(_partitionId); + + // Note the intentional copy of QueryStatsEntry. + // This will give us a snapshot of all the metrics we want to report. + for (auto&& [hash, metrics] : *partition) { + statsEntries.push_back(metrics); + } + } + _isLoaded = true; +} + +bool DocumentSourceQueryStats::CopiedPartition::isLoaded() const { + return _isLoaded; +} + +void DocumentSourceQueryStats::CopiedPartition::incrementPartitionId() { + // Ensure loaded state is reset when partitionId is incremented. + ++_partitionId; + _isLoaded = false; +} + +bool DocumentSourceQueryStats::CopiedPartition::isValidPartitionId( + QueryStatsStore::PartitionId maxNumPartitions) const { + return _partitionId < maxNumPartitions; +} + +const Date_t& DocumentSourceQueryStats::CopiedPartition::getReadTimestamp() const { + return _readTimestamp; +} + +bool DocumentSourceQueryStats::CopiedPartition::empty() const { + return statsEntries.empty(); +} +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_query_stats.h b/src/mongo/db/pipeline/document_source_query_stats.h new file mode 100644 index 00000000000..bc28a443d52 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats.h @@ -0,0 +1,191 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/pipeline/document_source.h" +#include "mongo/db/pipeline/document_source_query_stats_gen.h" +#include "mongo/db/pipeline/lite_parsed_document_source.h" +#include "mongo/db/query/query_stats/query_stats.h" +#include "mongo/util/producer_consumer_queue.h" + +namespace mongo { + +using namespace query_stats; + +class DocumentSourceQueryStats final : public DocumentSource { +public: + static constexpr StringData kStageName = "$queryStats"_sd; + + class LiteParsed final : public LiteParsedDocumentSource { + public: + static std::unique_ptr<LiteParsed> parse(const NamespaceString& nss, + const BSONElement& spec); + + LiteParsed(std::string parseTimeName, TransformAlgorithmEnum algorithm, std::string hmacKey) + : LiteParsedDocumentSource(std::move(parseTimeName)), + _algorithm(algorithm), + _hmacKey(hmacKey) {} + + stdx::unordered_set<NamespaceString> getInvolvedNamespaces() const override { + return stdx::unordered_set<NamespaceString>(); + } + + PrivilegeVector requiredPrivileges(bool isMongos, + bool bypassDocumentValidation) const override { + return _algorithm == TransformAlgorithmEnum::kNone + + ? PrivilegeVector{Privilege(ResourcePattern::forClusterResource(), + ActionType::queryStatsReadTransformed), + Privilege(ResourcePattern::forClusterResource(), + ActionType::queryStatsRead)} + : PrivilegeVector{Privilege(ResourcePattern::forClusterResource(), + ActionType::queryStatsReadTransformed)}; + } + + bool allowedToPassthroughFromMongos() const final { + // $queryStats must be run locally on a mongod. + return false; + } + + bool isInitialSource() const final { + return true; + } + + void assertSupportsMultiDocumentTransaction() const { + transactionNotSupported(kStageName); + } + + bool _transformIdentifiers; + + const TransformAlgorithmEnum _algorithm; + + std::string _hmacKey; + }; + + static boost::intrusive_ptr<DocumentSource> createFromBson( + BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& pExpCtx); + + virtual ~DocumentSourceQueryStats() = default; + + StageConstraints constraints( + Pipeline::SplitState = Pipeline::SplitState::kUnsplit) const override { + StageConstraints constraints{StreamType::kStreaming, + PositionRequirement::kFirst, + HostTypeRequirement::kLocalOnly, + DiskUseRequirement::kNoDiskUse, + FacetRequirement::kNotAllowed, + TransactionRequirement::kNotAllowed, + LookupRequirement::kNotAllowed, + UnionRequirement::kNotAllowed}; + + constraints.requiresInputDocSource = false; + constraints.isIndependentOfAnyCollection = true; + return constraints; + } + + boost::optional<DistributedPlanLogic> distributedPlanLogic() final { + return boost::none; + } + + const char* getSourceName() const override { + return kStageName.rawData(); + } + + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + +private: + /* + * CopiedPartition: This struct is representative of a copied ("materialized") partition + * which should be loaded from the QueryStatsStore. It is used to hold a copy of the + * QueryStatsEntries corresponding to the provided partitionId. + * Once a CopiedPartition has been loaded from QueryStatsStore, it provides access to the + * QueryStatsEntries of the partition without requiring holding the lock over the partition in + * the partitioned cache. + */ + struct CopiedPartition { + CopiedPartition(QueryStatsStore::PartitionId partitionId) + : statsEntries(), _readTimestamp(), _partitionId(partitionId) {} + + ~CopiedPartition() = default; + + bool isLoaded() const; + + void incrementPartitionId(); + + bool isValidPartitionId(QueryStatsStore::PartitionId maxNumPartitions) const; + + const Date_t& getReadTimestamp() const; + + bool empty() const; + + void load(QueryStatsStore& queryStatsStore); + + std::deque<QueryStatsEntry> statsEntries; + + private: + Date_t _readTimestamp; + QueryStatsStore::PartitionId _partitionId; + bool _isLoaded{false}; + }; + + DocumentSourceQueryStats(const boost::intrusive_ptr<ExpressionContext>& expCtx, + TransformAlgorithmEnum algorithm = TransformAlgorithmEnum::kNone, + std::string hmacKey = {}) + : DocumentSource(kStageName, expCtx), + _currentCopiedPartition(0), + _transformIdentifiers(algorithm != TransformAlgorithmEnum::kNone), + _algorithm(algorithm), + _hmacKey(hmacKey) {} + + BSONObj computeQueryStatsKey(std::shared_ptr<const Key> key) const; + + GetNextResult doGetNext() final; + + boost::optional<Document> toDocument(const Date_t& partitionReadTime, + const QueryStatsEntry& queryStatsEntry) const; + + // The current partition copied from query stats store to avoid holding lock during reads. + CopiedPartition _currentCopiedPartition; + + // When true, apply hmac to field names from returned query shapes. + bool _transformIdentifiers; + + // The type of algorithm to use for transform identifiers as an enum, currently only + // kHmacSha256 + // ("hmac-sha-256") is supported. + const TransformAlgorithmEnum _algorithm; + + /** + * Key used for SHA-256 HMAC application on field names. + */ + std::string _hmacKey; +}; + +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_query_stats.idl b/src/mongo/db/pipeline/document_source_query_stats.idl new file mode 100644 index 00000000000..5a4fdb3cda0 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats.idl @@ -0,0 +1,63 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# +global: + cpp_namespace: "mongo" + cpp_includes: + - "mongo/db/pipeline/document_source_query_stats_validators.h" +imports: + - "mongo/idl/basic_types.idl" + - "mongo/db/query/query_stats/transform_algorithm.idl" + +structs: + TransformIdentifiersSpec: + description: "Specification for the transformIdentifiers field of $queryStats." + strict: true + fields: + algorithm: + description: "The algorithm to use to transform the identifiers." + type: TransformAlgorithm + optional: false + validator: + {callback: "validateAlgo"} + hmacKey: + description: > + The hmac key to be used to transform the identifiers. This is considered sensitive + data and should be redacted from data dumps, output I/O, logs, and other persistent + storages. + type: bindata_sensitive + optional: true + validator: + {callback: "validateHmac"} + DocumentSourceQueryStatsSpec: + description: "Specification for a $queryStats stage." + strict: true + fields: + transformIdentifiers: + description: "The algorithm and hmac key to use when transforming identifiers." + type: TransformIdentifiersSpec + optional: true diff --git a/src/mongo/db/pipeline/document_source_query_stats_test.cpp b/src/mongo/db/pipeline/document_source_query_stats_test.cpp new file mode 100644 index 00000000000..8d08c0f2bc2 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats_test.cpp @@ -0,0 +1,170 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/bsontypes.h" +#include "mongo/db/exec/document_value/document.h" +#include "mongo/db/exec/document_value/document_value_test_util.h" +#include "mongo/db/pipeline/aggregation_context_fixture.h" +#include "mongo/db/pipeline/document_source_query_stats.h" +#include "mongo/idl/server_parameter_test_util.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/str.h" + +namespace mongo { +namespace { + +/** + * Subclass AggregationContextFixture to set the ExpressionContext's namespace to 'admin' with + * {aggregate: 1} by default, so that parsing tests other than those which validate the namespace do + * not need to explicitly set it. + */ +class DocumentSourceQueryStatsTest : public AggregationContextFixture { +public: + DocumentSourceQueryStatsTest() + : AggregationContextFixture(NamespaceString::makeCollectionlessAggregateNSS("admin")) {} +}; + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfSpecIsNotObject) { + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( + fromjson("{$queryStats: 1}").firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::FailedToParse); +} + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfNotRunOnAdmin) { + getExpCtx()->ns = NamespaceString::makeCollectionlessAggregateNSS("foo"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( + fromjson("{$queryStats: {}}").firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::InvalidNamespace); +} + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfNotRunWithAggregateOne) { + getExpCtx()->ns = NamespaceString("admin.foo"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( + fromjson("{$queryStats: {}}").firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::InvalidNamespace); +} + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfUnrecognisedParameterSpecified) { + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson( + fromjson("{$queryStats: {foo: true}}").firstElement(), getExpCtx()), + AssertionException, + 40415); +} + +TEST_F(DocumentSourceQueryStatsTest, ParseAndSerialize) { + const auto obj = fromjson("{$queryStats: {}}"); + const auto doc = DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()); + const auto queryStatsOp = static_cast<DocumentSourceQueryStats*>(doc.get()); + const auto expected = Document{{"$queryStats", Document{}}}; + const auto serialized = queryStatsOp->serialize().getDocument(); + ASSERT_DOCUMENT_EQ(expected, serialized); + + // Also make sure that we can parse out own serialization output. + + ASSERT_DOES_NOT_THROW( + DocumentSourceQueryStats::createFromBson(serialized.toBson().firstElement(), getExpCtx())); +} + +TEST_F(DocumentSourceQueryStatsTest, ParseAndSerializeShouldIncludeHmacKey) { + const auto obj = fromjson(R"({ + $queryStats: { + transformIdentifiers: { + algorithm: "hmac-sha-256", + hmacKey: { + $binary: "YW4gYXJiaXRyYXJ5IEhNQUNrZXkgZm9yIHRlc3Rpbmc=", + $type: "08" + } + } + } + })"); + const auto doc = DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()); + const auto queryStatsOp = static_cast<DocumentSourceQueryStats*>(doc.get()); + const auto expected = + Document{{"$queryStats", + Document{{"transformIdentifiers", + Document{{"algorithm", "hmac-sha-256"_sd}, + {"hmacKey", + BSONBinData("an arbitrary HMACkey for testing", + 32, + BinDataType::Sensitive)}}}}}}; + const auto serialized = queryStatsOp->serialize().getDocument(); + ASSERT_DOCUMENT_EQ(serialized, expected); + + // Also make sure that we can parse out own serialization output. + + ASSERT_DOES_NOT_THROW( + DocumentSourceQueryStats::createFromBson(serialized.toBson().firstElement(), getExpCtx())); +} + +TEST_F(DocumentSourceQueryStatsTest, ShouldFailToParseIfAlgorithmIsNotSupported) { + auto obj = fromjson(R"({ + $queryStats: { + transformIdentifiers: { + algorithm: "randomalgo" + } + } + })"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::BadValue); +} + +TEST_F(DocumentSourceQueryStatsTest, + ShouldFailToParseIfTransformIdentifiersSpecifiedButEmptyAlgorithm) { + auto obj = fromjson(R"({ + $queryStats: { + transformIdentifiers: { + algorithm: "" + } + } + })"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()), + AssertionException, + ErrorCodes::BadValue); +} + +TEST_F(DocumentSourceQueryStatsTest, + ShouldFailToParseIfTransformIdentifiersSpecifiedButNoAlgorithm) { + auto obj = fromjson(R"({ + $queryStats: { + transformIdentifiers: { + } + } + })"); + ASSERT_THROWS_CODE(DocumentSourceQueryStats::createFromBson(obj.firstElement(), getExpCtx()), + AssertionException, + 40414); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_query_stats_validators.cpp b/src/mongo/db/pipeline/document_source_query_stats_validators.cpp new file mode 100644 index 00000000000..8a3f234e325 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats_validators.cpp @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2021-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/pipeline/document_source_query_stats_validators.h" +#include "mongo/db/query/query_stats/transform_algorithm_gen.h" +#include "mongo/util/str.h" +#include <set> + +namespace mongo { + +Status validateAlgo(TransformAlgorithmEnum algorithm) { + if (algorithm == TransformAlgorithmEnum::kNone) { + return {ErrorCodes::FailedToParse, + str::stream() << "Algorithm specified but found no valid algorithm"}; + } + return Status::OK(); +} + +Status validateHmac(std::vector<uint8_t> hmacKey) { + if (hmacKey.size() < 32) { + return {ErrorCodes::BadValue, + str::stream() << "hmacKey must be greater than or equal to 32 bytes, found length: " + << hmacKey.size()}; + } + // length check + return Status::OK(); +} +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_query_stats_validators.h b/src/mongo/db/pipeline/document_source_query_stats_validators.h new file mode 100644 index 00000000000..077552da0b3 --- /dev/null +++ b/src/mongo/db/pipeline/document_source_query_stats_validators.h @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/db/query/query_stats/transform_algorithm_gen.h" + +namespace mongo { +/** + * Validate properties of the algorithm field of $queryStats.transformIdentifiers + */ +Status validateAlgo(TransformAlgorithmEnum algorithm); + +/** + * Validate properties of the hmac key field of $queryStats.transformIdentifiers + */ +Status validateHmac(std::vector<uint8_t> hmacKey); +} // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_queue.cpp b/src/mongo/db/pipeline/document_source_queue.cpp index c724db5ae10..a8cf1af797d 100644 --- a/src/mongo/db/pipeline/document_source_queue.cpp +++ b/src/mongo/db/pipeline/document_source_queue.cpp @@ -80,12 +80,13 @@ DocumentSource::GetNextResult DocumentSourceQueue::doGetNext() { return next; } -Value DocumentSourceQueue::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceQueue::serialize(const SerializationOptions& opts) const { ValueArrayStream vals; for (auto elem : _queue) { vals << elem.getDocument().getOwned(); } - return Value(DOC(kStageName << vals.done())); + // We treat the queue's documents as one literal in the context of redaction. + return Value(DOC(kStageName << opts.serializeLiteral(vals.done()))); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_queue.h b/src/mongo/db/pipeline/document_source_queue.h index e7eb6452d06..f170cca27d9 100644 --- a/src/mongo/db/pipeline/document_source_queue.h +++ b/src/mongo/db/pipeline/document_source_queue.h @@ -55,7 +55,7 @@ public: const char* getSourceName() const override; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const override; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const override; StageConstraints constraints(Pipeline::SplitState pipeState) const override { StageConstraints constraints{StreamType::kStreaming, diff --git a/src/mongo/db/pipeline/document_source_queue_test.cpp b/src/mongo/db/pipeline/document_source_queue_test.cpp index b7014bd3166..2833826ddbf 100644 --- a/src/mongo/db/pipeline/document_source_queue_test.cpp +++ b/src/mongo/db/pipeline/document_source_queue_test.cpp @@ -73,10 +73,19 @@ TEST_F(QueueStageTest, QueueStageSerialize) { ASSERT_TRUE(queueStage); - auto res = queueStage->serialize(boost::none); + auto res = queueStage->serialize(); ASSERT_VALUE_EQ(res, Value{DOC("$queue" << DOC_ARRAY(DOC("a1" << 1) << DOC("a2" << 2)))}); } +TEST_F(QueueStageTest, RedactsCorrectly) { + auto queueDoc = BSON("$queue" << BSON_ARRAY(BSON("a" << 1))); + auto queueStage = DocumentSourceQueue::createFromBson(queueDoc.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$queue":"?array<?object>"})", + redact(*queueStage)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_redact.cpp b/src/mongo/db/pipeline/document_source_redact.cpp index 80dc406c620..c14651b55c7 100644 --- a/src/mongo/db/pipeline/document_source_redact.cpp +++ b/src/mongo/db/pipeline/document_source_redact.cpp @@ -143,8 +143,11 @@ boost::optional<Document> DocumentSourceRedact::redactObject(const Document& roo return boost::optional<Document>(); } else if (simpleValueCmp.evaluate(expressionResult == descendVal)) { const Document in = variables.getDocument(_currentId, root); + in.loadIntoCache(); + MutableDocument out; out.copyMetaDataFrom(in); + FieldIterator fields(in); while (fields.more()) { const Document::FieldPair field(fields.next()); @@ -169,8 +172,8 @@ intrusive_ptr<DocumentSource> DocumentSourceRedact::optimize() { return this; } -Value DocumentSourceRedact::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << _expression.get()->serialize(static_cast<bool>(explain)))); +Value DocumentSourceRedact::serialize(const SerializationOptions& opts) const { + return Value(DOC(getSourceName() << _expression.get()->serialize(opts))); } intrusive_ptr<DocumentSource> DocumentSourceRedact::createFromBson( diff --git a/src/mongo/db/pipeline/document_source_redact.h b/src/mongo/db/pipeline/document_source_redact.h index b566f2c7adf..84f1f262f5b 100644 --- a/src/mongo/db/pipeline/document_source_redact.h +++ b/src/mongo/db/pipeline/document_source_redact.h @@ -66,7 +66,7 @@ public: static boost::intrusive_ptr<DocumentSource> createFromBson( BSONElement elem, const boost::intrusive_ptr<ExpressionContext>& expCtx); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<Expression> getExpression() { return _expression; diff --git a/src/mongo/db/pipeline/document_source_replace_root.h b/src/mongo/db/pipeline/document_source_replace_root.h index f288aa10321..a796cb069f4 100644 --- a/src/mongo/db/pipeline/document_source_replace_root.h +++ b/src/mongo/db/pipeline/document_source_replace_root.h @@ -60,9 +60,9 @@ public: _newRoot->optimize(); } - Document serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const final { - return Document{{"newRoot", _newRoot->serialize(static_cast<bool>(explain))}}; + Document serializeTransformation(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options = {}) const final { + return Document{{"newRoot", _newRoot->serialize(options)}}; } DepsTracker::State addDependencies(DepsTracker* deps) const final { diff --git a/src/mongo/db/pipeline/document_source_sample.cpp b/src/mongo/db/pipeline/document_source_sample.cpp index 24e86b38cba..40d5f47e3ad 100644 --- a/src/mongo/db/pipeline/document_source_sample.cpp +++ b/src/mongo/db/pipeline/document_source_sample.cpp @@ -83,8 +83,8 @@ DocumentSource::GetNextResult DocumentSourceSample::doGetNext() { return _sortStage->getNext(); } -Value DocumentSourceSample::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(kStageName << DOC("size" << _size))); +Value DocumentSourceSample::serialize(const SerializationOptions& opts) const { + return Value(DOC(kStageName << DOC("size" << opts.serializeLiteral(_size)))); } namespace { diff --git a/src/mongo/db/pipeline/document_source_sample.h b/src/mongo/db/pipeline/document_source_sample.h index 7dd31db3c41..577043c3ec0 100644 --- a/src/mongo/db/pipeline/document_source_sample.h +++ b/src/mongo/db/pipeline/document_source_sample.h @@ -41,7 +41,7 @@ public: const char* getSourceName() const final { return kStageName.rawData(); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; StageConstraints constraints(Pipeline::SplitState pipeState) const final { return {StreamType::kBlocking, diff --git a/src/mongo/db/pipeline/document_source_sample_from_random_cursor.cpp b/src/mongo/db/pipeline/document_source_sample_from_random_cursor.cpp index 1105966e3ac..08716b4fac5 100644 --- a/src/mongo/db/pipeline/document_source_sample_from_random_cursor.cpp +++ b/src/mongo/db/pipeline/document_source_sample_from_random_cursor.cpp @@ -143,9 +143,8 @@ DocumentSource::GetNextResult DocumentSourceSampleFromRandomCursor::getNextNonDu "sporadic failure, please try again."); } -Value DocumentSourceSampleFromRandomCursor::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << DOC("size" << _size))); +Value DocumentSourceSampleFromRandomCursor::serialize(const SerializationOptions& opts) const { + return Value(DOC(getSourceName() << DOC("size" << opts.serializeLiteral(_size)))); } DepsTracker::State DocumentSourceSampleFromRandomCursor::getDependencies(DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/document_source_sample_from_random_cursor.h b/src/mongo/db/pipeline/document_source_sample_from_random_cursor.h index 65521488158..4e7a73f0445 100644 --- a/src/mongo/db/pipeline/document_source_sample_from_random_cursor.h +++ b/src/mongo/db/pipeline/document_source_sample_from_random_cursor.h @@ -42,7 +42,7 @@ class DocumentSourceSampleFromRandomCursor final : public DocumentSource { public: static constexpr StringData kStageName = "$sampleFromRandomCursor"_sd; const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DepsTracker::State getDependencies(DepsTracker* deps) const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final { diff --git a/src/mongo/db/pipeline/document_source_sample_test.cpp b/src/mongo/db/pipeline/document_source_sample_test.cpp index 68cd93c4698..ade80752010 100644 --- a/src/mongo/db/pipeline/document_source_sample_test.cpp +++ b/src/mongo/db/pipeline/document_source_sample_test.cpp @@ -188,6 +188,17 @@ TEST_F(SampleBasics, ShouldPropagatePauses) { assertEOF(); } +TEST_F(SampleBasics, RedactsCorrectly) { + createSample(10); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$sample": { + "size": "?number" + } + })", + redact(*sample())); +} + /** * Fixture to test error cases of the $sample stage. */ @@ -402,5 +413,12 @@ DEATH_TEST_REGEX_F(SampleFromRandomCursorBasics, sample()->getNext(); } +TEST_F(SampleFromRandomCursorBasics, RedactsCorrectly) { + createSample(2); + ASSERT_VALUE_EQ_AUTO( // NOLINT + "{ $sampleFromRandomCursor: { size: \"?number\" } }", + redact(*sample())); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp b/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp index 39a13272df6..535c3f5d2be 100644 --- a/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp +++ b/src/mongo/db/pipeline/document_source_sequential_document_cache.cpp @@ -80,9 +80,10 @@ DocumentSource::GetNextResult DocumentSourceSequentialDocumentCache::doGetNext() Pipeline::SourceContainer::iterator DocumentSourceSequentialDocumentCache::doOptimizeAt( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { - // The DocumentSourceSequentialDocumentCache should always be the last stage in the pipeline - // pre-optimization. By the time optimization reaches this point, all preceding stages are in - // the final positions which they would have occupied if no cache stage was present. + // The DocumentSourceSequentialDocumentCache relies on all other stages in the pipeline being at + // the final positions which they would have occupied if no cache stage was present. This should + // be the case when we reach this function. The cache should always be the last stage in the + // pipeline pre-optimizing. invariant(_hasOptimizedPos || std::next(itr) == container->end()); invariant((*itr).get() == this); @@ -152,12 +153,12 @@ Pipeline::SourceContainer::iterator DocumentSourceSequentialDocumentCache::doOpt return container->end(); } -Value DocumentSourceSequentialDocumentCache::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - if (explain) { +Value DocumentSourceSequentialDocumentCache::serialize(const SerializationOptions& opts) const { + if (opts.verbosity) { return Value(Document{ {kStageName, - Document{{"maxSizeBytes"_sd, Value(static_cast<long long>(_cache->maxSizeBytes()))}, + Document{{"maxSizeBytes"_sd, + opts.serializeLiteral(static_cast<long long>(_cache->maxSizeBytes()))}, {"status"_sd, _cache->isBuilding() ? "kBuilding"_sd diff --git a/src/mongo/db/pipeline/document_source_sequential_document_cache.h b/src/mongo/db/pipeline/document_source_sequential_document_cache.h index 4e81390d4ef..ea6c99a5f9e 100644 --- a/src/mongo/db/pipeline/document_source_sequential_document_cache.h +++ b/src/mongo/db/pipeline/document_source_sequential_document_cache.h @@ -97,6 +97,10 @@ public: return newStage; } + bool hasOptimizedPos() const { + return _hasOptimizedPos; + } + protected: GetNextResult doGetNext() final; Pipeline::SourceContainer::iterator doOptimizeAt(Pipeline::SourceContainer::iterator itr, @@ -106,7 +110,7 @@ private: DocumentSourceSequentialDocumentCache(const boost::intrusive_ptr<ExpressionContext>& expCtx, SequentialDocumentCache* cache); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; SequentialDocumentCache* _cache; diff --git a/src/mongo/db/pipeline/document_source_sequential_document_cache_test.cpp b/src/mongo/db/pipeline/document_source_sequential_document_cache_test.cpp index d4480ca7c57..36b0f043faf 100644 --- a/src/mongo/db/pipeline/document_source_sequential_document_cache_test.cpp +++ b/src/mongo/db/pipeline/document_source_sequential_document_cache_test.cpp @@ -34,9 +34,11 @@ #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/json.h" +#include "mongo/db/exec/document_value/document_value_test_util.h" #include "mongo/db/pipeline/aggregation_context_fixture.h" #include "mongo/db/pipeline/document_source_mock.h" #include "mongo/db/pipeline/document_source_sequential_document_cache.h" +#include "mongo/unittest/bson_test_util.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -74,5 +76,27 @@ TEST_F(DocumentSourceSequentialDocumentCacheTest, ReturnsEOFAfterCacheExhausted) ASSERT(documentCache->getNext().isEOF()); ASSERT(documentCache->getNext().isEOF()); } + +TEST_F(DocumentSourceSequentialDocumentCacheTest, Redaction) { + SequentialDocumentCache cache(kDefaultMaxCacheSize); + cache.add(DOC("_id" << 0)); + cache.add(DOC("_id" << 1)); + auto documentCache = DocumentSourceSequentialDocumentCache::create(getExpCtx(), &cache); + std::vector<Value> vals; + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$sequentialCache":{"maxSizeBytes":"?number","status":"kBuilding"}})", + redact(*documentCache, true, ExplainOptions::Verbosity::kQueryPlanner)); + + cache.freeze(); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$sequentialCache":{"maxSizeBytes":"?number","status":"kServing"}})", + redact(*documentCache, true, ExplainOptions::Verbosity::kQueryPlanner)); + + cache.abandon(); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$sequentialCache":{"maxSizeBytes":"?number","status":"kAbandoned"}})", + redact(*documentCache, true, ExplainOptions::Verbosity::kQueryPlanner)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.cpp b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.cpp index 3d8b330cec3..d9c5455f271 100644 --- a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.cpp +++ b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.cpp @@ -54,13 +54,12 @@ REGISTER_INTERNAL_DOCUMENT_SOURCE( // to check the FCV. feature_flags::gFeatureFlagSearchShardedFacets.isEnabledAndIgnoreFCV()); -Value DocumentSourceSetVariableFromSubPipeline::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceSetVariableFromSubPipeline::serialize(const SerializationOptions& opts) const { const auto var = "$$" + Variables::getBuiltinVariableName(_variableID); SetVariableFromSubPipelineSpec spec; tassert(625298, "SubPipeline cannot be null during serialization", _subPipeline); - spec.setSetVariable(var); - spec.setPipeline(_subPipeline->serializeToBson(explain)); + spec.setSetVariable(opts.serializeIdentifier(var)); + spec.setPipeline(_subPipeline->serializeToBson(opts)); return Value(DOC(getSourceName() << spec.toBSON())); } diff --git a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.h b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.h index 7d366d9a912..e06cc9a2333 100644 --- a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.h +++ b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline.h @@ -111,7 +111,7 @@ protected: private: GetNextResult doGetNext() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; std::unique_ptr<Pipeline, PipelineDeleter> _subPipeline; Variables::Id _variableID; // $setVariableFromSubPipeline sets the value of $$SEARCH_META only on the first call to diff --git a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline_test.cpp b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline_test.cpp index e7cc8d71a27..8b30d3c95b7 100644 --- a/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline_test.cpp +++ b/src/mongo/db/pipeline/document_source_set_variable_from_subpipeline_test.cpp @@ -158,6 +158,41 @@ TEST_F(DocumentSourceSetVariableFromSubPipelineTest, testDoGetNext) { Value((BSON("d" << 1))), nullptr) == 0); } +TEST_F(DocumentSourceSetVariableFromSubPipelineTest, QueryShape) { + const auto inputDocs = + std::vector{Document{{"a", 1}}, Document{{"b", 1}}, Document{{"c", 1}}, Document{{"d", 1}}}; + auto expCtx = getExpCtx(); + const auto mockSourceForSetVarStage = DocumentSourceMock::createForTest(inputDocs[1], expCtx); + auto ctxForSubPipeline = expCtx->copyForSubPipeline(expCtx->ns); + const auto mockSourceForSubPipeline = + DocumentSourceMock::createForTest(inputDocs, ctxForSubPipeline); + auto setVariableFromSubPipeline = DocumentSourceSetVariableFromSubPipeline::create( + expCtx, + Pipeline::create({DocumentSourceMatch::create(BSON("d" << 1), ctxForSubPipeline)}, + ctxForSubPipeline), + Variables::kSearchMetaId); + setVariableFromSubPipeline->addSubPipelineInitialSource(mockSourceForSubPipeline); + setVariableFromSubPipeline->setSource(mockSourceForSetVarStage.get()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$setVariableFromSubPipeline": { + "setVariable": "HASH<$$SEARCH_META>", + "pipeline": [ + { + "mock": {} + }, + { + "$match": { + "HASH<d>": { + "$eq": "?number" + } + } + } + ] + } + })", + redact(*setVariableFromSubPipeline)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_set_window_fields.cpp b/src/mongo/db/pipeline/document_source_set_window_fields.cpp index 73806992561..e32fc78c04b 100644 --- a/src/mongo/db/pipeline/document_source_set_window_fields.cpp +++ b/src/mongo/db/pipeline/document_source_set_window_fields.cpp @@ -144,8 +144,8 @@ WindowFunctionStatement WindowFunctionStatement::parse(BSONElement elem, window_function::Expression::parse(elem.embeddedObject(), sortBy, expCtx)); } void WindowFunctionStatement::serialize(MutableDocument& outputFields, - boost::optional<ExplainOptions::Verbosity> explain) const { - outputFields[fieldName] = expr->serialize(explain); + const SerializationOptions& opts) const { + outputFields[opts.serializeFieldPathFromString(fieldName)] = expr->serialize(opts); } list<intrusive_ptr<DocumentSource>> document_source_set_window_fields::create( @@ -218,12 +218,10 @@ list<intrusive_ptr<DocumentSource>> document_source_set_window_fields::create( } else { // In DocumentSource we don't have a mechanism for generating non-colliding field names, // so we have to choose the tmp name carefully to make a collision unlikely in practice. - std::array<unsigned char, 16> nonce = UUID::gen().data(); - // We encode as a base64 string for a shorter, more performant field name (length 22). - std::string tmpField = base64::encode(nonce.data(), sizeof(nonce)); - simplePartitionBy = FieldPath{tmpField}; + auto tmp = "__internal_setWindowFields_partition_key"; + simplePartitionBy = FieldPath{tmp}; simplePartitionByExpr = ExpressionFieldPath::createPathFromString( - expCtx.get(), tmpField, expCtx->variablesParseState); + expCtx.get(), tmp, expCtx->variablesParseState); complexPartitionBy = partitionBy; } } @@ -288,39 +286,38 @@ intrusive_ptr<DocumentSource> DocumentSourceInternalSetWindowFields::optimize() return this; } -Value DocumentSourceInternalSetWindowFields::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceInternalSetWindowFields::serialize(const SerializationOptions& opts) const { MutableDocument spec; spec[SetWindowFieldsSpec::kPartitionByFieldName] = - _partitionBy ? (*_partitionBy)->serialize(false) : Value(); + _partitionBy ? (*_partitionBy)->serialize(opts) : Value(); - auto sortKeySerialization = explain + auto sortKeySerialization = opts.verbosity ? SortPattern::SortKeySerialization::kForExplain : SortPattern::SortKeySerialization::kForPipelineSerialization; spec[SetWindowFieldsSpec::kSortByFieldName] = - _sortBy ? Value(_sortBy->serialize(sortKeySerialization)) : Value(); + _sortBy ? Value(_sortBy->serialize(sortKeySerialization, opts)) : Value(); MutableDocument output; for (auto&& stmt : _outputFields) { - stmt.serialize(output, explain); + stmt.serialize(output, opts); } spec[SetWindowFieldsSpec::kOutputFieldName] = output.freezeToValue(); MutableDocument out; out[getSourceName()] = Value(spec.freeze()); - if (explain && *explain >= ExplainOptions::Verbosity::kExecStats) { + if (opts.verbosity && *opts.verbosity >= ExplainOptions::Verbosity::kExecStats) { MutableDocument md; for (auto&& [fieldName, function] : _executableOutputs) { - md[fieldName] = - Value(static_cast<long long>(_memoryTracker[fieldName].maxMemoryBytes())); + md[opts.serializeFieldPathFromString(fieldName)] = opts.serializeLiteral( + static_cast<long long>(_memoryTracker[fieldName].maxMemoryBytes())); } out["maxFunctionMemoryUsageBytes"] = Value(md.freezeToValue()); out["maxTotalMemoryUsageBytes"] = - Value(static_cast<long long>(_memoryTracker.maxMemoryBytes())); - out["usedDisk"] = Value(_iterator.usedDisk()); + opts.serializeLiteral(static_cast<long long>(_memoryTracker.maxMemoryBytes())); + out["usedDisk"] = opts.serializeLiteral(_iterator.usedDisk()); } return Value(out.freezeToValue()); @@ -477,13 +474,15 @@ DocumentSource::GetNextResult DocumentSourceInternalSetWindowFields::doGetNext() // Populate the output document with the result from each window function. auto projSpec = std::make_unique<projection_executor::InclusionNode>( ProjectionPolicies{ProjectionPolicies::DefaultIdPolicy::kIncludeId}); - for (auto&& [fieldName, function] : _executableOutputs) { + for (auto&& outputField : _outputFields) { try { // If we hit a uassert while evaluating expressions on user data, delete the temporary // table before aborting the operation. + auto& fieldName = outputField.fieldName; projSpec->addExpressionForPath( FieldPath(fieldName), - ExpressionConstant::create(pExpCtx.get(), function->getNext())); + ExpressionConstant::create(pExpCtx.get(), + _executableOutputs[fieldName]->getNext())); } catch (const DBException&) { _iterator.finalize(); throw; diff --git a/src/mongo/db/pipeline/document_source_set_window_fields.h b/src/mongo/db/pipeline/document_source_set_window_fields.h index f0f9b0742ce..43af58e2dad 100644 --- a/src/mongo/db/pipeline/document_source_set_window_fields.h +++ b/src/mongo/db/pipeline/document_source_set_window_fields.h @@ -70,8 +70,7 @@ struct WindowFunctionStatement { } } - void serialize(MutableDocument& outputFields, - boost::optional<ExplainOptions::Verbosity> explain) const; + void serialize(MutableDocument& outputFields, const SerializationOptions& opts) const; }; /** @@ -167,7 +166,7 @@ public: boost::intrusive_ptr<DocumentSource> optimize() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DocumentSource::GetNextResult doGetNext(); diff --git a/src/mongo/db/pipeline/document_source_set_window_fields_test.cpp b/src/mongo/db/pipeline/document_source_set_window_fields_test.cpp index 287497db445..2f35d4a6b94 100644 --- a/src/mongo/db/pipeline/document_source_set_window_fields_test.cpp +++ b/src/mongo/db/pipeline/document_source_set_window_fields_test.cpp @@ -212,5 +212,342 @@ TEST_F(DocumentSourceSetWindowFieldsTest, ReportsModifiedFields) { ASSERT_EQUALS(modified.paths.count("b"), 1U); ASSERT_TRUE(modified.renames.empty()); } + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnShiftOperator) { + auto spec = fromjson( + R"({ + $setWindowFields: { + partitionBy: '$foo', + sortBy: { + bar: 1 + }, + output: { + x: { + $shift: { + output: '$y', + by: 1, + default: 'BAZ' + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": "$HASH<foo>", + "sortBy": { + "HASH<bar>": 1 + }, + "output": { + "HASH<x>": { + "$shift": { + "by": "?number", + "output": "$HASH<y>", + "default": "?string" + } + } + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnExpMovingAvgOperator) { + auto spec = fromjson( + R"({ + $setWindowFields: { + partitionBy: '$foo.bar', + sortBy: { + bar: 1 + }, + output: { + x: { + $expMovingAvg: { + alpha: 0.5, + input: '$y' + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": "$HASH<foo>.HASH<bar>", + "sortBy": { + "HASH<bar>": 1 + }, + "output": { + "HASH<x>": { + "$expMovingAvg": { + "alpha": "?number", + "input": "$HASH<y>" + } + } + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnExpressionNOperator) { + auto spec = fromjson( + R"({ + $setWindowFields: { + partitionBy: '$a', + output: { + b: { + $minN: { + n: 3, + input: '$y' + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": "$HASH<a>", + "output": { + "HASH<b>": { + "$minN": { + "n": "?number", + "input": "$HASH<y>" + }, + "window": { + "documents": [ + "unbounded", + "unbounded" + ] + } + } + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnSumWithDocumentsWindow) { + auto spec = fromjson(R"({ + $setWindowFields: { + partitionBy: { + $year: '$x' + }, + sortBy: { + a: 1, + b: -1 + }, + output: { + cumulative: { + $sum: '$baz', + window: { + documents: [ + 'unbounded', + 'current' + ] + } + }, + maximum: { + $max: '$baz', + window: { + documents: [ + 'unbounded', + 'unbounded' + ] + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": { + "$year": { + "date": "$HASH<x>" + } + }, + "sortBy": { + "HASH<a>": 1, + "HASH<b>": -1 + }, + "output": { + "HASH<cumulative>": { + "$sum": "$HASH<baz>", + "window": { + "documents": [ + "unbounded", + "current" + ] + } + }, + "HASH<maximum>": { + "$max": "$HASH<baz>", + "window": { + "documents": [ + "unbounded", + "unbounded" + ] + } + } + } + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RedactionOnPushWithRangeWindowWithUnit) { + auto spec = fromjson(R"({ + $setWindowFields: { + partitionBy: '$foo', + sortBy: { + bar: 1 + }, + output: { + a: { + $push: '$b', + window: { + range: [ + 'unbounded', + -10 + ], + unit: 'month' + } + } + } + } + })"); + auto docSource = + DocumentSourceInternalSetWindowFields::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalSetWindowFields": { + "partitionBy": "$HASH<foo>", + "sortBy": { + "HASH<bar>": 1 + }, + "output": { + "HASH<a>": { + "$push": "$HASH<b>", + "window": { + "range": [ + "unbounded", + "?number" + ], + "unit": "month" + } + } + } + } + })", + redact(*docSource)); +} + +/** + * Helper function that parses the $setWindowFields aggregation stage from the input, serializes it + * to its representative shape, re-parses the representative shape, and compares to the original. + */ +void assertRepresentativeShapeIsStable(const boost::intrusive_ptr<ExpressionContext>& expCtx, + BSONObj inputStage, + BSONObj expectedRepresentativeStage) { + auto parsedStage = + DocumentSourceInternalSetWindowFields::createFromBson(inputStage.firstElement(), expCtx); + std::vector<Value> serialization; + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + parsedStage->serializeToArray(serialization, opts); + + auto serializedStage = serialization[0].getDocument().toBson(); + ASSERT_BSONOBJ_EQ(serializedStage, expectedRepresentativeStage); + + auto roundTripped = DocumentSourceInternalSetWindowFields::createFromBson( + serializedStage.firstElement(), expCtx); + + std::vector<Value> newSerialization; + roundTripped->serializeToArray(newSerialization, opts); + ASSERT_EQ(newSerialization.size(), 1UL); + ASSERT_VALUE_EQ(newSerialization[0], serialization[0]); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RoundTripSerializationDocumentWindowBounds) { + assertRepresentativeShapeIsStable(getExpCtx(), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {documents: [-10, 10]}}}}})"), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {documents: [0, 1]}}}}})")); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RoundTripSerializationRangeWindowBounds) { + assertRepresentativeShapeIsStable(getExpCtx(), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {range: [-10, 10]}}}}})"), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {range: [0, 1]}}}}})")); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RoundTripSerializationRangeWindowBoundsWithUnit) { + assertRepresentativeShapeIsStable(getExpCtx(), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {range: [-10, 10], unit: 'second'}}}}})"), + fromjson(R"( + {$_internalSetWindowFields: {partitionBy: '$state', sortBy: {city: 1}, output: {mySum: + {$sum: '$pop', window: {range: [0, 1], unit: 'second'}}}}})")); +} + +TEST_F(DocumentSourceSetWindowFieldsTest, RoundTripSerializationExpMovingAvg) { + assertRepresentativeShapeIsStable(getExpCtx(), + fromjson( + R"({ + $setWindowFields: { + partitionBy: '$foo.bar', + sortBy: { + bar: 1 + }, + output: { + x: { + $expMovingAvg: { + alpha: 0.5, + input: '$y' + } + } + } + } + })"), + fromjson( + R"({ + $_internalSetWindowFields: { + partitionBy: '$foo.bar', + sortBy: { + bar: 1 + }, + output: { + x: { + $expMovingAvg: { + alpha: 0.1, + input: '$y' + } + } + } + } + })")); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_single_document_transformation.cpp b/src/mongo/db/pipeline/document_source_single_document_transformation.cpp index 3e6e8373b56..33820b0ceaf 100644 --- a/src/mongo/db/pipeline/document_source_single_document_transformation.cpp +++ b/src/mongo/db/pipeline/document_source_single_document_transformation.cpp @@ -89,10 +89,11 @@ void DocumentSourceSingleDocumentTransformation::doDispose() { } Value DocumentSourceSingleDocumentTransformation::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(Document{{getSourceName(), - _parsedTransform ? _parsedTransform->serializeTransformation(explain) - : _cachedStageOptions}}); + const SerializationOptions& opts) const { + return Value( + Document{{getSourceName(), + _parsedTransform ? _parsedTransform->serializeTransformation(opts.verbosity, opts) + : _cachedStageOptions}}); } Pipeline::SourceContainer::iterator DocumentSourceSingleDocumentTransformation::doOptimizeAt( diff --git a/src/mongo/db/pipeline/document_source_single_document_transformation.h b/src/mongo/db/pipeline/document_source_single_document_transformation.h index eebe9adad22..b3cc1538bfe 100644 --- a/src/mongo/db/pipeline/document_source_single_document_transformation.h +++ b/src/mongo/db/pipeline/document_source_single_document_transformation.h @@ -62,7 +62,7 @@ public: const char* getSourceName() const final; boost::intrusive_ptr<DocumentSource> optimize() final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; DepsTracker::State getDependencies(DepsTracker* deps) const final; GetModPathsReturn getModifiedPaths() const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final { diff --git a/src/mongo/db/pipeline/document_source_skip.cpp b/src/mongo/db/pipeline/document_source_skip.cpp index df7537fd1dd..1a9ea0cd6c9 100644 --- a/src/mongo/db/pipeline/document_source_skip.cpp +++ b/src/mongo/db/pipeline/document_source_skip.cpp @@ -72,8 +72,8 @@ DocumentSource::GetNextResult DocumentSourceSkip::doGetNext() { return pSource->getNext(); } -Value DocumentSourceSkip::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << _nToSkip)); +Value DocumentSourceSkip::serialize(const SerializationOptions& opts) const { + return Value(DOC(getSourceName() << opts.serializeLiteral(_nToSkip))); } intrusive_ptr<DocumentSource> DocumentSourceSkip::optimize() { diff --git a/src/mongo/db/pipeline/document_source_skip.h b/src/mongo/db/pipeline/document_source_skip.h index 89dbeb95b11..667ab7cd85b 100644 --- a/src/mongo/db/pipeline/document_source_skip.h +++ b/src/mongo/db/pipeline/document_source_skip.h @@ -73,7 +73,7 @@ public: Pipeline::SourceContainer::iterator doOptimizeAt(Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<DocumentSource> optimize() final; diff --git a/src/mongo/db/pipeline/document_source_skip_test.cpp b/src/mongo/db/pipeline/document_source_skip_test.cpp index 1de0df02324..7e54cbdde7c 100644 --- a/src/mongo/db/pipeline/document_source_skip_test.cpp +++ b/src/mongo/db/pipeline/document_source_skip_test.cpp @@ -103,5 +103,12 @@ TEST_F(DocumentSourceSkipTest, SkipsChainedTogetherShouldNotOverFlowWhenOptimizi ASSERT_EQUALS(containerOptimized.size(), 1U); ASSERT_EQUALS(skipFirst->getSkip(), 2); } + +TEST_F(DocumentSourceSkipTest, Redaction) { + auto stage = DocumentSourceSkip::create(getExpCtx(), 1337); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"$skip":"?number"})", + redact(*stage)); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_sort.cpp b/src/mongo/db/pipeline/document_source_sort.cpp index 84b28d6fde9..235f98039df 100644 --- a/src/mongo/db/pipeline/document_source_sort.cpp +++ b/src/mongo/db/pipeline/document_source_sort.cpp @@ -44,6 +44,7 @@ #include "mongo/db/pipeline/lite_parsed_document_source.h" #include "mongo/db/pipeline/skip_and_limit.h" #include "mongo/db/query/collation/collation_index_key.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/db/stats/resource_consumption_metrics.h" #include "mongo/logv2/log.h" #include "mongo/platform/overflow_arithmetic.h" @@ -68,10 +69,10 @@ struct BoundMakerMin { doc.metadata().getTimeseriesBucketMinTime().toMillisSinceEpoch() + offset)}; } - Document serialize() const { + Document serialize(const SerializationOptions& opts) const { // Convert from millis to seconds. return Document{{{"base"_sd, DocumentSourceSort::kMin}, - {DocumentSourceSort::kOffset, (offset / 1000)}}}; + {DocumentSourceSort::kOffset, opts.serializeLiteral(offset / 1000)}}}; } }; @@ -84,10 +85,10 @@ struct BoundMakerMax { doc.metadata().getTimeseriesBucketMaxTime().toMillisSinceEpoch() + offset)}; } - Document serialize() const { + Document serialize(const SerializationOptions& opts) const { // Convert from millis to seconds. return Document{{{"base"_sd, DocumentSourceSort::kMax}, - {DocumentSourceSort::kOffset, (offset / 1000)}}}; + {DocumentSourceSort::kOffset, opts.serializeLiteral(offset / 1000)}}}; } }; struct CompAsc { @@ -286,8 +287,10 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceSort::clone( _sortExecutor->getMaxMemoryBytes()); } -void DocumentSourceSort::serializeToArray( - std::vector<Value>& array, boost::optional<ExplainOptions::Verbosity> explain) const { +void DocumentSourceSort::serializeToArray(std::vector<Value>& array, + const SerializationOptions& opts) const { + auto explain = opts.verbosity; + if (_timeSorter) { tassert(6369900, "$_internalBoundedSort should not absorb a $limit", @@ -295,20 +298,22 @@ void DocumentSourceSort::serializeToArray( // {$_internalBoundedSort: {sortKey, bound}} auto sortKey = _sortExecutor->sortPattern().serialize( - SortPattern::SortKeySerialization::kForPipelineSerialization); + SortPattern::SortKeySerialization::kForPipelineSerialization, opts); MutableDocument mutDoc{Document{{ {"$_internalBoundedSort"_sd, Document{{{"sortKey"_sd, std::move(sortKey)}, - {"bound"_sd, _timeSorter->serializeBound()}, - {"limit"_sd, static_cast<long long>(_timeSorter->limit())}}}}, + {"bound"_sd, _timeSorter->serializeBound(opts)}, + {"limit"_sd, + opts.serializeLiteral(static_cast<long long>(_timeSorter->limit()))}}}}, }}}; if (explain >= ExplainOptions::Verbosity::kExecStats) { mutDoc["totalDataSizeSortedBytesEstimate"] = - Value(static_cast<long long>(_timeSorter->totalDataSizeBytes())); - mutDoc["usedDisk"] = Value(_timeSorter->stats().spilledRanges() > 0); - mutDoc["spills"] = Value(static_cast<long long>(_timeSorter->stats().spilledRanges())); + opts.serializeLiteral(static_cast<long long>(_timeSorter->totalDataSizeBytes())); + mutDoc["usedDisk"] = opts.serializeLiteral(_timeSorter->stats().spilledRanges() > 0); + mutDoc["spills"] = + opts.serializeLiteral(static_cast<long long>(_timeSorter->stats().spilledRanges())); } array.push_back(Value{mutDoc.freeze()}); @@ -319,31 +324,31 @@ void DocumentSourceSort::serializeToArray( if (!explain) { // one Value for $sort and maybe a Value for $limit MutableDocument inner(_sortExecutor->sortPattern().serialize( - SortPattern::SortKeySerialization::kForPipelineSerialization)); + SortPattern::SortKeySerialization::kForPipelineSerialization, opts)); array.push_back(Value(DOC(kStageName << inner.freeze()))); if (_sortExecutor->hasLimit()) { auto limitSrc = DocumentSourceLimit::create(pExpCtx, limit); - limitSrc->serializeToArray(array); + limitSrc->serializeToArray(array, opts); } return; } - MutableDocument mutDoc( - DOC(kStageName << DOC("sortKey" - << _sortExecutor->sortPattern().serialize( - SortPattern::SortKeySerialization::kForExplain) - << "limit" - << (_sortExecutor->hasLimit() ? Value(static_cast<long long>(limit)) - : Value())))); + MutableDocument mutDoc(DOC( + kStageName << DOC("sortKey" << _sortExecutor->sortPattern().serialize( + SortPattern::SortKeySerialization::kForExplain, opts) + << "limit" + << (_sortExecutor->hasLimit() + ? opts.serializeLiteral(static_cast<long long>(limit)) + : Value())))); if (explain >= ExplainOptions::Verbosity::kExecStats) { auto& stats = _sortExecutor->stats(); mutDoc["totalDataSizeSortedBytesEstimate"] = - Value(static_cast<long long>(stats.totalDataSizeBytes)); - mutDoc["usedDisk"] = Value(stats.spills > 0); - mutDoc["spills"] = Value(static_cast<long long>(stats.spills)); + opts.serializeLiteral(static_cast<long long>(stats.totalDataSizeBytes)); + mutDoc["usedDisk"] = opts.serializeLiteral(stats.spills > 0); + mutDoc["spills"] = opts.serializeLiteral(static_cast<long long>(stats.spills)); } array.push_back(Value(mutDoc.freeze())); diff --git a/src/mongo/db/pipeline/document_source_sort.h b/src/mongo/db/pipeline/document_source_sort.h index ba087ee4a3f..d28e29a77ea 100644 --- a/src/mongo/db/pipeline/document_source_sort.h +++ b/src/mongo/db/pipeline/document_source_sort.h @@ -72,7 +72,7 @@ public: void serializeToArray( std::vector<Value>& array, - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + const SerializationOptions& opts = SerializationOptions{}) const final override; boost::intrusive_ptr<DocumentSource> clone( const boost::intrusive_ptr<ExpressionContext>& newExpCtx) const final; @@ -201,8 +201,8 @@ private: uint64_t limit, uint64_t maxMemoryUsageBytes); - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final { - MONGO_UNREACHABLE; // Should call serializeToArray instead. + Value serialize(const SerializationOptions& opts) const final override { + MONGO_UNREACHABLE_TASSERT(7484302); // Should call serializeToArray instead. } /** diff --git a/src/mongo/db/pipeline/document_source_sort_by_count_test.cpp b/src/mongo/db/pipeline/document_source_sort_by_count_test.cpp index d4ab7f15c55..df92e9b5e07 100644 --- a/src/mongo/db/pipeline/document_source_sort_by_count_test.cpp +++ b/src/mongo/db/pipeline/document_source_sort_by_count_test.cpp @@ -70,7 +70,8 @@ public: // Serialize the DocumentSourceGroup and DocumentSourceSort from $sortByCount so that we can // check the explain output to make sure $group and $sort have the correct fields. - const auto explain = ExplainOptions::Verbosity::kQueryPlanner; + const auto explain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; vector<Value> explainedStages; groupStage->serializeToArray(explainedStages, explain); sortStage->serializeToArray(explainedStages, explain); diff --git a/src/mongo/db/pipeline/document_source_sort_test.cpp b/src/mongo/db/pipeline/document_source_sort_test.cpp index 013ef9b6a53..31c6c9323a5 100644 --- a/src/mongo/db/pipeline/document_source_sort_test.cpp +++ b/src/mongo/db/pipeline/document_source_sort_test.cpp @@ -27,6 +27,7 @@ * it in the license file. */ +#include "mongo/db/query/explain_options.h" #include "mongo/platform/basic.h" #include <boost/intrusive_ptr.hpp> @@ -538,5 +539,79 @@ TEST_F(DocumentSourceSortExecutionTest, ShouldCorrectlyTrackMemoryUsageBetweenPa sort->getNext(), AssertionException, ErrorCodes::QueryExceededMemoryLimitNoDiskUseAllowed); } +TEST_F(DocumentSourceSortTest, Redaction) { + createSort(BSON("a" << 1)); + auto boundedSort = DocumentSourceSort::createBoundedSort( + sort()->getSortKeyPattern(), DocumentSourceSort::kMin, 1337, 10, getExpCtx()); + + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"$sort":{"HASH<a>":1}})", + redact(*sort(), true)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalBoundedSort": { + "sortKey": { + "HASH<a>": 1 + }, + "bound": { + "base": "min", + "offsetSeconds": "?number" + }, + "limit": "?number" + } + })", + redact(*boundedSort, true)); + + ASSERT_BSONOBJ_EQ_AUTO( // + R"({"$sort":{"sortKey":{"HASH<a>":1}}})", + redact(*sort(), true, ExplainOptions::Verbosity::kQueryPlanner)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalBoundedSort": { + "sortKey": { + "HASH<a>": 1 + }, + "bound": { + "base": "min", + "offsetSeconds": "?number" + }, + "limit": "?number" + } + })", + redact(*boundedSort, true, ExplainOptions::Verbosity::kQueryPlanner)); + + ASSERT_BSONOBJ_EQ_AUTO( // + R"({ + "$sort": { + "sortKey": { + "HASH<a>": 1 + } + }, + "totalDataSizeSortedBytesEstimate": "?number", + "usedDisk": "?bool", + "spills": "?number" + })", + redact(*sort(), true, ExplainOptions::Verbosity::kExecStats)); + + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$_internalBoundedSort": { + "sortKey": { + "HASH<a>": 1 + }, + "bound": { + "base": "min", + "offsetSeconds": "?number" + }, + "limit": "?number" + }, + "totalDataSizeSortedBytesEstimate": "?number", + "usedDisk": "?bool", + "spills": "?number" + })", + redact(*boundedSort, true, ExplainOptions::Verbosity::kExecStats)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_streaming_group.cpp b/src/mongo/db/pipeline/document_source_streaming_group.cpp index 44702ab550d..0aeae62201a 100644 --- a/src/mongo/db/pipeline/document_source_streaming_group.cpp +++ b/src/mongo/db/pipeline/document_source_streaming_group.cpp @@ -153,13 +153,13 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceStreamingGroup::createFromBso } void DocumentSourceStreamingGroup::serializeAdditionalFields( - MutableDocument& out, boost::optional<ExplainOptions::Verbosity> explain) const { + MutableDocument& out, const SerializationOptions& opts) const { std::vector<Value> monotonicIdFields; if (_idFieldNames.empty()) { - monotonicIdFields.emplace_back("_id"_sd); + monotonicIdFields.emplace_back(opts.serializeFieldPath("_id")); } else { for (size_t i : _monotonicExpressionIndexes) { - monotonicIdFields.emplace_back(_idFieldNames[i]); + monotonicIdFields.emplace_back(opts.serializeFieldPathFromString(_idFieldNames[i])); } } out[kMonotonicIdFieldsSpecField] = Value(std::move(monotonicIdFields)); diff --git a/src/mongo/db/pipeline/document_source_streaming_group.h b/src/mongo/db/pipeline/document_source_streaming_group.h index bd72a3a7e7d..e69db4ef026 100644 --- a/src/mongo/db/pipeline/document_source_streaming_group.h +++ b/src/mongo/db/pipeline/document_source_streaming_group.h @@ -90,8 +90,9 @@ protected: GetNextResult doGetNext() final; bool isSpecFieldReserved(StringData fieldName) final; - void serializeAdditionalFields(MutableDocument& out, - boost::optional<ExplainOptions::Verbosity> explain) const final; + void serializeAdditionalFields( + MutableDocument& out, + const SerializationOptions& opts = SerializationOptions{}) const final; private: static constexpr StringData kMonotonicIdFieldsSpecField = "$monotonicIdFields"_sd; diff --git a/src/mongo/db/pipeline/document_source_tee_consumer.cpp b/src/mongo/db/pipeline/document_source_tee_consumer.cpp index 739cec3498c..47539fb491c 100644 --- a/src/mongo/db/pipeline/document_source_tee_consumer.cpp +++ b/src/mongo/db/pipeline/document_source_tee_consumer.cpp @@ -71,9 +71,8 @@ void DocumentSourceTeeConsumer::doDispose() { _bufferSource->dispose(_facetId); } -Value DocumentSourceTeeConsumer::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceTeeConsumer::serialize(const SerializationOptions& opts) const { // We only serialize this stage in the context of explain. - return explain ? Value(DOC(_stageName << Document())) : Value(); + return opts.verbosity ? Value(DOC(_stageName << Document())) : Value(); } } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_tee_consumer.h b/src/mongo/db/pipeline/document_source_tee_consumer.h index da56c76a37b..26a8ee02a2f 100644 --- a/src/mongo/db/pipeline/document_source_tee_consumer.h +++ b/src/mongo/db/pipeline/document_source_tee_consumer.h @@ -79,7 +79,7 @@ public: const char* getSourceName() const override; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; protected: GetNextResult doGetNext() final; diff --git a/src/mongo/db/pipeline/document_source_test_optimizations.h b/src/mongo/db/pipeline/document_source_test_optimizations.h index 2f6e484be10..d7f8f4bec13 100644 --- a/src/mongo/db/pipeline/document_source_test_optimizations.h +++ b/src/mongo/db/pipeline/document_source_test_optimizations.h @@ -71,8 +71,9 @@ public: } private: - virtual Value serialize(boost::optional<ExplainOptions::Verbosity>) const override { - MONGO_UNREACHABLE; + virtual Value serialize( + const SerializationOptions& opts = SerializationOptions{}) const final override { + MONGO_UNREACHABLE_TASSERT(7484301); } }; diff --git a/src/mongo/db/pipeline/document_source_union_with.cpp b/src/mongo/db/pipeline/document_source_union_with.cpp index 2f08fdfc0c6..fdf1f724787 100644 --- a/src/mongo/db/pipeline/document_source_union_with.cpp +++ b/src/mongo/db/pipeline/document_source_union_with.cpp @@ -51,21 +51,21 @@ REGISTER_DOCUMENT_SOURCE(unionWith, AllowedWithApiStrict::kAlways); namespace { +void validatorCallback(const Pipeline& pipeline) { + const auto& sources = pipeline.getSources(); + std::for_each(sources.begin(), sources.end(), [](auto& src) { + uassert(31441, + str::stream() << src->getSourceName() + << " is not allowed within a $unionWith's sub-pipeline", + src->constraints().isAllowedInUnionPipeline()); + }); +} + std::unique_ptr<Pipeline, PipelineDeleter> buildPipelineFromViewDefinition( const boost::intrusive_ptr<ExpressionContext>& expCtx, ExpressionContext::ResolvedNamespace resolvedNs, std::vector<BSONObj> currentPipeline) { - auto validatorCallback = [](const Pipeline& pipeline) { - const auto& sources = pipeline.getSources(); - std::for_each(sources.begin(), sources.end(), [](auto& src) { - uassert(31441, - str::stream() << src->getSourceName() - << " is not allowed within a $unionWith's sub-pipeline", - src->constraints().isAllowedInUnionPipeline()); - }); - }; - MakePipelineOptions opts; opts.attachCursorSource = false; // Only call optimize() here if we actually have a pipeline to resolve in the view definition. @@ -78,6 +78,22 @@ std::unique_ptr<Pipeline, PipelineDeleter> buildPipelineFromViewDefinition( } // namespace +DocumentSourceUnionWith::DocumentSourceUnionWith( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + std::unique_ptr<Pipeline, PipelineDeleter> pipeline) + : DocumentSource(kStageName, expCtx), _pipeline(std::move(pipeline)) {} + +DocumentSourceUnionWith::DocumentSourceUnionWith( + const boost::intrusive_ptr<ExpressionContext>& expCtx, + NamespaceString unionNss, + std::vector<BSONObj> pipeline) + : DocumentSourceUnionWith(expCtx, + buildPipelineFromViewDefinition( + expCtx, expCtx->getResolvedNamespace(unionNss), pipeline)) { + _userNss = std::move(unionNss); + _userPipeline = std::move(pipeline); +} + DocumentSourceUnionWith::~DocumentSourceUnionWith() { if (_pipeline && _pipeline->getContext()->explain) { _pipeline->dispose(pExpCtx->opCtx); @@ -197,9 +213,7 @@ boost::intrusive_ptr<DocumentSource> DocumentSourceUnionWith::createFromBson( pipeline = unionWithSpec.getPipeline().value_or(std::vector<BSONObj>{}); } return make_intrusive<DocumentSourceUnionWith>( - expCtx, - buildPipelineFromViewDefinition( - expCtx, expCtx->getResolvedNamespace(std::move(unionNss)), std::move(pipeline))); + expCtx, std::move(unionNss), std::move(pipeline)); } DocumentSource::GetNextResult DocumentSourceUnionWith::doGetNext() { @@ -278,11 +292,7 @@ Pipeline::SourceContainer::iterator DocumentSourceUnionWith::doOptimizeAt( _pipeline->addFinalSource(nextStage->clone(_pipeline->getContext())); // Apply the same rewrite to the cached pipeline if available. if (pExpCtx->explain >= ExplainOptions::Verbosity::kExecStats) { - auto cloneForExplain = nextStage->clone(_pipeline->getContext()); - if (!_cachedPipeline.empty()) { - cloneForExplain->setSource(_cachedPipeline.back().get()); - } - _cachedPipeline.push_back(std::move(cloneForExplain)); + _pushedDownStages.push_back(nextStage->serialize().getDocument().toBson()); } auto newStageItr = container->insert(itr, std::move(nextStage)); container->erase(std::next(itr)); @@ -315,14 +325,16 @@ void DocumentSourceUnionWith::doDispose() { if (!_pipeline->getContext()->explain) { _pipeline->dispose(pExpCtx->opCtx); + _userPipeline.clear(); + _pushedDownStages.clear(); _pipeline.reset(); } } } -Value DocumentSourceUnionWith::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value DocumentSourceUnionWith::serialize(const SerializationOptions& opts) const { auto collectionless = _pipeline->getContext()->ns.isCollectionlessAggregateNS(); - if (explain) { + if (opts.verbosity) { // There are several different possible states depending on the explain verbosity as well as // the other stages in the pipeline: // * If verbosity is queryPlanner, then the sub-pipeline should be untouched and we can @@ -332,44 +344,65 @@ Value DocumentSourceUnionWith::serialize(boost::optional<ExplainOptions::Verbosi // $limit stage after the $unionWith which results in only reading from the base collection // branch and not the sub-pipeline. Pipeline* pipeCopy = nullptr; - if (*explain == ExplainOptions::Verbosity::kQueryPlanner) { + if (*opts.verbosity == ExplainOptions::Verbosity::kQueryPlanner) { pipeCopy = Pipeline::create(_pipeline->getSources(), _pipeline->getContext()).release(); - } else if (*explain >= ExplainOptions::Verbosity::kExecStats && + } else if (*opts.verbosity >= ExplainOptions::Verbosity::kExecStats && _executionState > ExecutionProgress::kIteratingSource) { + std::vector<BSONObj> recoveredPipeline; // We've either exhausted the sub-pipeline or at least started iterating it. Use the - // cached pipeline to get the explain output since the '_pipeline' may have been - // modified for any optimizations or pushdowns into the initial $cursor stage. - pipeCopy = Pipeline::create(_cachedPipeline, _pipeline->getContext()).release(); + // cached user pipeline and pushed down stages to get the explain output since the + // '_pipeline' may have been modified for any optimizations or pushdowns into the + // initial $cursor stage. + recoveredPipeline.reserve(_userPipeline.size() + _pushedDownStages.size()); + std::move( + _userPipeline.begin(), _userPipeline.end(), std::back_inserter(recoveredPipeline)); + std::move(_pushedDownStages.begin(), + _pushedDownStages.end(), + std::back_inserter(recoveredPipeline)); + pipeCopy = + Pipeline::parse(recoveredPipeline, _pipeline->getContext(), validatorCallback) + .release(); } else { // The plan does not require reading from the sub-pipeline, so just include the // serialization in the explain output. BSONArrayBuilder bab; - for (auto&& stage : _pipeline->serialize(explain)) + for (auto&& stage : _pipeline->serialize(opts)) bab << stage; auto spec = collectionless ? DOC("pipeline" << bab.arr()) - : DOC("coll" << _pipeline->getContext()->ns.coll() << "pipeline" << bab.arr()); + : DOC("coll" << opts.serializeIdentifier(_pipeline->getContext()->ns.coll()) + << "pipeline" << bab.arr()); return Value(DOC(getSourceName() << spec)); } invariant(pipeCopy); BSONObj explainLocal = - pExpCtx->mongoProcessInterface->preparePipelineAndExplain(pipeCopy, *explain); + pExpCtx->mongoProcessInterface->preparePipelineAndExplain(pipeCopy, *opts.verbosity); LOGV2_DEBUG(4553501, 3, "$unionWith attached cursor to pipeline for explain"); // We expect this to be an explanation of a pipeline -- there should only be one field. invariant(explainLocal.nFields() == 1); - auto spec = collectionless ? DOC("pipeline" << explainLocal.firstElement()) - : DOC("coll" << _pipeline->getContext()->ns.coll() << "pipeline" - << explainLocal.firstElement()); + auto spec = collectionless + ? DOC("pipeline" << explainLocal.firstElement()) + : DOC("coll" << opts.serializeIdentifier(_pipeline->getContext()->ns.coll()) + << "pipeline" << explainLocal.firstElement()); return Value(DOC(getSourceName() << spec)); } else { - BSONArrayBuilder bab; - for (auto&& stage : _pipeline->serialize()) - bab << stage; - auto spec = collectionless - ? DOC("pipeline" << bab.arr()) - : DOC("coll" << _pipeline->getContext()->ns.coll() << "pipeline" << bab.arr()); + // Query shapes must reflect the original, unresolved and unoptimized pipeline, so we need a + // special case here if we are serializing the stage for that purpose. Otherwise, we should + // return the current (optimized) pipeline for introspection with explain, etc. + auto serializedPipeline = [&]() -> std::vector<BSONObj> { + if (opts.transformIdentifiers || + opts.literalPolicy != LiteralSerializationPolicy::kUnchanged) { + return Pipeline::parse(_userPipeline, _pipeline->getContext(), validatorCallback) + ->serializeToBson(opts); + } + return _pipeline->serializeToBson(opts); + }(); + + auto spec = collectionless ? DOC("pipeline" << serializedPipeline) + : DOC("coll" << opts.serializeIdentifier(_userNss.coll()) + << "pipeline" << serializedPipeline); return Value(DOC(getSourceName() << spec)); } } diff --git a/src/mongo/db/pipeline/document_source_union_with.h b/src/mongo/db/pipeline/document_source_union_with.h index 741f2e13e68..9d9f9aae9c8 100644 --- a/src/mongo/db/pipeline/document_source_union_with.h +++ b/src/mongo/db/pipeline/document_source_union_with.h @@ -61,18 +61,19 @@ public: }; DocumentSourceUnionWith(const boost::intrusive_ptr<ExpressionContext>& expCtx, - std::unique_ptr<Pipeline, PipelineDeleter> pipeline) - : DocumentSource(kStageName, expCtx), _pipeline(std::move(pipeline)) { - // If this pipeline is being run as part of explain, then cache a copy to use later during - // serialization. - if (expCtx->explain >= ExplainOptions::Verbosity::kExecStats) { - _cachedPipeline = _pipeline->getSources(); - } - } + NamespaceString unionNss, + std::vector<BSONObj> pipeline); + + // Expose a constructor that skips the parsing step for testing purposes. + DocumentSourceUnionWith(const boost::intrusive_ptr<ExpressionContext>& expCtx, + std::unique_ptr<Pipeline, PipelineDeleter> pipeline); DocumentSourceUnionWith(const DocumentSourceUnionWith& original, const boost::intrusive_ptr<ExpressionContext>& newExpCtx) - : DocumentSource(kStageName, newExpCtx), _pipeline(original._pipeline->clone()) {} + : DocumentSource(kStageName, newExpCtx), + _pipeline(original._pipeline->clone()), + _userNss(original._userNss), + _userPipeline(original._userPipeline) {} ~DocumentSourceUnionWith(); @@ -180,7 +181,7 @@ private: kFinished }; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; void addViewDefinition(NamespaceString nss, std::vector<BSONObj> viewPipeline); @@ -189,7 +190,15 @@ private: const ExceptionFor<ErrorCodes::CommandOnShardedViewNotSupportedOnMongod>& e); std::unique_ptr<Pipeline, PipelineDeleter> _pipeline; - Pipeline::SourceContainer _cachedPipeline; + // The original, unresolved namespace to union. + NamespaceString _userNss; + // The aggregation pipeline defined with the user request, prior to optimization and view + // resolution. + std::vector<BSONObj> _userPipeline; + // Match and/or project stages after a $unionWith can be pushed down into the $unionWith (and + // the head of the pipeline). If we're doing an explain with execution stats, we will need + // copies of these stages as they may be pushed down to the find layer. + std::vector<BSONObj> _pushedDownStages; ExecutionProgress _executionState = ExecutionProgress::kIteratingSource; UnionWithStats _stats; }; diff --git a/src/mongo/db/pipeline/document_source_union_with.idl b/src/mongo/db/pipeline/document_source_union_with.idl index 77b3ea64a82..11bbaa499dc 100644 --- a/src/mongo/db/pipeline/document_source_union_with.idl +++ b/src/mongo/db/pipeline/document_source_union_with.idl @@ -31,6 +31,7 @@ global: imports: - "mongo/idl/basic_types.idl" + - "mongo/db/pipeline/aggregate_command.idl" structs: UnionWithSpec: @@ -44,4 +45,4 @@ structs: pipeline: description: An optional pipeline to apply to the collection being unioned. optional: true - type: array<object> + type: pipeline diff --git a/src/mongo/db/pipeline/document_source_union_with_test.cpp b/src/mongo/db/pipeline/document_source_union_with_test.cpp index 2f2b819174d..6dff25c53b1 100644 --- a/src/mongo/db/pipeline/document_source_union_with_test.cpp +++ b/src/mongo/db/pipeline/document_source_union_with_test.cpp @@ -572,5 +572,62 @@ TEST_F(DocumentSourceUnionWithTest, StricterConstraintsFromSubSubPipelineAreInhe StageConstraints::UnionRequirement::kAllowed); ASSERT_TRUE(unionStage.constraints(Pipeline::SplitState::kUnsplit) == expectedConstraints); } + +TEST_F(DocumentSourceUnionWithTest, RedactsCorrectlyBasic) { + auto expCtx = getExpCtx(); + NamespaceString nsToUnionWith(expCtx->ns.db(), "coll"); + expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {nsToUnionWith.coll().toString(), {nsToUnionWith, std::vector<BSONObj>()}}}); + + auto docSource = DocumentSourceUnionWith::createFromBson( + BSON("$unionWith" << nsToUnionWith.coll()).firstElement(), expCtx); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$unionWith": { + "coll": "HASH<coll>", + "pipeline": [] + } + })", + redact(*docSource)); +} + +TEST_F(DocumentSourceUnionWithTest, RedactsCorrectlyWithPipeline) { + auto expCtx = getExpCtx(); + NamespaceString nsToUnionWith(expCtx->ns.db(), "coll"); + expCtx->setResolvedNamespaces(StringMap<ExpressionContext::ResolvedNamespace>{ + {nsToUnionWith.coll().toString(), {nsToUnionWith, std::vector<BSONObj>()}}}); + + BSONArrayBuilder pipeline; + pipeline << BSON("$match" << BSON("a" << 15)); + pipeline << BSON("$project" << BSON("a" << 1 << "b" << 1)); + auto docSource = DocumentSourceUnionWith::createFromBson( + BSON("$unionWith" << BSON("coll" << nsToUnionWith.coll() << "pipeline" << pipeline.arr())) + .firstElement(), + expCtx); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$unionWith": { + "coll": "HASH<coll>", + "pipeline": [ + { + "$match": { + "HASH<a>": { + "$eq": "?number" + } + } + }, + { + "$project": { + "HASH<_id>": true, + "HASH<a>": true, + "HASH<b>": true + } + } + ] + } + })", + redact(*docSource)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/pipeline/document_source_unwind.cpp b/src/mongo/db/pipeline/document_source_unwind.cpp index 2161b0b9bbb..45ba4aed308 100644 --- a/src/mongo/db/pipeline/document_source_unwind.cpp +++ b/src/mongo/db/pipeline/document_source_unwind.cpp @@ -91,6 +91,10 @@ private: // Index into the _inputArray to return next. size_t _index = 0; + + // True if we are including the array index and it's path is a parent of the unwind path. If + // this is true, we will just return the array indices and ignore the array values. + bool _conflictingPaths; }; DocumentSourceUnwind::Unwinder::Unwinder(const FieldPath& unwindPath, @@ -100,7 +104,8 @@ DocumentSourceUnwind::Unwinder::Unwinder(const FieldPath& unwindPath, : _unwindPath(unwindPath), _preserveNullAndEmptyArrays(preserveNullAndEmptyArrays), _indexPath(indexPath), - _strict(strict) {} + _strict(strict), + _conflictingPaths(indexPath ? indexPath->isPrefixOf(unwindPath) : false) {} void DocumentSourceUnwind::Unwinder::resetDocument(const Document& document) { // Reset document specific attributes. @@ -141,8 +146,11 @@ DocumentSource::GetNextResult DocumentSourceUnwind::Unwinder::getNext() { // across documents that have come out of this pipeline operator. This is a partial deep // clone. Because the value at the end will be replaced, everything along the path // leading to that will be replaced in order not to share that change with any other - // clones (or the original). - _output.setNestedField(_unwindPathFieldIndexes, _inputArray[_index]); + // clones (or the original). If the array index path is a parent of the unwind path, we + // ignore the array value since it would be overwritten by the index. + if (!_conflictingPaths) { + _output.setNestedField(_unwindPathFieldIndexes, _inputArray[_index]); + } indexForOutput = _index; _index++; _haveNext = _index < length; @@ -232,19 +240,16 @@ bool DocumentSourceUnwind::canPushSortBack(const DocumentSourceSort* sort) const // otherwise when we swap the limit and unwind, we could end up providing fewer results to the // user than expected. if (!sort->hasLimit() || _preserveNullAndEmptyArrays) { - auto unwindPath = _unwindPath.fullPath(); + auto modifiedPaths = getModifiedPaths(); // Checks if any of the $sort's paths depend on the unwind path (or vice versa). SortPattern sortKeyPattern = sort->getSortKeyPattern(); - bool sortPathMatchesUnwindPath = + bool sortDependsOnUnwind = std::any_of(sortKeyPattern.begin(), sortKeyPattern.end(), [&](auto& sortKey) { // If 'sortKey' is a $meta expression, we can do the swap. - if (!sortKey.fieldPath) - return false; - return expression::bidirectionalPathPrefixOf(unwindPath, - sortKey.fieldPath->fullPath()); + return sortKey.fieldPath && modifiedPaths.canModify(*sortKey.fieldPath); }); - return !sortPathMatchesUnwindPath; + return !sortDependsOnUnwind; } return false; } @@ -293,12 +298,13 @@ Pipeline::SourceContainer::iterator DocumentSourceUnwind::doOptimizeAt( return std::next(itr); } -Value DocumentSourceUnwind::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return Value(DOC(getSourceName() << DOC( - "path" << _unwindPath.fullPathWithPrefix() << "preserveNullAndEmptyArrays" - << (_preserveNullAndEmptyArrays ? Value(true) : Value()) - << "includeArrayIndex" - << (_indexPath ? Value((*_indexPath).fullPath()) : Value())))); +Value DocumentSourceUnwind::serialize(const SerializationOptions& opts) const { + return Value(DOC( + getSourceName() << DOC( + "path" << opts.serializeFieldPathWithPrefix(_unwindPath) << "preserveNullAndEmptyArrays" + << (_preserveNullAndEmptyArrays ? opts.serializeLiteral(true) : Value()) + << "includeArrayIndex" + << (_indexPath ? Value(opts.serializeFieldPath(*_indexPath)) : Value())))); } DepsTracker::State DocumentSourceUnwind::getDependencies(DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/document_source_unwind.h b/src/mongo/db/pipeline/document_source_unwind.h index d7f401dd99c..81a2fb58882 100644 --- a/src/mongo/db/pipeline/document_source_unwind.h +++ b/src/mongo/db/pipeline/document_source_unwind.h @@ -43,7 +43,7 @@ public: // virtuals from DocumentSource const char* getSourceName() const final; - Value serialize(boost::optional<ExplainOptions::Verbosity> explain = boost::none) const final; + Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; /** * Returns the unwound path, and the 'includeArrayIndex' path, if specified. diff --git a/src/mongo/db/pipeline/document_source_unwind_test.cpp b/src/mongo/db/pipeline/document_source_unwind_test.cpp index aa8a550ec06..c8c7b46f0c3 100644 --- a/src/mongo/db/pipeline/document_source_unwind_test.cpp +++ b/src/mongo/db/pipeline/document_source_unwind_test.cpp @@ -734,6 +734,61 @@ TEST_F(UnwindStageTest, UnwindIncludesIndexPathWhenIncludingIndex) { ASSERT_EQUALS(1U, modifiedPaths.paths.count("arrIndex")); } +TEST_F(UnwindStageTest, UnwindIndexPathIsSamePathAsArrayPath) { + const bool includeNullIfEmptyOrMissing = false; + const boost::optional<std::string> includeArrayIndex = std::string("array"); + auto unwind = DocumentSourceUnwind::create( + getExpCtx(), "array", includeNullIfEmptyOrMissing, includeArrayIndex); + auto source = DocumentSourceMock::createForTest( + {Document{{"array", vector<Value>{Value(10), Value(20)}}}, + Document{{"array", vector<Value>{Value(30), Value(40)}}}}, + getExpCtx()); + + unwind->setSource(source.get()); + + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(0)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(1)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(0)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(1)); +} + +TEST_F(UnwindStageTest, UnwindIndexPathIsParentOfArrayPath) { + const bool includeNullIfEmptyOrMissing = false; + const boost::optional<std::string> includeArrayIndex = std::string("obj"); + auto unwind = DocumentSourceUnwind::create( + getExpCtx(), "obj.array", includeNullIfEmptyOrMissing, includeArrayIndex); + auto source = DocumentSourceMock::createForTest( + {Document{{"obj", Document{{"array", vector<Value>{Value(10), Value(20)}}}}}, + Document{{"obj", Document{{"array", vector<Value>{Value(30), Value(40)}}}}}}, + getExpCtx()); + + unwind->setSource(source.get()); + + Document res; + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["obj"], Value(0)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["obj"], Value(1)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["obj"], Value(0)); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["obj"], Value(1)); +} + +TEST_F(UnwindStageTest, UnwindIndexPathIsChildOfArrayPath) { + const bool includeNullIfEmptyOrMissing = false; + const boost::optional<std::string> includeArrayIndex = std::string("array.index"); + auto unwind = DocumentSourceUnwind::create( + getExpCtx(), "array", includeNullIfEmptyOrMissing, includeArrayIndex); + auto source = DocumentSourceMock::createForTest( + {Document{{"array", vector<Value>{Value(10), Value(20)}}}, + Document{{"array", vector<Value>{Value(30), Value(40)}}}}, + getExpCtx()); + + unwind->setSource(source.get()); + + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(BSON("index" << 0))); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(BSON("index" << 1))); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(BSON("index" << 0))); + ASSERT_VALUE_EQ(unwind->getNext().getDocument()["array"], Value(BSON("index" << 1))); +} + // // Error cases. // @@ -816,6 +871,26 @@ TEST_F(UnwindStageTest, ShouldRejectUnrecognizedOption) { 28811); } +TEST_F(UnwindStageTest, Redaction) { + auto spec = fromjson(R"({ + $unwind: { + path: "$foo.bar", + includeArrayIndex: "foo.baz", + preserveNullAndEmptyArrays: true + } + })"); + auto docSource = DocumentSourceUnwind::createFromBson(spec.firstElement(), getExpCtx()); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "$unwind": { + "path": "$HASH<foo>.HASH<bar>", + "preserveNullAndEmptyArrays": "?bool", + "includeArrayIndex": "HASH<foo>.HASH<baz>" + } + })", + redact(*docSource)); +} + class All : public OldStyleSuiteSpecification { public: All() : OldStyleSuiteSpecification("DocumentSourceUnwindTests") {} diff --git a/src/mongo/db/pipeline/exchange_spec.idl b/src/mongo/db/pipeline/exchange_spec.idl index 5f07f822693..0bc67b96d20 100644 --- a/src/mongo/db/pipeline/exchange_spec.idl +++ b/src/mongo/db/pipeline/exchange_spec.idl @@ -46,25 +46,30 @@ enums: structs: ExchangeSpec: description: "exchange aggregation request specification" + query_shape_component: true fields: policy: type: ExchangePolicy description: A string indicating a policy of how documents are distributed to consumers. unstable: false + query_shape: parameter consumers: type: int description: Number of consumers. unstable: false + query_shape: literal orderPreserving: type: bool default: false description: A flag indicating documents are merged while preserving the order. unstable: false + query_shape: parameter bufferSize: type: int default: 16777216 description: The size of exchange buffers. unstable: false + query_shape: literal key: type: object default: "BSONObj()" @@ -74,14 +79,17 @@ structs: field listed here, or if any prefix of any path is multikey (i.e. an array is encountered while traversing a path listed here), then it is by definition sent to consumer 0. + query_shape: literal boundaries: type: array<object> optional: true description: Range/hash split points. unstable: false + query_shape: literal consumerIds: type: array<int> optional: true description: Mapping from a range index to a consumer id. unstable: false + query_shape: literal diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp index 2d4c9a61aaa..dc2aeeb7e0e 100644 --- a/src/mongo/db/pipeline/expression.cpp +++ b/src/mongo/db/pipeline/expression.cpp @@ -72,13 +72,25 @@ using std::pair; using std::string; using std::vector; -/// Helper function to easily wrap constants with $const. -static Value serializeConstant(Value val) { +Value ExpressionConstant::serializeConstant(const SerializationOptions& opts, + Value val, + bool wrapRepresentativeValue) { if (val.missing()) { return Value("$$REMOVE"_sd); } + // It's safer to wrap constants in $const when generating representative shapes to avoid + // ambiguity when re-parsing (SERVER-88296, SERVER-85376). However, we allow certain expressions + // to override this behavior in order to reduce shape verbosity if the expression takes many + // constant arguments (e.g. variadic expressions - SERVER-84159). + // Debug shapes never wrap constants in $const to reduce shape size (and because re-parsing + // support is not a consideration there). + if ((opts.literalPolicy == LiteralSerializationPolicy::kUnchanged) || + (wrapRepresentativeValue && + opts.literalPolicy == LiteralSerializationPolicy::kToRepresentativeParseableValue)) { + return Value(DOC("$const" << opts.serializeLiteral(val))); + } - return Value(DOC("$const" << val)); + return opts.serializeLiteral(val); } /* --------------------------- Expression ------------------------------ */ @@ -643,11 +655,16 @@ Value ExpressionArray::evaluate(const Document& root, Variables* variables) cons return Value(std::move(values)); } -Value ExpressionArray::serialize(bool explain) const { +Value ExpressionArray::serialize(const SerializationOptions& options) const { + if (options.literalPolicy != LiteralSerializationPolicy::kUnchanged && + selfAndChildrenAreConstant()) { + return ExpressionConstant::serializeConstant( + options, evaluate(Document{}, &(getExpressionContext()->variables))); + } vector<Value> expressions; expressions.reserve(_children.size()); for (auto&& expr : _children) { - expressions.push_back(expr->serialize(explain)); + expressions.push_back(expr->serialize(options)); } return Value(std::move(expressions)); } @@ -670,6 +687,15 @@ intrusive_ptr<Expression> ExpressionArray::optimize() { return this; } +bool ExpressionArray::selfAndChildrenAreConstant() const { + for (auto&& exprPointer : _children) { + if (!exprPointer->selfAndChildrenAreConstant()) { + return false; + } + } + return true; +} + const char* ExpressionArray::getOpName() const { // This should never be called, but is needed to inherit from ExpressionNary. return "$array"; @@ -970,11 +996,11 @@ Value ExpressionCoerceToBool::evaluate(const Document& root, Variables* variable return Value(false); } -Value ExpressionCoerceToBool::serialize(bool explain) const { +Value ExpressionCoerceToBool::serialize(const SerializationOptions& options) const { // When not explaining, serialize to an $and expression. When parsed, the $and expression // will be optimized back into a ExpressionCoerceToBool. - const char* name = explain ? "$coerceToBool" : "$and"; - return Value(DOC(name << DOC_ARRAY(pExpression->serialize(explain)))); + const char* name = options.verbosity ? "$coerceToBool" : "$and"; + return Value(DOC(name << DOC_ARRAY(pExpression->serialize(options)))); } /* ----------------------- ExpressionCompare --------------------------- */ @@ -1223,8 +1249,8 @@ Value ExpressionConstant::evaluate(const Document& root, Variables* variables) c return _value; } -Value ExpressionConstant::serialize(bool explain) const { - return serializeConstant(_value); +Value ExpressionConstant::serialize(const SerializationOptions& options) const { + return ExpressionConstant::serializeConstant(options, _value); } REGISTER_STABLE_EXPRESSION(const, ExpressionConstant::parse); @@ -1438,20 +1464,20 @@ intrusive_ptr<Expression> ExpressionDateFromParts::optimize() { return this; } -Value ExpressionDateFromParts::serialize(bool explain) const { +Value ExpressionDateFromParts::serialize(const SerializationOptions& options) const { return Value(Document{ {"$dateFromParts", - Document{{"year", _year ? _year->serialize(explain) : Value()}, - {"month", _month ? _month->serialize(explain) : Value()}, - {"day", _day ? _day->serialize(explain) : Value()}, - {"hour", _hour ? _hour->serialize(explain) : Value()}, - {"minute", _minute ? _minute->serialize(explain) : Value()}, - {"second", _second ? _second->serialize(explain) : Value()}, - {"millisecond", _millisecond ? _millisecond->serialize(explain) : Value()}, - {"isoWeekYear", _isoWeekYear ? _isoWeekYear->serialize(explain) : Value()}, - {"isoWeek", _isoWeek ? _isoWeek->serialize(explain) : Value()}, - {"isoDayOfWeek", _isoDayOfWeek ? _isoDayOfWeek->serialize(explain) : Value()}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}}}}); + Document{{"year", _year ? _year->serialize(options) : Value()}, + {"month", _month ? _month->serialize(options) : Value()}, + {"day", _day ? _day->serialize(options) : Value()}, + {"hour", _hour ? _hour->serialize(options) : Value()}, + {"minute", _minute ? _minute->serialize(options) : Value()}, + {"second", _second ? _second->serialize(options) : Value()}, + {"millisecond", _millisecond ? _millisecond->serialize(options) : Value()}, + {"isoWeekYear", _isoWeekYear ? _isoWeekYear->serialize(options) : Value()}, + {"isoWeek", _isoWeek ? _isoWeek->serialize(options) : Value()}, + {"isoDayOfWeek", _isoDayOfWeek ? _isoDayOfWeek->serialize(options) : Value()}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}}}}); } bool ExpressionDateFromParts::evaluateNumberWithDefault(const Document& root, @@ -1695,14 +1721,14 @@ intrusive_ptr<Expression> ExpressionDateFromString::optimize() { return this; } -Value ExpressionDateFromString::serialize(bool explain) const { +Value ExpressionDateFromString::serialize(const SerializationOptions& options) const { return Value( Document{{"$dateFromString", - Document{{"dateString", _dateString->serialize(explain)}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}, - {"format", _format ? _format->serialize(explain) : Value()}, - {"onNull", _onNull ? _onNull->serialize(explain) : Value()}, - {"onError", _onError ? _onError->serialize(explain) : Value()}}}}); + Document{{"dateString", _dateString->serialize(options)}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}, + {"format", _format ? _format->serialize(options) : Value()}, + {"onNull", _onNull ? _onNull->serialize(options) : Value()}, + {"onError", _onError ? _onError->serialize(options) : Value()}}}}); } Value ExpressionDateFromString::evaluate(const Document& root, Variables* variables) const { @@ -1853,12 +1879,12 @@ intrusive_ptr<Expression> ExpressionDateToParts::optimize() { return this; } -Value ExpressionDateToParts::serialize(bool explain) const { +Value ExpressionDateToParts::serialize(const SerializationOptions& options) const { return Value( Document{{"$dateToParts", - Document{{"date", _date->serialize(explain)}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}, - {"iso8601", _iso8601 ? _iso8601->serialize(explain) : Value()}}}}); + Document{{"date", _date->serialize(options)}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}, + {"iso8601", _iso8601 ? _iso8601->serialize(options) : Value()}}}}); } boost::optional<int> ExpressionDateToParts::evaluateIso8601Flag(const Document& root, @@ -2011,13 +2037,13 @@ intrusive_ptr<Expression> ExpressionDateToString::optimize() { return this; } -Value ExpressionDateToString::serialize(bool explain) const { +Value ExpressionDateToString::serialize(const SerializationOptions& options) const { return Value( Document{{"$dateToString", - Document{{"date", _date->serialize(explain)}, - {"format", _format ? _format->serialize(explain) : Value()}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}, - {"onNull", _onNull ? _onNull->serialize(explain) : Value()}}}}); + Document{{"date", _date->serialize(options)}, + {"format", _format ? _format->serialize(options) : Value()}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}, + {"onNull", _onNull ? _onNull->serialize(options) : Value()}}}}); } Value ExpressionDateToString::evaluate(const Document& root, Variables* variables) const { @@ -2158,14 +2184,14 @@ boost::intrusive_ptr<Expression> ExpressionDateDiff::optimize() { return this; }; -Value ExpressionDateDiff::serialize(bool explain) const { +Value ExpressionDateDiff::serialize(const SerializationOptions& options) const { return Value{Document{ {"$dateDiff"_sd, - Document{{"startDate"_sd, _startDate->serialize(explain)}, - {"endDate"_sd, _endDate->serialize(explain)}, - {"unit"_sd, _unit->serialize(explain)}, - {"timezone"_sd, _timeZone ? _timeZone->serialize(explain) : Value{}}, - {"startOfWeek"_sd, _startOfWeek ? _startOfWeek->serialize(explain) : Value{}}}}}}; + Document{{"startDate"_sd, _startDate->serialize(options)}, + {"endDate"_sd, _endDate->serialize(options)}, + {"unit"_sd, _unit->serialize(options)}, + {"timezone"_sd, _timeZone ? _timeZone->serialize(options) : Value{}}, + {"startOfWeek"_sd, _startOfWeek ? _startOfWeek->serialize(options) : Value{}}}}}}; }; Date_t ExpressionDateDiff::convertToDate(const Value& value, StringData parameterName) { @@ -2379,10 +2405,24 @@ Value ExpressionObject::evaluate(const Document& root, Variables* variables) con return outputDoc.freezeToValue(); } -Value ExpressionObject::serialize(bool explain) const { +bool ExpressionObject::selfAndChildrenAreConstant() const { + for (auto&& [_, exprPointer] : _expressions) { + if (!exprPointer->selfAndChildrenAreConstant()) { + return false; + } + } + return true; +} + +Value ExpressionObject::serialize(const SerializationOptions& options) const { + if (options.literalPolicy != LiteralSerializationPolicy::kUnchanged && + selfAndChildrenAreConstant()) { + return ExpressionConstant::serializeConstant(options, Value(Document{})); + } MutableDocument outputDoc; for (auto&& pair : _expressions) { - outputDoc.addField(pair.first, pair.second->serialize(explain)); + outputDoc.addField(options.serializeFieldPathFromString(pair.first), + pair.second->serialize(options)); } return outputDoc.freezeToValue(); } @@ -2557,14 +2597,34 @@ Value ExpressionFieldPath::evaluate(const Document& root, Variables* variables) } } -Value ExpressionFieldPath::serialize(bool explain) const { - if (_fieldPath.getFieldName(0) == "CURRENT" && _fieldPath.getPathLength() > 1) { +namespace { +// Shared among expressions that need to serialize dotted paths and redact the path components. +auto getPrefixAndPath(FieldPath path) { + if (path.getFieldName(0) == "CURRENT" && path.getPathLength() > 1) { // use short form for "$$CURRENT.foo" but not just "$$CURRENT" - return Value("$" + _fieldPath.tail().fullPath()); + return std::make_pair(std::string("$"), path.tail()); } else { - return Value("$$" + _fieldPath.fullPath()); + return std::make_pair(std::string("$$"), path); } } +} // namespace + +Value ExpressionFieldPath::serialize(const SerializationOptions& options) const { + auto [prefix, path] = getPrefixAndPath(_fieldPath); + // First handles special cases for redaction of system variables. User variables will fall + // through to the default full redaction case. + if (options.transformIdentifiers && prefix.length() == 2) { + if (path.getPathLength() == 1 && Variables::isBuiltin(_variable)) { + // Nothing to redact for builtin variables. + return Value(prefix + path.fullPath()); + } else if (path.getPathLength() > 1 && Variables::isBuiltin(_variable)) { + // The first component of this path is a system variable, so keep that and redact + // the rest. + return Value(prefix + path.front() + "." + options.serializeFieldPath(path.tail())); + } + } + return Value(prefix + options.serializeFieldPath(path)); +} Expression::ComputedPaths ExpressionFieldPath::getComputedPaths(const std::string& exprFieldPath, Variables::Id renamingVar) const { @@ -2705,14 +2765,14 @@ intrusive_ptr<Expression> ExpressionFilter::optimize() { return this; } -Value ExpressionFilter::serialize(bool explain) const { +Value ExpressionFilter::serialize(const SerializationOptions& options) const { if (_limit) { - return Value(DOC("$filter" << DOC("input" << _input->serialize(explain) << "as" << _varName - << "cond" << _cond->serialize(explain) << "limit" - << (*_limit)->serialize(explain)))); + return Value(DOC("$filter" << DOC("input" << _input->serialize(options) << "as" << _varName + << "cond" << _cond->serialize(options) << "limit" + << (*_limit)->serialize(options)))); } - return Value(DOC("$filter" << DOC("input" << _input->serialize(explain) << "as" << _varName - << "cond" << _cond->serialize(explain)))); + return Value(DOC("$filter" << DOC("input" << _input->serialize(options) << "as" << _varName + << "cond" << _cond->serialize(options)))); } Value ExpressionFilter::evaluate(const Document& root, Variables* variables) const { @@ -2904,15 +2964,19 @@ intrusive_ptr<Expression> ExpressionLet::optimize() { return this; } -Value ExpressionLet::serialize(bool explain) const { +Value ExpressionLet::serialize(const SerializationOptions& options) const { MutableDocument vars; for (VariableMap::const_iterator it = _variables.begin(), end = _variables.end(); it != end; ++it) { - vars[it->second.name] = it->second.expression->serialize(explain); + auto key = it->second.name; + if (options.transformIdentifiers) { + key = options.transformIdentifiersCallback(key); + } + vars[key] = it->second.expression->serialize(options); } return Value( - DOC("$let" << DOC("vars" << vars.freeze() << "in" << _subExpression->serialize(explain)))); + DOC("$let" << DOC("vars" << vars.freeze() << "in" << _subExpression->serialize(options)))); } Value ExpressionLet::evaluate(const Document& root, Variables* variables) const { @@ -3006,9 +3070,9 @@ intrusive_ptr<Expression> ExpressionMap::optimize() { return this; } -Value ExpressionMap::serialize(bool explain) const { - return Value(DOC("$map" << DOC("input" << _input->serialize(explain) << "as" << _varName << "in" - << _each->serialize(explain)))); +Value ExpressionMap::serialize(const SerializationOptions& options) const { + return Value(DOC("$map" << DOC("input" << _input->serialize(options) << "as" << _varName << "in" + << _each->serialize(options)))); } Value ExpressionMap::evaluate(const Document& root, Variables* variables) const { @@ -3150,7 +3214,7 @@ ExpressionMeta::ExpressionMeta(ExpressionContext* const expCtx, MetaType metaTyp expCtx->sbeCompatible = false; } -Value ExpressionMeta::serialize(bool explain) const { +Value ExpressionMeta::serialize(const SerializationOptions& options) const { const auto nameIter = kMetaTypeToMetaName.find(_metaType); invariant(nameIter != kMetaTypeToMetaName.end()); return Value(DOC("$meta" << nameIter->second)); @@ -3786,7 +3850,7 @@ Value ExpressionLn::evaluateNumericArg(const Value& numericArg) const { if (numericArg.getType() == NumberDecimal) { Decimal128 argDecimal = numericArg.getDecimal(); if (argDecimal.isGreater(Decimal128::kNormalizedZero)) - return Value(argDecimal.logarithm()); + return Value(argDecimal.naturalLogarithm()); // Fall through for error case. } double argDouble = numericArg.coerceToDouble(); @@ -3940,9 +4004,9 @@ Value toValue(const std::array<std::uint8_t, 32>& buf) { return Value(BSONBinData(vec.data(), vec.size(), BinDataType::Encrypt)); } -Value ExpressionInternalFLEEqual::serialize(bool explain) const { +Value ExpressionInternalFLEEqual::serialize(const SerializationOptions& options) const { return Value(Document{{kInternalFleEq, - Document{{"field", _children[0]->serialize(explain)}, + Document{{"field", _children[0]->serialize(options)}, {"edc", toValue(_edcToken)}, {"counter", Value(static_cast<long long>(_contentionFactor))}, {"server", toValue(_serverToken)}}}}); @@ -4097,13 +4161,22 @@ void ExpressionNary::addOperand(const intrusive_ptr<Expression>& pExpression) { _children.push_back(pExpression); } -Value ExpressionNary::serialize(bool explain) const { +Value ExpressionNary::serialize(const SerializationOptions& options) const { const size_t nOperand = _children.size(); vector<Value> array; /* build up the array */ - for (size_t i = 0; i < nOperand; i++) - array.push_back(_children[i]->serialize(explain)); - + for (size_t i = 0; i < nOperand; i++) { + // If this input is a constant, bypass the standard serialization that wraps the + // representative value in $const. This does not lead to ambiguity for variadic operators + // but avoids bloating the representative shape for operators that have many inputs. + ExpressionConstant const* exprConst = dynamic_cast<ExpressionConstant*>(_children[i].get()); + if (exprConst) { + array.push_back(exprConst->serializeConstant( + options, exprConst->getValue(), false /* wrapRepresentativeValue */)); + } else { + array.push_back(_children[i]->serialize(options)); + } + } return Value(DOC(getOpName() << array)); } @@ -4568,11 +4641,11 @@ void ExpressionReduce::_doAddDependencies(DepsTracker* deps) const { _in->addDependencies(deps); } -Value ExpressionReduce::serialize(bool explain) const { +Value ExpressionReduce::serialize(const SerializationOptions& options) const { return Value(Document{{"$reduce", - Document{{"input", _input->serialize(explain)}, - {"initialValue", _initial->serialize(explain)}, - {"in", _in->serialize(explain)}}}}); + Document{{"input", _input->serialize(options)}, + {"initialValue", _initial->serialize(options)}, + {"in", _in->serialize(options)}}}}); } /* ------------------------ ExpressionReplaceBase ------------------------ */ @@ -4583,11 +4656,11 @@ void ExpressionReplaceBase::_doAddDependencies(DepsTracker* deps) const { _replacement->addDependencies(deps); } -Value ExpressionReplaceBase::serialize(bool explain) const { +Value ExpressionReplaceBase::serialize(const SerializationOptions& options) const { return Value(Document{{getOpName(), - Document{{"input", _input->serialize(explain)}, - {"find", _find->serialize(explain)}, - {"replacement", _replacement->serialize(explain)}}}}); + Document{{"input", _input->serialize(options)}, + {"find", _find->serialize(options)}, + {"replacement", _replacement->serialize(options)}}}}); } namespace { @@ -4880,9 +4953,9 @@ void ExpressionSortArray::_doAddDependencies(DepsTracker* deps) const { _input->addDependencies(deps); } -Value ExpressionSortArray::serialize(bool explain) const { +Value ExpressionSortArray::serialize(const SerializationOptions& options) const { return Value(Document{{kName, - Document{{"input", _input->serialize(explain)}, + Document{{"input", _input->serialize(options)}, {"sortBy", _sortBy.getOriginalElement()}}}}); } @@ -5871,20 +5944,20 @@ boost::intrusive_ptr<Expression> ExpressionSwitch::optimize() { return this; } -Value ExpressionSwitch::serialize(bool explain) const { +Value ExpressionSwitch::serialize(const SerializationOptions& options) const { std::vector<Value> serializedBranches; serializedBranches.reserve(numBranches()); for (int i = 0; i < numBranches(); ++i) { auto [caseExpr, thenExpr] = getBranch(i); - serializedBranches.push_back(Value(Document{{"case", caseExpr->serialize(explain)}, - {"then", thenExpr->serialize(explain)}})); + serializedBranches.push_back(Value(Document{{"case", caseExpr->serialize(options)}, + {"then", thenExpr->serialize(options)}})); } if (defaultExpr()) { return Value(Document{{"$switch", Document{{"branches", Value(serializedBranches)}, - {"default", defaultExpr()->serialize(explain)}}}}); + {"default", defaultExpr()->serialize(options)}}}}); } return Value(Document{{"$switch", Document{{"branches", Value(serializedBranches)}}}}); @@ -6126,11 +6199,11 @@ boost::intrusive_ptr<Expression> ExpressionTrim::optimize() { return this; } -Value ExpressionTrim::serialize(bool explain) const { +Value ExpressionTrim::serialize(const SerializationOptions& options) const { return Value( Document{{_name, - Document{{"input", _input->serialize(explain)}, - {"chars", _characters ? _characters->serialize(explain) : Value()}}}}); + Document{{"input", _input->serialize(options)}, + {"chars", _characters ? _characters->serialize(options) : Value()}}}}); } void ExpressionTrim::_doAddDependencies(DepsTracker* deps) const { @@ -6428,17 +6501,17 @@ boost::intrusive_ptr<Expression> ExpressionZip::optimize() { return this; } -Value ExpressionZip::serialize(bool explain) const { +Value ExpressionZip::serialize(const SerializationOptions& options) const { vector<Value> serializedInput; vector<Value> serializedDefaults; Value serializedUseLongestLength = Value(_useLongestLength); for (auto&& expr : _inputs) { - serializedInput.push_back(expr.get()->serialize(explain)); + serializedInput.push_back(expr.get()->serialize(options)); } for (auto&& expr : _defaults) { - serializedDefaults.push_back(expr.get()->serialize(explain)); + serializedDefaults.push_back(expr.get()->serialize(options)); } return Value(DOC("$zip" << DOC("inputs" << Value(serializedInput) << "defaults" @@ -7019,14 +7092,6 @@ boost::intrusive_ptr<Expression> ExpressionConvert::optimize() { return this; } -Value ExpressionConvert::serialize(bool explain) const { - return Value(Document{{"$convert", - Document{{"input", _input->serialize(explain)}, - {"to", _to->serialize(explain)}, - {"onError", _onError ? _onError->serialize(explain) : Value()}, - {"onNull", _onNull ? _onNull->serialize(explain) : Value()}}}}); -} - void ExpressionConvert::_doAddDependencies(DepsTracker* deps) const { _input->addDependencies(deps); _to->addDependencies(deps); @@ -7038,6 +7103,29 @@ void ExpressionConvert::_doAddDependencies(DepsTracker* deps) const { } } +Value ExpressionConvert::serialize(const SerializationOptions& options) const { + // Since the 'to' field is a parameter from a set of valid values and not free user input, + // we want to avoid boiling it down to the representative value in the query shape. The first + // condition is so that we can keep serializing correctly whenever the 'to' field is an + // expression that gets resolved down to a string of a valid type, or its corresponding + // numerical value. If it's just the constant, we want to wrap it in a $const except when the + // serialization policy is debug. + auto constExpr = dynamic_cast<ExpressionConstant*>(_to.get()); + Value toField = Value(); + if (!constExpr) { + toField = _to->serialize(options); + } else if (options.literalPolicy == LiteralSerializationPolicy::kToDebugTypeString) { + toField = constExpr->getValue(); + } else { + toField = Value(DOC("$const" << constExpr->getValue())); + } + return Value(Document{{"$convert", + Document{{"input", _input->serialize(options)}, + {"to", toField}, + {"onError", _onError ? _onError->serialize(options) : Value()}, + {"onNull", _onNull ? _onNull->serialize(options) : Value()}}}}); +} + BSONType ExpressionConvert::computeTargetType(Value targetTypeName) const { BSONType targetType; if (targetTypeName.getType() == BSONType::String) { @@ -7300,12 +7388,12 @@ void ExpressionRegex::_compile(RegexExecutionState* executionState) const { executionState->capturesBuffer.resize((1 + executionState->numCaptures) * 3); } -Value ExpressionRegex::serialize(bool explain) const { +Value ExpressionRegex::serialize(const SerializationOptions& options) const { return Value( Document{{_opName, - Document{{"input", _input->serialize(explain)}, - {"regex", _regex->serialize(explain)}, - {"options", _options ? _options->serialize(explain) : Value()}}}}); + Document{{"input", _input->serialize(options)}, + {"regex", _regex->serialize(options)}, + {"options", _options ? _options->serialize(options) : Value()}}}}); } void ExpressionRegex::_extractInputField(RegexExecutionState* executionState, @@ -7568,7 +7656,7 @@ void ExpressionRandom::_doAddDependencies(DepsTracker* deps) const { deps->needRandomGenerator = true; } -Value ExpressionRandom::serialize(const bool explain) const { +Value ExpressionRandom::serialize(const SerializationOptions& options) const { return Value(DOC(getOpName() << Document())); } @@ -7591,8 +7679,8 @@ Value ExpressionToHashedIndexKey::evaluate(const Document& root, Variables* vari BSONElementHasher::DEFAULT_HASH_SEED)); } -Value ExpressionToHashedIndexKey::serialize(bool explain) const { - return Value(DOC("$toHashedIndexKey" << _children[0]->serialize(explain))); +Value ExpressionToHashedIndexKey::serialize(const SerializationOptions& options) const { + return Value(DOC("$toHashedIndexKey" << _children[0]->serialize(options))); } void ExpressionToHashedIndexKey::_doAddDependencies(DepsTracker* deps) const { @@ -7668,13 +7756,13 @@ boost::intrusive_ptr<Expression> ExpressionDateArithmetics::optimize() { return intrusive_ptr<Expression>(this); } -Value ExpressionDateArithmetics::serialize(bool explain) const { +Value ExpressionDateArithmetics::serialize(const SerializationOptions& options) const { return Value( Document{{_opName, - Document{{"startDate", _startDate->serialize(explain)}, - {"unit", _unit->serialize(explain)}, - {"amount", _amount->serialize(explain)}, - {"timezone", _timeZone ? _timeZone->serialize(explain) : Value()}}}}); + Document{{"startDate", _startDate->serialize(options)}, + {"unit", _unit->serialize(options)}, + {"amount", _amount->serialize(options)}, + {"timezone", _timeZone ? _timeZone->serialize(options) : Value()}}}}); } Value ExpressionDateArithmetics::evaluate(const Document& root, Variables* variables) const { @@ -7867,14 +7955,14 @@ boost::intrusive_ptr<Expression> ExpressionDateTrunc::optimize() { return this; }; -Value ExpressionDateTrunc::serialize(bool explain) const { +Value ExpressionDateTrunc::serialize(const SerializationOptions& options) const { return Value{Document{ {"$dateTrunc"_sd, - Document{{"date"_sd, _date->serialize(explain)}, - {"unit"_sd, _unit->serialize(explain)}, - {"binSize"_sd, _binSize ? _binSize->serialize(explain) : Value{}}, - {"timezone"_sd, _timeZone ? _timeZone->serialize(explain) : Value{}}, - {"startOfWeek"_sd, _startOfWeek ? _startOfWeek->serialize(explain) : Value{}}}}}}; + Document{{"date"_sd, _date->serialize(options)}, + {"unit"_sd, _unit->serialize(options)}, + {"binSize"_sd, _binSize ? _binSize->serialize(options) : Value{}}, + {"timezone"_sd, _timeZone ? _timeZone->serialize(options) : Value{}}, + {"startOfWeek"_sd, _startOfWeek ? _startOfWeek->serialize(options) : Value{}}}}}}; }; Date_t ExpressionDateTrunc::convertToDate(const Value& value) { @@ -8056,7 +8144,6 @@ Value ExpressionGetField::evaluate(const Document& root, Variables* variables) c return Value(); } - return inputValue.getDocument().getField(fieldValue.getString()); } @@ -8069,10 +8156,22 @@ void ExpressionGetField::_doAddDependencies(DepsTracker* deps) const { _field->addDependencies(deps); } -Value ExpressionGetField::serialize(const bool explain) const { +Value ExpressionGetField::serialize(const SerializationOptions& options) const { + // The parser guarantees that the '_field' expression evaluates to a constant string. + auto strPath = static_cast<ExpressionConstant*>(_field.get())->getValue().getString(); + + Value maybeRedactedPath{options.serializeFieldPathFromString(strPath)}; + // This is a pretty unique option to serialize. It is both a constant and a field path, which + // means that it: + // - should be redacted (if that option is set). + // - should *not* be wrapped in $const iff we are serializing for a debug string + if (options.literalPolicy != LiteralSerializationPolicy::kToDebugTypeString) { + maybeRedactedPath = Value(Document{{"$const"_sd, maybeRedactedPath}}); + } + return Value(Document{{"$getField"_sd, - Document{{"field"_sd, _field->serialize(explain)}, - {"input"_sd, _input->serialize(explain)}}}}); + Document{{"field"_sd, std::move(maybeRedactedPath)}, + {"input"_sd, _input->serialize(options)}}}}); } /* -------------------------- ExpressionSetField ------------------------------ */ @@ -8129,46 +8228,11 @@ intrusive_ptr<Expression> ExpressionSetField::parse(ExpressionContext* const exp uassert(4161103, str::stream() << name << " requires 'value' to be specified", valueExpr); uassert(4161109, str::stream() << name << " requires 'input' to be specified", inputExpr); - // The 'field' argument to '$setField' must evaluate to a constant string, for example, - // {$const: "$a.b"}. In case the user has forgotten to wrap the value into a '$const' or - // '$literal' expression, we will raise an error with a more meaningful description. - if (auto fieldPathExpr = dynamic_cast<ExpressionFieldPath*>(fieldExpr.get()); fieldPathExpr) { - auto fp = fieldPathExpr->getFieldPath().fullPathWithPrefix(); - uasserted(4161108, - str::stream() << "'" << fp - << "' is a field path reference which is not allowed " - "in this context. Did you mean {$literal: '" - << fp << "'}?"); - } - - auto constFieldExpr = dynamic_cast<ExpressionConstant*>(fieldExpr.get()); - uassert(4161106, - str::stream() << name - << " requires 'field' to evaluate to a constant, " - "but got a non-constant argument", - constFieldExpr); - uassert(4161107, - str::stream() << name - << " requires 'field' to evaluate to type String, " - "but got " - << typeName(constFieldExpr->getValue().getType()), - constFieldExpr->getValue().getType() == BSONType::String); - - - return make_intrusive<ExpressionSetField>(expCtx, fieldExpr, inputExpr, valueExpr); + return make_intrusive<ExpressionSetField>( + expCtx, std::move(fieldExpr), std::move(inputExpr), std::move(valueExpr)); } Value ExpressionSetField::evaluate(const Document& root, Variables* variables) const { - auto field = _field->evaluate(root, variables); - - // The parser guarantees that the '_field' expression evaluates to a constant string. - tassert(4161104, - str::stream() << kExpressionName - << " requires 'field' to evaluate to type String, " - "but got " - << typeName(field.getType()), - field.getType() == BSONType::String); - auto input = _input->evaluate(root, variables); if (input.nullish()) { return Value(BSONNULL); @@ -8182,7 +8246,7 @@ Value ExpressionSetField::evaluate(const Document& root, Variables* variables) c // Build output document and modify 'field'. MutableDocument outputDoc(input.getDocument()); - outputDoc.setField(field.getString(), value); + outputDoc.setField(_fieldName, value); return outputDoc.freezeToValue(); } @@ -8196,11 +8260,59 @@ void ExpressionSetField::_doAddDependencies(DepsTracker* deps) const { _value->addDependencies(deps); } -Value ExpressionSetField::serialize(const bool explain) const { +Value ExpressionSetField::serialize(const SerializationOptions& options) const { + // The parser guarantees that the '_field' expression evaluates to a constant string. + auto strPath = static_cast<ExpressionConstant*>(_field.get())->getValue().getString(); + + Value maybeRedactedPath{options.serializeFieldPathFromString(strPath)}; + // This is a pretty unique option to serialize. It is both a constant and a field path, which + // means that it: + // - should be redacted (if that option is set). + // - should *not* be wrapped in $const iff we are serializing for a debug string + if (options.literalPolicy != LiteralSerializationPolicy::kToDebugTypeString) { + maybeRedactedPath = Value(Document{{"$const"_sd, maybeRedactedPath}}); + } + return Value(Document{{"$setField"_sd, - Document{{"field"_sd, _field->serialize(explain)}, - {"input"_sd, _input->serialize(explain)}, - {"value"_sd, _value->serialize(explain)}}}}); + Document{{"field"_sd, std::move(maybeRedactedPath)}, + {"input"_sd, _input->serialize(options)}, + {"value"_sd, _value->serialize(options)}}}}); +} + +std::string ExpressionSetField::getValidFieldName(boost::intrusive_ptr<Expression> fieldExpr) { + tassert(9534701, + str::stream() << kExpressionName << " requires 'field' to be specified", + fieldExpr); + + // The 'field' argument to '$setField' must evaluate to a constant string, for example, + // {$const: "$a.b"}. In case the user has forgotten to wrap the value into a '$const' or + // '$literal' expression, we will raise an error with a more meaningful description. + if (auto fieldPathExpr = dynamic_cast<ExpressionFieldPath*>(fieldExpr.get()); fieldPathExpr) { + auto fp = fieldPathExpr->getFieldPath().fullPathWithPrefix(); + uasserted(4161108, + str::stream() << "'" << fp + << "' is a field path reference which is not allowed " + "in this context. Did you mean {$literal: '" + << fp << "'}?"); + } + + auto constFieldExpr = dynamic_cast<ExpressionConstant*>(fieldExpr.get()); + uassert(4161106, + str::stream() << kExpressionName + << " requires 'field' to evaluate to a constant, " + "but got a non-constant argument", + constFieldExpr); + uassert(4161107, + str::stream() << kExpressionName + << " requires 'field' to evaluate to type String, " + "but got " + << typeName(constFieldExpr->getValue().getType()), + constFieldExpr->getValue().getType() == BSONType::String); + uassert(9534700, + str::stream() << kExpressionName << ": 'field' cannot contain an embedded null byte", + constFieldExpr->getValue().getStringData().find('\0') == std::string::npos); + + return constFieldExpr->getValue().getString(); } /* ------------------------- ExpressionTsSecond ----------------------------- */ @@ -8289,12 +8401,12 @@ boost::intrusive_ptr<Expression> ExpressionInternalKeyStringValue::parse( return make_intrusive<ExpressionInternalKeyStringValue>(expCtx, inputExpr, collationExpr); } -Value ExpressionInternalKeyStringValue::serialize(bool explain) const { +Value ExpressionInternalKeyStringValue::serialize(const SerializationOptions& options) const { return Value( Document{{getOpName(), - Document{{"input", _children[_kInput]->serialize(explain)}, + Document{{"input", _children[_kInput]->serialize(options)}, {"collation", - _children[_kCollation] ? _children[_kCollation]->serialize(explain) + _children[_kCollation] ? _children[_kCollation]->serialize(options) : Value()}}}}); } diff --git a/src/mongo/db/pipeline/expression.h b/src/mongo/db/pipeline/expression.h index 26f04430e0b..c5140186fca 100644 --- a/src/mongo/db/pipeline/expression.h +++ b/src/mongo/db/pipeline/expression.h @@ -54,6 +54,7 @@ #include "mongo/db/query/allowed_contexts.h" #include "mongo/db/query/datetime/date_time_support.h" #include "mongo/db/query/query_feature_flags_gen.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/db/query/sort_pattern.h" #include "mongo/db/server_options.h" #include "mongo/db/update/pattern_cmp.h" @@ -223,7 +224,7 @@ public: * If 'explain' is false, the returned Value must result in the same Expression when parsed by * parseOperand(). */ - virtual Value serialize(bool explain) const = 0; + virtual Value serialize(const SerializationOptions& options = {}) const = 0; /** * Evaluate the expression with respect to the Document given by 'root' and the Variables given @@ -352,6 +353,13 @@ public: return monotonic::State::NonMonotonic; } + /** + * Helper to determine whether this expression always evaluates to the same value. + */ + virtual bool selfAndChildrenAreConstant() const { + return false; + } + protected: using ExpressionVector = std::vector<boost::intrusive_ptr<Expression>>; @@ -384,12 +392,105 @@ private: }; /** + * A constant expression. Repeated calls to evaluate() will always return the same thing. + */ +class ExpressionConstant final : public Expression { +public: + ExpressionConstant(ExpressionContext* expCtx, const Value& value); + + boost::intrusive_ptr<Expression> optimize() final; + Value evaluate(const Document& root, Variables* variables) const final; + Value serialize(const SerializationOptions& options = {}) const final; + + const char* getOpName() const; + + /** + * Creates a new ExpressionConstant with value 'value'. + */ + static boost::intrusive_ptr<ExpressionConstant> create(ExpressionContext* expCtx, + const Value& value); + + static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, + BSONElement bsonExpr, + const VariablesParseState& vps); + + /** + * Returns true if 'expression' is nullptr or if 'expression' is an instance of an + * ExpressionConstant. + */ + static bool isNullOrConstant(boost::intrusive_ptr<Expression> expression) { + return !expression || dynamic_cast<ExpressionConstant*>(expression.get()); + } + + /** + * Returns true if 'expression' is an instance of an ExpressionConstant. + */ + static bool isConstant(boost::intrusive_ptr<Expression> expression) { + return dynamic_cast<ExpressionConstant*>(expression.get()); + } + + static Value serializeConstant(const SerializationOptions& opts, + Value val, + bool wrapRepresentativeValue = true); + + bool selfAndChildrenAreConstant() const override final { + return true; + } + + /** + * Returns true if every expression in 'expressions' is either a nullptr or an instance of an + * ExpressionConstant. + */ + static bool allNullOrConstant( + const std::initializer_list<boost::intrusive_ptr<Expression>>& expressions) { + return std::all_of(expressions.begin(), expressions.end(), [](auto exp) { + return ExpressionConstant::isNullOrConstant(exp); + }); + } + template <typename ExpressionContainer> + static bool allConstant(const ExpressionContainer& expressions) { + return std::all_of(expressions.begin(), expressions.end(), [](auto exp) { + return ExpressionConstant::isConstant(exp); + }); + } + + /** + * Returns the constant value represented by this Expression. + */ + Value getValue() const { + return _value; + } + + void setValue(const Value& value) { + _value = value; + }; + + void acceptVisitor(ExpressionMutableVisitor* visitor) final { + return visitor->visit(this); + } + + void acceptVisitor(ExpressionConstVisitor* visitor) const final { + return visitor->visit(this); + } + +protected: + void _doAddDependencies(DepsTracker* deps) const override; + +private: + monotonic::State getMonotonicState(const FieldPath& sortedFieldPath) const final { + return monotonic::State::Constant; + } + + Value _value; +}; + +/** * Inherit from ExpressionVariadic or ExpressionFixedArity instead of directly from this class. */ class ExpressionNary : public Expression { public: boost::intrusive_ptr<Expression> optimize() override; - Value serialize(bool explain) const override; + Value serialize(const SerializationOptions& options = {}) const override; /* Add an operand to the n-ary expression. @@ -454,6 +555,33 @@ public: : ExpressionNaryBase<SubClass>(expCtx) {} ExpressionVariadic(ExpressionContext* const expCtx, Expression::ExpressionVector&& children) : ExpressionNaryBase<SubClass>(expCtx, std::move(children)) {} + + Value serialize(const SerializationOptions& options = {}) const { + // As a special case, we would like to serialize a variadic number of children as + // "?array<?subtype>" if they are all constant. Check for that here, otherwise default to + // the normal one-by-one serialization of the children. + if (options.literalPolicy == LiteralSerializationPolicy::kToDebugTypeString && + ExpressionConstant::allConstant(this->_children)) { + // We could evaluate the expression right here and now and end up with just the one + // constant answer, but this is not an optimization funciton, it is meant to just + // serialize what we have, so let's preserve the array of constants. + auto args = [&]() { + std::vector<Value> values; + const auto& constants = this->_children; + values.reserve(constants.size()); + std::transform(constants.begin(), + constants.end(), + std::back_inserter(values), + [](const auto& exp) { + return static_cast<ExpressionConstant*>(exp.get())->getValue(); + }); + return values; + }(); + return Value(Document{ + {this->getOpName(), ExpressionConstant::serializeConstant(options, Value(args))}}); + } + return ExpressionNary::serialize(options); + } }; /** @@ -569,9 +697,9 @@ public: return AccumulatorN::kName.rawData(); } - Value serialize(bool explain) const { + Value serialize(const SerializationOptions& options = {}) const { MutableDocument md; - AccumulatorN::serializeHelper(_n, _output, explain, md); + AccumulatorN::serializeHelper(_n, _output, options, md); return Value(DOC(getOpName() << md.freeze())); } @@ -686,78 +814,6 @@ public: }; /** - * A constant expression. Repeated calls to evaluate() will always return the same thing. - */ -class ExpressionConstant final : public Expression { -public: - ExpressionConstant(ExpressionContext* expCtx, const Value& value); - - boost::intrusive_ptr<Expression> optimize() final; - Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; - - const char* getOpName() const; - - /** - * Creates a new ExpressionConstant with value 'value'. - */ - static boost::intrusive_ptr<ExpressionConstant> create(ExpressionContext* expCtx, - const Value& value); - - static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, - BSONElement bsonExpr, - const VariablesParseState& vps); - - /** - * Returns true if 'expression' is nullptr or if 'expression' is an instance of an - * ExpressionConstant. - */ - static bool isNullOrConstant(boost::intrusive_ptr<Expression> expression) { - return !expression || dynamic_cast<ExpressionConstant*>(expression.get()); - } - - /** - * Returns true if every expression in 'expressions' is either a nullptr or an instance of an - * ExpressionConstant. - */ - static bool allNullOrConstant( - const std::initializer_list<boost::intrusive_ptr<Expression>>& expressions) { - return std::all_of(expressions.begin(), expressions.end(), [](auto exp) { - return ExpressionConstant::isNullOrConstant(exp); - }); - } - - /** - * Returns the constant value represented by this Expression. - */ - Value getValue() const { - return _value; - } - - void setValue(const Value& value) { - _value = value; - }; - - void acceptVisitor(ExpressionMutableVisitor* visitor) final { - return visitor->visit(this); - } - - void acceptVisitor(ExpressionConstVisitor* visitor) const final { - return visitor->visit(this); - } - -protected: - void _doAddDependencies(DepsTracker* deps) const override; - -private: - monotonic::State getMonotonicState(const FieldPath& sortedFieldPath) const final { - return monotonic::State::Constant; - } - - Value _value; -}; - -/** * Inherit from this class if your expression works with date types, and accepts either a single * argument which is a date, or an object {date: <date>, timezone: <string>}. */ @@ -798,11 +854,11 @@ public: * Always serializes to the full {date: <date arg>, timezone: <timezone arg>} format, leaving * off the timezone if not specified. */ - Value serialize(bool explain) const final { - auto timezone = _timeZone ? _timeZone->serialize(explain) : Value(); + Value serialize(const SerializationOptions& options = {}) const final { + auto timezone = _timeZone ? _timeZone->serialize(options) : Value(); return Value(Document{ {_opName, - Document{{"date", _date->serialize(explain)}, {"timezone", std::move(timezone)}}}}); + Document{{"date", _date->serialize(options)}, {"timezone", std::move(timezone)}}}}); } boost::intrusive_ptr<Expression> optimize() final { @@ -1056,7 +1112,7 @@ public: } Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; static boost::intrusive_ptr<ExpressionArray> create( ExpressionContext* const expCtx, std::vector<boost::intrusive_ptr<Expression>>&& children) { @@ -1073,6 +1129,8 @@ public: void acceptVisitor(ExpressionConstVisitor* visitor) const final { return visitor->visit(this); } + + bool selfAndChildrenAreConstant() const override final; }; @@ -1222,7 +1280,7 @@ class ExpressionCoerceToBool final : public Expression { public: boost::intrusive_ptr<Expression> optimize() final; Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; static boost::intrusive_ptr<ExpressionCoerceToBool> create( ExpressionContext* expCtx, boost::intrusive_ptr<Expression> pExpression); @@ -1385,8 +1443,8 @@ public: boost::intrusive_ptr<Expression> onNull, boost::intrusive_ptr<Expression> onError); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -1427,8 +1485,8 @@ public: boost::intrusive_ptr<Expression> isoDayOfWeek, boost::intrusive_ptr<Expression> timeZone); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -1507,8 +1565,8 @@ public: boost::intrusive_ptr<Expression> timeZone, boost::intrusive_ptr<Expression> iso8601); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -1541,8 +1599,8 @@ public: boost::intrusive_ptr<Expression> date, boost::intrusive_ptr<Expression> timeZone, boost::intrusive_ptr<Expression> onNull); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -1653,8 +1711,8 @@ public: boost::intrusive_ptr<Expression> unit, boost::intrusive_ptr<Expression> timezone, boost::intrusive_ptr<Expression> startOfWeek); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, @@ -1786,7 +1844,7 @@ public: boost::intrusive_ptr<Expression> optimize() final; Value evaluate(const Document& root, Variables* variables) const; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; /* Create a field path expression using old semantics (rooted off of CURRENT). @@ -1887,8 +1945,8 @@ private: class ExpressionFilter final : public Expression { public: - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -2124,8 +2182,8 @@ public: class ExpressionLet final : public Expression { public: - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -2241,7 +2299,7 @@ public: ConstDataRange serverToken, int64_t contentionFactor, ConstDataRange edcToken); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; const char* getOpName() const; @@ -2275,8 +2333,8 @@ public: boost::intrusive_ptr<Expression> input, // yields array to iterate boost::intrusive_ptr<Expression> each); // yields results to be added to output array - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -2308,7 +2366,7 @@ class ExpressionMeta final : public Expression { public: ExpressionMeta(ExpressionContext* expCtx, DocumentMetadataFields::MetaType metaType); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, @@ -2502,7 +2560,7 @@ class ExpressionObject final : public Expression { public: boost::intrusive_ptr<Expression> optimize() final; Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; static boost::intrusive_ptr<ExpressionObject> create( ExpressionContext* expCtx, @@ -2534,8 +2592,10 @@ public: return visitor->visit(this); } + protected: void _doAddDependencies(DepsTracker* deps) const final; + bool selfAndChildrenAreConstant() const override final; private: ExpressionObject( @@ -2645,7 +2705,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vps); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -2681,8 +2741,8 @@ public: virtual const char* getOpName() const = 0; Value evaluate(const Document& root, Variables* variables) const final; - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; protected: void _doAddDependencies(DepsTracker* deps) const final; @@ -2962,7 +3022,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vps); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3078,10 +3138,14 @@ private: FieldPath getFieldPath() const { auto inputConstExpression = dynamic_cast<ExpressionConstant*>(_children[0].get()); uassert(5511201, - "Expected const expression as argument to _internalUnwindAllAlongPath", + "Expected const expression as argument to _internalFindAllValuesAtPath", inputConstExpression); auto constVal = inputConstExpression->getValue(); - // getString asserts if type != string, which is the correct behavior for what we want. + + uassert(9567004, + str::stream() << getOpName() << " requires argument to be a string", + constVal.getType() == BSONType::String); + return FieldPath(constVal.getString()); } }; @@ -3337,7 +3401,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vpsIn); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3452,7 +3516,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vpsIn); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3703,7 +3767,7 @@ public: static boost::intrusive_ptr<Expression> parse(ExpressionContext* expCtx, BSONElement expr, const VariablesParseState& vpsIn); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3742,8 +3806,8 @@ public: const VariablesParseState& vpsIn); Value evaluate(const Document& root, Variables* variables) const final; - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -3848,7 +3912,7 @@ public: boost::optional<std::pair<boost::optional<std::string>, std::string>> getConstantPatternAndOptions() const; - Value serialize(bool explain) const; + Value serialize(const SerializationOptions& options = {}) const; const std::string& getOpName() const { return _opName; @@ -3965,7 +4029,7 @@ public: BSONElement exprElement, const VariablesParseState& vps); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; @@ -4011,7 +4075,7 @@ public: } Value evaluate(const Document& root, Variables* variables) const; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; protected: void _doAddDependencies(DepsTracker* deps) const final; @@ -4034,8 +4098,8 @@ public: _timeZone(_children[3]), _opName(opName) {} - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; protected: @@ -4162,8 +4226,8 @@ public: boost::intrusive_ptr<Expression> binSize, boost::intrusive_ptr<Expression> timezone, boost::intrusive_ptr<Expression> startOfWeek); - boost::intrusive_ptr<Expression> optimize() final; - Value serialize(bool explain) const final; + boost::intrusive_ptr<Expression> optimize() override; + Value serialize(const SerializationOptions& options = {}) const override; Value evaluate(const Document& root, Variables* variables) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); @@ -4253,7 +4317,7 @@ public: expCtx->sbeCompatible = false; } - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; @@ -4294,11 +4358,12 @@ public: : Expression(expCtx, {std::move(field), std::move(input), std::move(value)}), _field(_children[0]), _input(_children[1]), - _value(_children[2]) { + _value(_children[2]), + _fieldName(getValidFieldName(_children[0])) { expCtx->sbeCompatible = false; } - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; @@ -4318,9 +4383,18 @@ protected: void _doAddDependencies(DepsTracker* deps) const final override; private: + /** + * Ensures 'fieldExpr' is a constant string representing a valid field name and returns it as a + * string. If 'fieldExpr' is not valid, this function will throw a 'uassert()'. + */ + std::string getValidFieldName(boost::intrusive_ptr<Expression> fieldExpr); + boost::intrusive_ptr<Expression>& _field; boost::intrusive_ptr<Expression>& _input; boost::intrusive_ptr<Expression>& _value; + + // This is pre-validated by the constructor. + const std::string _fieldName; }; class ExpressionTsSecond final : public ExpressionFixedArity<ExpressionTsSecond, 1> { @@ -4429,7 +4503,7 @@ public: BSONElement expr, const VariablesParseState& vps); - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options = {}) const final; Value evaluate(const Document& root, Variables* variables) const final; diff --git a/src/mongo/db/pipeline/expression_and_test.cpp b/src/mongo/db/pipeline/expression_and_test.cpp index cbfa37a77d1..9671c08b2db 100644 --- a/src/mongo/db/pipeline/expression_and_test.cpp +++ b/src/mongo/db/pipeline/expression_and_test.cpp @@ -76,8 +76,8 @@ static BSONObj toBson(const Value& value) { } /** Convert Expression to BSON. */ -static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); +static BSONObj expressionToBson(const boost::intrusive_ptr<Expression>& expression) { + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } /** Convert Document to BSON. */ diff --git a/src/mongo/db/pipeline/expression_compare_test.cpp b/src/mongo/db/pipeline/expression_compare_test.cpp index 11836d77267..941eceeaa59 100644 --- a/src/mongo/db/pipeline/expression_compare_test.cpp +++ b/src/mongo/db/pipeline/expression_compare_test.cpp @@ -75,7 +75,7 @@ static BSONObj toBson(const Value& value) { /** Convert Expression to BSON. */ static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } /** Convert Document to BSON. */ diff --git a/src/mongo/db/pipeline/expression_context.cpp b/src/mongo/db/pipeline/expression_context.cpp index 7258456a1ac..626db2f6016 100644 --- a/src/mongo/db/pipeline/expression_context.cpp +++ b/src/mongo/db/pipeline/expression_context.cpp @@ -49,6 +49,49 @@ ExpressionContext::ResolvedNamespace::ResolvedNamespace(NamespaceString ns, : ns(std::move(ns)), pipeline(std::move(pipeline)), uuid(collUUID) {} ExpressionContext::ExpressionContext(OperationContext* opCtx, + const FindCommandRequest& findCmd, + std::unique_ptr<CollatorInterface> collator, + bool mayDbProfile, + boost::optional<ExplainOptions::Verbosity> verbosity, + bool allowDiskUseDefault) + // Although both 'find' and 'aggregate' commands have an ExpressionContext, some of the data + // members in the ExpressionContext are used exclusively by the aggregation subsystem. This + // includes the following fields which here we simply initialize to some meaningless default + // value: + // - explain + // - fromMongos + // - needsMerge + // - bypassDocumentValidation + // - mongoProcessInterface + // - resolvedNamespaces + // - uuid + // + // As we change the code to make the find and agg systems more tightly coupled, it would make + // sense to start initializing these fields for find operations as well. + : ExpressionContext(opCtx, + verbosity, + false, // fromMongos + false, // needsMerge + findCmd.getAllowDiskUse().value_or(allowDiskUseDefault), + false, // bypassDocumentValidation + false, // isMapReduceCommand + findCmd.getNamespaceOrUUID().nss().has_value() + ? *findCmd.getNamespaceOrUUID().nss() + : NamespaceString{}, + findCmd.getLegacyRuntimeConstants(), + std::move(collator), + nullptr, // mongoProcessInterface + {}, // resolvedNamespaces + [&findCmd]() -> boost::optional<UUID> { + if (findCmd.getNamespaceOrUUID().uuid().has_value()) { + return findCmd.getNamespaceOrUUID().uuid(); + } + return boost::none; + }(), + findCmd.getLet(), + mayDbProfile) {} + +ExpressionContext::ExpressionContext(OperationContext* opCtx, const AggregateCommandRequest& request, std::unique_ptr<CollatorInterface> collator, std::shared_ptr<MongoProcessInterface> processInterface, @@ -159,6 +202,44 @@ ExpressionContext::ExpressionContext( variables.seedVariablesWithLetParameters(this, *letParameters); } +ExpressionContext::ExpressionContext(OperationContext* opCtx, + const NamespaceString& nss, + const boost::optional<BSONObj>& letParameters) + : explain(boost::none), + allowDiskUse(false), + ns(nss), + opCtx(opCtx), + jsHeapLimitMB(internalQueryJavaScriptHeapSizeLimitMB.load()), + mongoProcessInterface(std::make_shared<StubMongoProcessInterface>()), + timeZoneDatabase(opCtx && opCtx->getServiceContext() + ? TimeZoneDatabase::get(opCtx->getServiceContext()) + : nullptr), + variablesParseState(variables.useIdGenerator()), + maxFeatureCompatibilityVersion(boost::none), // Ensure all features are allowed. + mayDbProfile(true), + _collator(nullptr), + _documentComparator(_collator.get()), + _valueComparator(_collator.get()) { + // This is a shortcut to avoid reading the clock and the vector clock, since we don't actually + // care about their values for this 'blank' ExpressionContext codepath. + variables.setLegacyRuntimeConstants({Date_t::min(), Timestamp()}); + // Expression counters are reported in serverStatus to indicate how often clients use certain + // expressions/stages, so it's a side effect tied to parsing. We must stop expression counters + // before re-parsing to avoid adding to the counters more than once per a given query. + stopExpressionCounters(); + if (letParameters) + variables.seedVariablesWithLetParameters(this, *letParameters); +} + +boost::intrusive_ptr<ExpressionContext> ExpressionContext::makeBlankExpressionContext( + OperationContext* opCtx, + const NamespaceStringOrUUID& nssOrUUID, + boost::optional<BSONObj> shapifiedLet) { + const auto nss = nssOrUUID.nss().has_value() ? *nssOrUUID.nss() : NamespaceString{}; + // This constructor is private, so we can't use `boost::make_instrusive()`. + return new ExpressionContext(opCtx, nss, shapifiedLet); +} + void ExpressionContext::checkForInterruptSlow() { // This check could be expensive, at least in relative terms, so don't check every time. invariant(opCtx); diff --git a/src/mongo/db/pipeline/expression_context.h b/src/mongo/db/pipeline/expression_context.h index c5ce9b74f27..aa2d212f53d 100644 --- a/src/mongo/db/pipeline/expression_context.h +++ b/src/mongo/db/pipeline/expression_context.h @@ -114,6 +114,16 @@ public: }; /** + * Constructs an ExpressionContext to be used for find command parsing and evaluation. + */ + ExpressionContext(OperationContext* opCtx, + const FindCommandRequest& findCmd, + std::unique_ptr<CollatorInterface> collator, + bool mayDbProfile, + boost::optional<ExplainOptions::Verbosity> verbosity = boost::none, + bool allowDiskUseByDefault = false); + + /** * Constructs an ExpressionContext to be used for Pipeline parsing and evaluation. * 'resolvedNamespaces' maps collection names (not full namespaces) to ResolvedNamespaces. */ @@ -161,6 +171,22 @@ public: boost::optional<ExplainOptions::Verbosity> explain = boost::none); /** + * Constructs a blank ExpressionContext suitable for creating Query Shapes, but it could be + * applied to other use cases as well. + * + * The process for creating a Query Shape sometimes requires re-parsing the BSON into a proper + * AST, and for that you need an ExpressionContext. + * + * Note: this is meant for introspection and is not suitable for using to execute queries - + * since it does not contain for example a collation argument or a real MongoProcessInterface + * for execution. + */ + static boost::intrusive_ptr<ExpressionContext> makeBlankExpressionContext( + OperationContext* opCtx, + const NamespaceStringOrUUID& nssOrUUID, + boost::optional<BSONObj> shapifiedLet = boost::none); + + /** * Used by a pipeline to check for interrupts so that killOp() works. Throws a UserAssertion if * this aggregation pipeline has been interrupted. */ @@ -278,7 +304,8 @@ public: */ const ResolvedNamespace& getResolvedNamespace(const NamespaceString& nss) const { auto it = _resolvedNamespaces.find(nss.coll()); - invariant(it != _resolvedNamespaces.end()); + invariant(it != _resolvedNamespaces.end(), + str::stream() << "No resolved namespace provided for " << nss.toString()); return it->second; }; @@ -323,10 +350,8 @@ public: * created yet. Initializes the Scope with the 'jsScope' variables from the runtimeConstants. * Loads the Scope with the functions stored in system.js if the expression isn't executed on * mongos and is called from a MapReduce command or `forceLoadOfStoredProcedures` is true. - * - * Returns a JsExec and a boolean indicating whether the Scope was created as part of this call. */ - auto getJsExecWithScope(bool forceLoadOfStoredProcedures = false) const { + JsExecution* getJsExecWithScope(bool forceLoadOfStoredProcedures = false) const { uassert(31264, "Cannot run server-side javascript without the javascript engine enabled", getGlobalScriptEngine()); @@ -348,9 +373,17 @@ public: "$where."); } - auto scopeObj = BSONObj(); + // If there is a cached JsExecution object, return it. This is a performance optimization + // that avoids potentially copying the scope object, which is not needed if a cached exec + // object already exists. + JsExecution* jsExec = JsExecution::getCached(opCtx, loadStoredProcedures); + if (jsExec) { + return jsExec; + } + + BSONObj scopeObj = BSONObj(); if (variables.hasValue(Variables::kJsScopeId)) { - auto scopeVar = variables.getValue(Variables::kJsScopeId); + Value scopeVar = variables.getValue(Variables::kJsScopeId); invariant(scopeVar.isObject()); scopeObj = scopeVar.getDocument().toBson(); } @@ -558,6 +591,14 @@ protected: bool _requiresTimeseriesExtendedRangeSupport = false; private: + // Instantiates an ExpressionContext which does not increment expression counters and does not + // enforce FCV restrictions. It is used for implementing the `makeBlankExpressionContext()` + // factory method. Please also note that the runtime constants are not given real/accurate + // values of '$$NOW' and '$$CLUSTER_TIME', in the name of efficiency. + ExpressionContext(OperationContext* opCtx, + const NamespaceString& ns, + const boost::optional<BSONObj>& letParameters = boost::none); + boost::optional<ExpressionCounters> _expressionCounters = boost::none; }; diff --git a/src/mongo/db/pipeline/expression_context_test.cpp b/src/mongo/db/pipeline/expression_context_test.cpp index b105dce531b..2ff0ef20b1e 100644 --- a/src/mongo/db/pipeline/expression_context_test.cpp +++ b/src/mongo/db/pipeline/expression_context_test.cpp @@ -39,15 +39,6 @@ #include "mongo/db/vector_clock_mutable.h" #include "mongo/unittest/unittest.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString(); \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace mongo { namespace { diff --git a/src/mongo/db/pipeline/expression_convert_test.cpp b/src/mongo/db/pipeline/expression_convert_test.cpp index 678c29b5b16..c1cfd639ce1 100644 --- a/src/mongo/db/pipeline/expression_convert_test.cpp +++ b/src/mongo/db/pipeline/expression_convert_test.cpp @@ -67,10 +67,11 @@ TEST_F(ExpressionConvertTest, ParseAndSerializeWithoutOptionalArguments) { auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}}}")), - convertExp->serialize(false)); + convertExp->serialize()); ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}}}")), - convertExp->serialize(true)); + convertExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})); } TEST_F(ExpressionConvertTest, ParseAndSerializeWithOnError) { @@ -85,11 +86,12 @@ TEST_F(ExpressionConvertTest, ParseAndSerializeWithOnError) { ASSERT_VALUE_EQ( Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}, onError: {$const: 0}}}")), - convertExp->serialize(false)); + convertExp->serialize()); ASSERT_VALUE_EQ( Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}, onError: {$const: 0}}}")), - convertExp->serialize(true)); + convertExp->serialize( + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})); } TEST_F(ExpressionConvertTest, ParseAndSerializeWithOnNull) { @@ -104,11 +106,12 @@ TEST_F(ExpressionConvertTest, ParseAndSerializeWithOnNull) { ASSERT_VALUE_EQ( Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}, onNull: {$const: 0}}}")), - convertExp->serialize(false)); + convertExp->serialize()); ASSERT_VALUE_EQ( Value(fromjson("{$convert: {input: '$path1', to: {$const: 'int'}, onNull: {$const: 0}}}")), - convertExp->serialize(true)); + convertExp->serialize( + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})); } TEST_F(ExpressionConvertTest, ConvertWithoutInputFailsToParse) { @@ -141,6 +144,32 @@ TEST_F(ExpressionConvertTest, ConvertWithoutToFailsToParse) { }); } +TEST_F(ExpressionConvertTest, RoundTripSerialization) { + auto expCtx = getExpCtx(); + + // Round-trip serialization of an argument that *looks* like an expression. + auto spec = BSON("$convert" << BSON("input" << BSON("$literal" << BSON("$toString" + << "this is a string")) + << "to" + << "string")); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + auto opts = SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue}; + auto serialized = convertExp->serialize(opts); + ASSERT_VALUE_EQ(Value(BSON("$convert" << BSON("input" << BSON("$const" << BSON("?" + << "?")) + << "to" + << BSON("$const" + << "string")))), + serialized); + + auto roundTrip = Expression::parseExpression(expCtx.get(), + serialized.getDocument().toBson(), + expCtx->variablesParseState) + ->serialize(opts); + ASSERT_VALUE_EQ(roundTrip, serialized); +} + TEST_F(ExpressionConvertTest, InvalidTypeNameFails) { auto expCtx = getExpCtx(); @@ -3445,4 +3474,82 @@ TEST_F(ExpressionConvertShortcutsTest, ThrowsOnConversionFailure) { } } // namespace ExpressionConvertShortcutsTest + +namespace ExpressionConvertSerializationTest { + +using ExpressionConvertSerializationTest = AggregationContextFixture; + +TEST_F(ExpressionConvertSerializationTest, ConvertToTypeSerializesCorrectly) { + auto expCtx = getExpCtx(); + auto spec = fromjson("{$convert: {input: '$funnyTest', to: 'bool'}}"); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'bool'}}}")), + convertExp->serialize(SerializationOptions{})); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'bool'}}}")), + convertExp->serialize(SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: 'bool'}}")), + convertExp->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +TEST_F(ExpressionConvertSerializationTest, ConvertToDollarSerializesCorrectly) { + auto expCtx = getExpCtx(); + auto spec = fromjson("{$convert: {input: '$funnyTest', to: {$add: [7, 2]}}}"); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + ASSERT_VALUE_EQ( + Value( + fromjson("{$convert: {input: '$funnyTest', to: {$add: [{$const: 7}, {$const: 2}]}}}")), + convertExp->serialize(SerializationOptions{})); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$add: [1, 1]}}}")), + convertExp->serialize(SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$add: '?array<?number>'}}}")), + convertExp->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +TEST_F(ExpressionConvertSerializationTest, ToStringSerializesCorrectly) { + auto expCtx = getExpCtx(); + + auto spec = BSON("$toString" + << "$funnyTest"); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'string'}}}")), + convertExp->serialize(SerializationOptions{})); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'string'}}}")), + convertExp->serialize(SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: 'string'}}")), + convertExp->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +TEST_F(ExpressionConvertSerializationTest, ToBoolSerializesCorrectly) { + auto expCtx = getExpCtx(); + + auto spec = BSON("$toBool" + << "$funnyTest"); + auto convertExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'bool'}}}")), + convertExp->serialize(SerializationOptions{})); + + ASSERT_VALUE_EQ( + Value(fromjson("{$convert: {input: '$funnyTest', to: {$const: 'bool'}}}")), + convertExp->serialize(SerializationOptions::kRepresentativeQueryShapeSerializeOptions)); + + ASSERT_VALUE_EQ(Value(fromjson("{$convert: {input: '$funnyTest', to: 'bool'}}")), + convertExp->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions)); +} + +} // namespace ExpressionConvertSerializationTest + } // namespace mongo diff --git a/src/mongo/db/pipeline/expression_date_test.cpp b/src/mongo/db/pipeline/expression_date_test.cpp index 9205ea29633..c4b7a01db42 100644 --- a/src/mongo/db/pipeline/expression_date_test.cpp +++ b/src/mongo/db/pipeline/expression_date_test.cpp @@ -64,8 +64,10 @@ TEST_F(ExpressionDateFromPartsTest, SerializesToObjectSyntax) { {"second", Document{{"$const", 15}}}, {"millisecond", Document{{"$const", 414}}}, {"timezone", Document{{"$const", "America/Los_Angeles"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } TEST_F(ExpressionDateFromPartsTest, OptimizesToConstantIfAllInputsAreConstant) { @@ -183,8 +185,10 @@ TEST_F(ExpressionDateToPartsTest, SerializesToObjectSyntax) { Document{{"date", Document{{"$const", Date_t{}}}}, {"timezone", Document{{"$const", "Europe/London"_sd}}}, {"iso8601", Document{{"$const", false}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } TEST_F(ExpressionDateToPartsTest, OptimizesToConstantIfAllInputsAreConstant) { @@ -430,22 +434,28 @@ TEST_F(DateExpressionTest, SerializesToObjectSyntax) { Value(Document{{expName, Document{{"date", Document{{"$const", Date_t{}}}}, {"timezone", Document{{"$const", "Europe/London"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); // Test that it serializes to the full format if given a date. spec = BSON(expName << Date_t{}); expectedSerialization = Value(Document{{expName, Document{{"date", Document{{"$const", Date_t{}}}}}}}); dateExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); // Test that it serializes to the full format if given a date within an array. spec = BSON(expName << BSON_ARRAY(Date_t{})); dateExp = Expression::parseExpression(expCtx.get(), spec, expCtx->variablesParseState); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } } @@ -598,8 +608,10 @@ TEST_F(ExpressionDateToStringTest, SerializesToObjectSyntax) { {"timezone", Document{{"$const", "Europe/London"_sd}}}, {"onNull", Document{{"$const", "nullDefault"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } TEST_F(ExpressionDateToStringTest, OptimizesToConstantIfAllInputsAreConstant) { @@ -824,8 +836,10 @@ TEST_F(ExpressionDateFromStringTest, SerializesToObjectSyntax) { Document{{"$dateFromString", Document{{"dateString", Document{{"$const", "2017-07-04T13:06:44Z"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); // Test that it serializes to the full format if given an object specification. spec = BSON("$dateFromString" << BSON("dateString" @@ -838,8 +852,10 @@ TEST_F(ExpressionDateFromStringTest, SerializesToObjectSyntax) { Document{{"dateString", Document{{"$const", "2017-07-04T13:06:44Z"_sd}}}, {"timezone", Document{{"$const", "Europe/London"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); spec = BSON("$dateFromString" << BSON("dateString" << "2017-07-04T13:06:44Z" @@ -854,8 +870,10 @@ TEST_F(ExpressionDateFromStringTest, SerializesToObjectSyntax) { {"timezone", Document{{"$const", "Europe/London"_sd}}}, {"format", Document{{"$const", "%Y-%d-%mT%H:%M:%S"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); spec = BSON("$dateFromString" << BSON("dateString" << "2017-07-04T13:06:44Z" @@ -876,8 +894,10 @@ TEST_F(ExpressionDateFromStringTest, SerializesToObjectSyntax) { {"onNull", Document{{"$const", "nullDefault"_sd}}}, {"onError", Document{{"$const", "errorDefault"_sd}}}}}}); - ASSERT_VALUE_EQ(dateExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateExp->serialize(), expectedSerialization); } TEST_F(ExpressionDateFromStringTest, OptimizesToConstantIfAllInputsAreConstant) { @@ -1416,13 +1436,15 @@ void assertParsesAndSerializesExpression(boost::intrusive_ptr<ExpressionContextF const auto expression = Expression::parseExpression(expCtx.get(), expressionSpec, expCtx->variablesParseState); const auto expectedSerialization = Value(expectedSerializedExpressionSpec); - ASSERT_VALUE_EQ(expression->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(expression->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(expression->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(expression->serialize(), expectedSerialization); // Verify that parsed and then serialized expression is the same. ASSERT_VALUE_EQ(Expression::parseExpression( expCtx.get(), expectedSerializedExpressionSpec, expCtx->variablesParseState) - ->serialize(false), + ->serialize(), expectedSerialization); } @@ -1893,8 +1915,10 @@ TEST_F(ExpressionDateArithmeticsTest, SerializesToObject) { Document{{"startDate", Document{{"$const", Date_t{}}}}, {"unit", Document{{"$const", "day"_sd}}}, {"amount", Document{{"$const", 1}}}}}}); - ASSERT_VALUE_EQ(dateAddExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateAddExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateAddExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateAddExp->serialize(), expectedSerialization); // with timezone doc = BSON(expName << BSON("startDate" << Date_t{} << "unit" @@ -1908,8 +1932,10 @@ TEST_F(ExpressionDateArithmeticsTest, SerializesToObject) { {"unit", Document{{"$const", "day"_sd}}}, {"amount", Document{{"$const", -1}}}, {"timezone", Document{{"$const", "America/New_York"_sd}}}}}}); - ASSERT_VALUE_EQ(dateAddExp->serialize(true), expectedSerialization); - ASSERT_VALUE_EQ(dateAddExp->serialize(false), expectedSerialization); + ASSERT_VALUE_EQ(dateAddExp->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}), + expectedSerialization); + ASSERT_VALUE_EQ(dateAddExp->serialize(), expectedSerialization); } } diff --git a/src/mongo/db/pipeline/expression_field_path_test.cpp b/src/mongo/db/pipeline/expression_field_path_test.cpp index 9fa73620dfe..b04484c47ab 100644 --- a/src/mongo/db/pipeline/expression_field_path_test.cpp +++ b/src/mongo/db/pipeline/expression_field_path_test.cpp @@ -200,6 +200,72 @@ TEST(FieldPath, ScalarVariableWithDottedFieldPathOptimizesToConstantMissingValue ASSERT_VALUE_EQ(Value(), constantExpr->getValue()); } +TEST(FieldPath, SerializeWithRedaction) { + SerializationOptions options = SerializationOptions::kMarkIdentifiers_FOR_TEST; + + auto expCtx = ExpressionContextForTest{}; + intrusive_ptr<Expression> expression = + ExpressionFieldPath::createPathFromString(&expCtx, "bar", expCtx.variablesParseState); + ASSERT_VALUE_EQ_AUTO( // NOLINT + "\"$HASH<bar>\"", + expression->serialize(options)); + + // Repeat with a dotted path. + expression = + ExpressionFieldPath::createPathFromString(&expCtx, "a.b.c", expCtx.variablesParseState); + ASSERT_VALUE_EQ_AUTO( // NOLINT + "\"$HASH<a>.HASH<b>.HASH<c>\"", + expression->serialize(options)); + + auto expr = [&](const std::string& json) { + return Expression::parseExpression(&expCtx, fromjson(json), expCtx.variablesParseState); + }; + + // Expression with multiple field paths. + expression = expr(R"({$and: [{$gt: ["$foo", 5]}, {$lt: ["$foo", 10]}]})"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$and": [ + { + "$gt": [ + "$HASH<foo>", + { + "$const": 5 + } + ] + }, + { + "$lt": [ + "$HASH<foo>", + { + "$const": 10 + } + ] + } + ] + })", + expression->serialize(options).getDocument()); + + // Test that a variable followed by user fields is properly hashed. + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + + expression = expr(R"({$gt: ["$$ROOT.a.b", 5]})"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$gt":["$$ROOT.HASH<a>.HASH<b>","?number"]})", + expression->serialize(options).getDocument()); + + expression = expr(R"({$gt: ["$foo", "$$NOW"]})"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$gt":["$HASH<foo>","$$NOW"]})", + expression->serialize(options).getDocument()); + + // Repeat the above test with a dotted path. + expression = expr(R"({$gt: ["$foo.a.b", "$$NOW"]})"); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$gt":["$HASH<foo>.HASH<a>.HASH<b>","$$NOW"]})", + expression->serialize(options).getDocument()); +} + /** The field path itself is a dependency. */ class Dependencies { public: @@ -424,7 +490,7 @@ public: ExpressionFieldPath::deprecatedCreate(&expCtx, "a.b.c"); ASSERT_BSONOBJ_BINARY_EQ(BSON("foo" << "$a.b.c"), - BSON("foo" << expression->serialize(false))); + BSON("foo" << expression->serialize())); } }; @@ -436,7 +502,7 @@ public: intrusive_ptr<Expression> expression = ExpressionFieldPath::deprecatedCreate(&expCtx, "a.b.c"); BSONArrayBuilder bab; - bab << expression->serialize(false); + bab << expression->serialize(); ASSERT_BSONOBJ_BINARY_EQ(BSON_ARRAY("$a.b.c"), bab.arr()); } }; diff --git a/src/mongo/db/pipeline/expression_find_internal.h b/src/mongo/db/pipeline/expression_find_internal.h index 68d535e5b28..36f88ded4aa 100644 --- a/src/mongo/db/pipeline/expression_find_internal.h +++ b/src/mongo/db/pipeline/expression_find_internal.h @@ -78,7 +78,7 @@ public: return visitor->visit(this); } - Value serialize(bool explain) const final { + Value serialize(const SerializationOptions& options) const final { MONGO_UNREACHABLE; } @@ -153,7 +153,7 @@ public: return visitor->visit(this); } - Value serialize(bool explain) const final { + Value serialize(const SerializationOptions& options) const final { MONGO_UNREACHABLE; } @@ -214,7 +214,7 @@ public: return visitor->visit(this); } - Value serialize(bool explain) const final { + Value serialize(const SerializationOptions& options) const final { MONGO_UNREACHABLE; } diff --git a/src/mongo/db/pipeline/expression_function.cpp b/src/mongo/db/pipeline/expression_function.cpp index c53b0004b31..7bf43ffd1c9 100644 --- a/src/mongo/db/pipeline/expression_function.cpp +++ b/src/mongo/db/pipeline/expression_function.cpp @@ -46,16 +46,18 @@ ExpressionFunction::ExpressionFunction(ExpressionContext* const expCtx, expCtx->sbeCompatible = false; } -Value ExpressionFunction::serialize(bool explain) const { - MutableDocument d; - d["body"] = Value(_funcSource); - d["args"] = Value(_passedArgs->serialize(explain)); - d["lang"] = Value(_lang); +Value ExpressionFunction::serialize(const SerializationOptions& options) const { + MutableDocument innerOpts(Document{{"body"_sd, options.serializeLiteral(_funcSource)}, + {"args"_sd, _passedArgs->serialize(options)}, + // "lang" is purposefully not treated as a literal since it + // is more of a selection of an enum + {"lang"_sd, _lang}}); + // This field will only be seralized when desugaring $where in $expr + $_internalJs if (_assignFirstArgToThis) { - d["_internalSetObjToThis"] = Value(_assignFirstArgToThis); + innerOpts["_internalSetObjToThis"] = options.serializeLiteral(_assignFirstArgToThis); } - return Value(Document{{kExpressionName, d.freezeToValue()}}); + return Value(Document{{kExpressionName, innerOpts.freezeToValue()}}); } void ExpressionFunction::_doAddDependencies(mongo::DepsTracker* deps) const { diff --git a/src/mongo/db/pipeline/expression_function.h b/src/mongo/db/pipeline/expression_function.h index b8d6b95e35a..a4d50dcb588 100644 --- a/src/mongo/db/pipeline/expression_function.h +++ b/src/mongo/db/pipeline/expression_function.h @@ -69,7 +69,7 @@ public: Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); diff --git a/src/mongo/db/pipeline/expression_function_test.cpp b/src/mongo/db/pipeline/expression_function_test.cpp new file mode 100644 index 00000000000..aa631a53c77 --- /dev/null +++ b/src/mongo/db/pipeline/expression_function_test.cpp @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/exec/document_value/document_value_test_util.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/db/pipeline/expression_function.h" +#include "mongo/db/query/query_shape/serialization_options.h" +#include "mongo/dbtests/dbtests.h" + +namespace mongo { + +namespace { + +TEST(ExpressionFunction, SerializeAndRedactArgs) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + + auto expCtx = ExpressionContextForTest(); + auto expr = BSON("$function" << BSON("body" + << "function(age) {return age >= 21;}" + << "args" << BSON_ARRAY("$age") << "lang" + << "js")); + VariablesParseState vps = expCtx.variablesParseState; + auto exprFunc = ExpressionFunction::parse(&expCtx, expr.firstElement(), vps); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({"$function":{"body":"?string","args":["$HASH<age>"],"lang":"js"}})", + exprFunc->serialize(options).getDocument()); +} +} // namespace +} // namespace mongo diff --git a/src/mongo/db/pipeline/expression_js_emit.cpp b/src/mongo/db/pipeline/expression_js_emit.cpp index ca6ba238c6e..dd43f587016 100644 --- a/src/mongo/db/pipeline/expression_js_emit.cpp +++ b/src/mongo/db/pipeline/expression_js_emit.cpp @@ -130,10 +130,10 @@ boost::intrusive_ptr<Expression> ExpressionInternalJsEmit::parse(ExpressionConte return new ExpressionInternalJsEmit(expCtx, std::move(thisRef), std::move(funcSourceString)); } -Value ExpressionInternalJsEmit::serialize(bool explain) const { +Value ExpressionInternalJsEmit::serialize(const SerializationOptions& options) const { return Value( Document{{kExpressionName, - Document{{"eval", _funcSource}, {"this", _thisRef->serialize(explain)}}}}); + Document{{"eval", _funcSource}, {"this", _thisRef->serialize(options)}}}}); } Value ExpressionInternalJsEmit::evaluate(const Document& root, Variables* variables) const { diff --git a/src/mongo/db/pipeline/expression_js_emit.h b/src/mongo/db/pipeline/expression_js_emit.h index e19907de84e..5202582d8ee 100644 --- a/src/mongo/db/pipeline/expression_js_emit.h +++ b/src/mongo/db/pipeline/expression_js_emit.h @@ -55,7 +55,7 @@ public: Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); diff --git a/src/mongo/db/pipeline/expression_let_test.cpp b/src/mongo/db/pipeline/expression_let_test.cpp new file mode 100644 index 00000000000..6b851f18523 --- /dev/null +++ b/src/mongo/db/pipeline/expression_let_test.cpp @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/exec/document_value/document_value_test_util.h" +#include "mongo/db/pipeline/expression.h" +#include "mongo/db/pipeline/expression_context_for_test.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace ExpressionTests { +namespace { + +TEST(RedactionTest, ExpressionLet) { + SerializationOptions options = SerializationOptions::kMarkIdentifiers_FOR_TEST; + + auto expCtx = ExpressionContextForTest{}; + + auto expression = Expression::parseExpression(&expCtx, + fromjson(R"( + {$let: {vars: {foo: 35}, in: {$gt: ["$$foo", 23]}}} + )"), + expCtx.variablesParseState); + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$let": { + "vars": { + "HASH<foo>": { + "$const": 35 + } + }, + "in": { + "$gt": [ + "$$HASH<foo>", + { + "$const": 23 + } + ] + } + } + })", + expression->serialize(options).getDocument()); + + expression = Expression::parseExpression(&expCtx, + fromjson(R"( + {$let: {vars: {foo: 35, myNow: "$$NOW", obj: {hello: 23}}, in: {$gt: ["$$myNow", "$$obj.hello"]}}} + )"), + expCtx.variablesParseState); + + ASSERT_DOCUMENT_EQ_AUTO( // NOLINT + R"({ + "$let": { + "vars": { + "HASH<foo>": { + "$const": 35 + }, + "HASH<myNow>": "$$NOW", + "HASH<obj>": { + "HASH<hello>": { + "$const": 23 + } + } + }, + "in": { + "$gt": [ + "$$HASH<myNow>", + "$$HASH<obj>.HASH<hello>" + ] + } + } + })", + expression->serialize(options).getDocument()); +} +} // namespace +} // namespace ExpressionTests +} // namespace mongo diff --git a/src/mongo/db/pipeline/expression_nary_test.cpp b/src/mongo/db/pipeline/expression_nary_test.cpp index 15b8e824f7c..cc01d807462 100644 --- a/src/mongo/db/pipeline/expression_nary_test.cpp +++ b/src/mongo/db/pipeline/expression_nary_test.cpp @@ -120,7 +120,7 @@ static BSONObj constify(const BSONObj& obj, bool parentIsArray = false) { /** Convert Expression to BSON. */ static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } class ExpressionBaseTest : public unittest::Test { @@ -249,15 +249,59 @@ TEST_F(ExpressionNaryTest, ValidateObjectExpressionDependency) { TEST_F(ExpressionNaryTest, SerializationToBsonObj) { _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(5))); ASSERT_BSONOBJ_EQ(BSON("foo" << BSON("$testable" << BSON_ARRAY(BSON("$const" << 5)))), - BSON("foo" << _notAssociativeNorCommutative->serialize(false))); + BSON("foo" << _notAssociativeNorCommutative->serialize())); } TEST_F(ExpressionNaryTest, SerializationToBsonArr) { _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(5))); ASSERT_BSONOBJ_EQ(constify(BSON_ARRAY(BSON("$testable" << BSON_ARRAY(5)))), - BSON_ARRAY(_notAssociativeNorCommutative->serialize(false))); + BSON_ARRAY(_notAssociativeNorCommutative->serialize())); } +TEST_F(ExpressionNaryTest, RedactsCorrectlyWithConstantArguments) { + _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(5))); + _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(10))); + _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(15))); + + SerializationOptions opts; + + // The default shape should wrap the constants in $const. + ASSERT_BSONOBJ_EQ( + BSON("foo" << BSON("$testable" << BSON_ARRAY(BSON("$const" << 5) << BSON("$const" << 10) + << BSON("$const" << 15)))), + BSON("foo" << _notAssociativeNorCommutative->serialize(opts))); + + // The representative shape should be an array of raw constants (i.e. not wrapped in $const). + opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue; + ASSERT_BSONOBJ_EQ(BSON("foo" << BSON("$testable" << BSON_ARRAY(1 << 1 << 1))), + BSON("foo" << _notAssociativeNorCommutative->serialize(opts))); +} + +TEST_F(ExpressionNaryTest, RedactsCorrectlyWithMixedArguments) { + VariablesParseState vps = expCtx.variablesParseState; + _notAssociativeNorCommutative->addOperand(ExpressionConstant::create(&expCtx, Value(5))); + _notAssociativeNorCommutative->addOperand( + Expression::parseExpression(&expCtx, BSON("$sum" << BSON_ARRAY(1 << 2)), vps)); + _notAssociativeNorCommutative->addOperand(ExpressionFieldPath::parse(&expCtx, "$b", vps)); + + SerializationOptions opts; + + // The default shape should wrap the constants in $const. + ASSERT_BSONOBJ_EQ(BSON("foo" << BSON("$testable" << BSON_ARRAY( + BSON("$const" << 5) + << BSON("$sum" << BSON_ARRAY(BSON("$const" << 1) + << BSON("$const" << 2))) + << "$b"))), + BSON("foo" << _notAssociativeNorCommutative->serialize(opts))); + + // The representative shape should not wrap the constant in $const. + opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue; + ASSERT_BSONOBJ_EQ(BSON("foo" << BSON("$testable" << BSON_ARRAY( + 1 << BSON("$sum" << BSON_ARRAY(1 << 1)) << "$b"))), + BSON("foo" << _notAssociativeNorCommutative->serialize(opts))); +} + + // Verify that the internal operands are optimized TEST_F(ExpressionNaryTest, InternalOperandOptimizationIsDone) { BSONArray spec = BSON_ARRAY(BSON("$and" << BSONArray()) << "$abc"); diff --git a/src/mongo/db/pipeline/expression_object_test.cpp b/src/mongo/db/pipeline/expression_object_test.cpp index 7246e9569e3..b308b94efaa 100644 --- a/src/mongo/db/pipeline/expression_object_test.cpp +++ b/src/mongo/db/pipeline/expression_object_test.cpp @@ -64,7 +64,7 @@ TEST(ExpressionObjectParse, ShouldAcceptEmptyObject) { auto expCtx = ExpressionContextForTest{}; VariablesParseState vps = expCtx.variablesParseState; auto object = ExpressionObject::parse(&expCtx, BSONObj(), vps); - ASSERT_VALUE_EQ(Value(Document{}), object->serialize(false)); + ASSERT_VALUE_EQ(Value(Document{}), object->serialize()); } TEST(ExpressionObjectParse, ShouldAcceptLiteralsAsValues) { @@ -77,7 +77,7 @@ TEST(ExpressionObjectParse, ShouldAcceptLiteralsAsValues) { vps); auto expectedResult = Value(Document{{"a", literal(5)}, {"b", literal("string"_sd)}, {"c", literal(BSONNULL)}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ExpressionObjectParse, ShouldAccept_idAsFieldName) { @@ -85,7 +85,7 @@ TEST(ExpressionObjectParse, ShouldAccept_idAsFieldName) { VariablesParseState vps = expCtx.variablesParseState; auto object = ExpressionObject::parse(&expCtx, BSON("_id" << 5), vps); auto expectedResult = Value(Document{{"_id", literal(5)}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ExpressionObjectParse, ShouldAcceptFieldNameContainingDollar) { @@ -93,7 +93,7 @@ TEST(ExpressionObjectParse, ShouldAcceptFieldNameContainingDollar) { VariablesParseState vps = expCtx.variablesParseState; auto object = ExpressionObject::parse(&expCtx, BSON("a$b" << 5), vps); auto expectedResult = Value(Document{{"a$b", literal(5)}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ExpressionObjectParse, ShouldAcceptNestedObjects) { @@ -104,7 +104,7 @@ TEST(ExpressionObjectParse, ShouldAcceptNestedObjects) { auto expectedResult = Value(Document{{"a", Document{{"b", literal(1)}}}, {"c", Document{{"d", Document{{"e", literal(1)}, {"f", literal(1)}}}}}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ExpressionObjectParse, ShouldAcceptArrays) { @@ -113,15 +113,14 @@ TEST(ExpressionObjectParse, ShouldAcceptArrays) { auto object = ExpressionObject::parse(&expCtx, fromjson("{a: [1, 2]}"), vps); auto expectedResult = Value(Document{{"a", vector<Value>{Value(literal(1)), Value(literal(2))}}}); - ASSERT_VALUE_EQ(expectedResult, object->serialize(false)); + ASSERT_VALUE_EQ(expectedResult, object->serialize()); } TEST(ObjectParsing, ShouldAcceptExpressionAsValue) { auto expCtx = ExpressionContextForTest{}; VariablesParseState vps = expCtx.variablesParseState; auto object = ExpressionObject::parse(&expCtx, BSON("a" << BSON("$and" << BSONArray())), vps); - ASSERT_VALUE_EQ(object->serialize(false), - Value(Document{{"a", Document{{"$and", BSONArray()}}}})); + ASSERT_VALUE_EQ(object->serialize(), Value(Document{{"a", Document{{"$and", BSONArray()}}}})); } // diff --git a/src/mongo/db/pipeline/expression_or_test.cpp b/src/mongo/db/pipeline/expression_or_test.cpp index ca4f5fdf843..00d8a9deb74 100644 --- a/src/mongo/db/pipeline/expression_or_test.cpp +++ b/src/mongo/db/pipeline/expression_or_test.cpp @@ -77,7 +77,7 @@ static BSONObj toBson(const Value& value) { /** Convert Expression to BSON. */ static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } /** Convert Document to BSON. */ diff --git a/src/mongo/db/pipeline/expression_test.cpp b/src/mongo/db/pipeline/expression_test.cpp index 109fc4acf16..591225258e2 100644 --- a/src/mongo/db/pipeline/expression_test.cpp +++ b/src/mongo/db/pipeline/expression_test.cpp @@ -44,6 +44,7 @@ #include "mongo/db/pipeline/expression.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/collation/collator_interface_mock.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/dbtests/dbtests.h" #include "mongo/idl/server_parameter_test_util.h" #include "mongo/logv2/log.h" @@ -123,7 +124,7 @@ static BSONObj toBson(const Value& value) { /** Convert Expression to BSON. */ static BSONObj expressionToBson(const intrusive_ptr<Expression>& expression) { - return BSON("" << expression->serialize(false)).firstElement().embeddedObject().getOwned(); + return BSON("" << expression->serialize()).firstElement().embeddedObject().getOwned(); } /** Convert Document to BSON. */ @@ -169,7 +170,6 @@ void parseAndVerifyResults( ASSERT_VALUE_EQ(expr->evaluate({}, &expCtx.variables), expected); } - /** * Generates a random double with a variable number of decimal places between 1 and 15. */ @@ -710,7 +710,7 @@ public: private: static BSONObj toBsonObj(const intrusive_ptr<Expression>& expression) { - return BSON("field" << expression->serialize(false)); + return BSON("field" << expression->serialize()); } }; @@ -729,7 +729,7 @@ public: private: static BSONArray toBsonArray(const intrusive_ptr<Expression>& expression) { BSONArrayBuilder bab; - bab << expression->serialize(false); + bab << expression->serialize(); return bab.arr(); } }; @@ -805,7 +805,7 @@ public: private: static BSONObj toBsonObj(const intrusive_ptr<Expression>& expression) { - return BSON("field" << expression->serialize(false)); + return BSON("field" << expression->serialize()); } }; @@ -822,7 +822,7 @@ public: private: static BSONObj toBsonArray(const intrusive_ptr<Expression>& expression) { BSONArrayBuilder bab; - bab << expression->serialize(false); + bab << expression->serialize(); return bab.obj(); } }; @@ -840,7 +840,26 @@ TEST(ExpressionConstantTest, ConstantOfValueMissingSerializesToRemoveSystemVar) intrusive_ptr<Expression> expression = ExpressionConstant::create(&expCtx, Value()); ASSERT_BSONOBJ_BINARY_EQ(BSON("field" << "$$REMOVE"), - BSON("field" << expression->serialize(false))); + BSON("field" << expression->serialize())); +} + +TEST(ExpressionConstantTest, ConstantRedaction) { + SerializationOptions options; + options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString; + + // Test that a constant is replaced. + auto expCtx = ExpressionContextForTest{}; + intrusive_ptr<Expression> expression = ExpressionConstant::create(&expCtx, Value("my_ssn"_sd)); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"field":"?string"})", + BSON("field" << expression->serialize(options))); + + auto expressionBSON = BSON("$and" << BSON_ARRAY(BSON("$gt" << BSON_ARRAY("$foo" << 5)) + << BSON("$lt" << BSON_ARRAY("$foo" << 10)))); + expression = Expression::parseExpression(&expCtx, expressionBSON, expCtx.variablesParseState); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"field":{"$and":[{"$gt":["$foo","?number"]},{"$lt":["$foo","?number"]}]}})", + BSON("field" << expression->serialize(options))); } } // namespace Constant @@ -1583,7 +1602,7 @@ TEST(ParseExpression, ShouldRecognizeConstExpression) { auto resultExpression = parseExpression(BSON("$const" << 5)); auto constExpression = dynamic_cast<ExpressionConstant*>(resultExpression.get()); ASSERT_TRUE(constExpression); - ASSERT_VALUE_EQ(constExpression->serialize(false), Value(Document{{"$const", 5}})); + ASSERT_VALUE_EQ(constExpression->serialize(), Value(Document{{"$const", 5}})); } TEST(ParseExpression, ShouldRejectUnknownExpression) { @@ -1617,15 +1636,14 @@ TEST(ParseExpression, ShouldParseExpressionWithMultipleArguments) { ASSERT_TRUE(strCaseCmpExpression); vector<Value> arguments = {Value(Document{{"$const", "foo"_sd}}), Value(Document{{"$const", "FOO"_sd}})}; - ASSERT_VALUE_EQ(strCaseCmpExpression->serialize(false), - Value(Document{{"$strcasecmp", arguments}})); + ASSERT_VALUE_EQ(strCaseCmpExpression->serialize(), Value(Document{{"$strcasecmp", arguments}})); } TEST(ParseExpression, ShouldParseExpressionWithNoArguments) { auto resultExpression = parseExpression(BSON("$and" << BSONArray())); auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", vector<Value>{}}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", vector<Value>{}}})); } TEST(ParseExpression, ShouldParseExpressionWithOneArgument) { @@ -1633,7 +1651,7 @@ TEST(ParseExpression, ShouldParseExpressionWithOneArgument) { auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", arguments}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", arguments}})); } TEST(ParseExpression, ShouldAcceptArgumentWithoutArrayForVariadicExpressions) { @@ -1641,7 +1659,7 @@ TEST(ParseExpression, ShouldAcceptArgumentWithoutArrayForVariadicExpressions) { auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", arguments}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", arguments}})); } TEST(ParseExpression, ShouldAcceptArgumentWithoutArrayAsSingleArgument) { @@ -1649,7 +1667,7 @@ TEST(ParseExpression, ShouldAcceptArgumentWithoutArrayAsSingleArgument) { auto notExpression = dynamic_cast<ExpressionNot*>(resultExpression.get()); ASSERT_TRUE(notExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(notExpression->serialize(false), Value(Document{{"$not", arguments}})); + ASSERT_VALUE_EQ(notExpression->serialize(), Value(Document{{"$not", arguments}})); } TEST(ParseExpression, ShouldAcceptObjectAsSingleArgument) { @@ -1657,7 +1675,7 @@ TEST(ParseExpression, ShouldAcceptObjectAsSingleArgument) { auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", arguments}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", arguments}})); } TEST(ParseExpression, ShouldAcceptObjectInsideArrayAsSingleArgument) { @@ -1665,7 +1683,7 @@ TEST(ParseExpression, ShouldAcceptObjectInsideArrayAsSingleArgument) { auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); vector<Value> arguments = {Value(Document{{"$const", 1}})}; - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", arguments}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", arguments}})); } } // namespace Expression @@ -1691,7 +1709,7 @@ TEST(ParseOperand, ShouldRecognizeFieldPath) { << "$field")); auto fieldPathExpression = dynamic_cast<ExpressionFieldPath*>(resultExpression.get()); ASSERT_TRUE(fieldPathExpression); - ASSERT_VALUE_EQ(fieldPathExpression->serialize(false), Value("$field"_sd)); + ASSERT_VALUE_EQ(fieldPathExpression->serialize(), Value("$field"_sd)); } TEST(ParseOperand, ShouldRecognizeStringLiteral) { @@ -1699,7 +1717,7 @@ TEST(ParseOperand, ShouldRecognizeStringLiteral) { << "foo")); auto constantExpression = dynamic_cast<ExpressionConstant*>(resultExpression.get()); ASSERT_TRUE(constantExpression); - ASSERT_VALUE_EQ(constantExpression->serialize(false), Value(Document{{"$const", "foo"_sd}})); + ASSERT_VALUE_EQ(constantExpression->serialize(), Value(Document{{"$const", "foo"_sd}})); } TEST(ParseOperand, ShouldRecognizeNestedArray) { @@ -1709,21 +1727,21 @@ TEST(ParseOperand, ShouldRecognizeNestedArray) { ASSERT_TRUE(arrayExpression); vector<Value> expectedSerializedArray = {Value(Document{{"$const", "foo"_sd}}), Value("$field"_sd)}; - ASSERT_VALUE_EQ(arrayExpression->serialize(false), Value(expectedSerializedArray)); + ASSERT_VALUE_EQ(arrayExpression->serialize(), Value(expectedSerializedArray)); } TEST(ParseOperand, ShouldRecognizeNumberLiteral) { auto resultExpression = parseOperand(BSON("" << 5)); auto constantExpression = dynamic_cast<ExpressionConstant*>(resultExpression.get()); ASSERT_TRUE(constantExpression); - ASSERT_VALUE_EQ(constantExpression->serialize(false), Value(Document{{"$const", 5}})); + ASSERT_VALUE_EQ(constantExpression->serialize(), Value(Document{{"$const", 5}})); } TEST(ParseOperand, ShouldRecognizeNestedExpression) { auto resultExpression = parseOperand(BSON("" << BSON("$and" << BSONArray()))); auto andExpression = dynamic_cast<ExpressionAnd*>(resultExpression.get()); ASSERT_TRUE(andExpression); - ASSERT_VALUE_EQ(andExpression->serialize(false), Value(Document{{"$and", vector<Value>{}}})); + ASSERT_VALUE_EQ(andExpression->serialize(), Value(Document{{"$and", vector<Value>{}}})); } } // namespace Operand @@ -1764,7 +1782,7 @@ public: if (ValueComparator().evaluate(result != expected)) { string errMsg = str::stream() << "for expression " << field.first.toString() << " with argument " - << args.toString() << " full tree: " << expr->serialize(false).toString() + << args.toString() << " full tree: " << expr->serialize().toString() << " expected: " << expected.toString() << " but got: " << result.toString(); FAIL(errMsg); @@ -2576,7 +2594,7 @@ TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectly) { auto expression = ExpressionFieldPath::parse(&expCtx, "$$REMOVE", vps); ASSERT_BSONOBJ_EQ(BSON("foo" << "$$REMOVE"), - BSON("foo" << expression->serialize(false))); + BSON("foo" << expression->serialize())); } TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectlyWithTrailingPath) { @@ -2585,7 +2603,7 @@ TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectlyWithTrailingPath) { auto expression = ExpressionFieldPath::parse(&expCtx, "$$REMOVE.a.b", vps); ASSERT_BSONOBJ_EQ(BSON("foo" << "$$REMOVE.a.b"), - BSON("foo" << expression->serialize(false))); + BSON("foo" << expression->serialize())); } TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectlyAfterOptimization) { @@ -2596,7 +2614,7 @@ TEST(BuiltinRemoveVariableTest, RemoveSerializesCorrectlyAfterOptimization) { ASSERT(dynamic_cast<ExpressionConstant*>(optimizedExpression.get())); ASSERT_BSONOBJ_EQ(BSON("foo" << "$$REMOVE"), - BSON("foo" << optimizedExpression->serialize(false))); + BSON("foo" << optimizedExpression->serialize())); } } // namespace BuiltinRemoveVariable @@ -2814,7 +2832,7 @@ public: if (ValueComparator().evaluate(result != expected)) { string errMsg = str::stream() << "for expression " << field.first.toString() << " with argument " - << args.toString() << " full tree: " << expr->serialize(false).toString() + << args.toString() << " full tree: " << expr->serialize().toString() << " expected: " << expected.toString() << " but got: " << result.toString(); FAIL(errMsg); @@ -3505,7 +3523,7 @@ TEST(ExpressionRandom, Basic) { intrusive_ptr<Expression> expression = Expression::parseExpression(&expCtx, fromjson("{ $rand: {} }"), vps); - const std::string& serialized = expression->serialize(false).getDocument().toString(); + const std::string& serialized = expression->serialize().getDocument().toString(); ASSERT_EQ("{$rand: {}}", serialized); const auto randFn = [&expression, &expCtx]() -> double { @@ -3642,11 +3660,18 @@ TEST(ExpressionGetFieldTest, GetFieldSerializesStringArgumentCorrectly) { VariablesParseState vps = expCtx.variablesParseState; BSONObj expr = fromjson("{$meta: \"foo\"}"); auto expression = ExpressionGetField::parse(&expCtx, expr.firstElement(), vps); - ASSERT_BSONOBJ_EQ(BSON("ignoredField" << BSON("$getField" << BSON("field" << BSON("$const" - << "foo") - << "input" - << "$$CURRENT"))), - BSON("ignoredField" << expression->serialize(false))); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "ignoredField": { + "$getField": { + "field": { + "$const": "foo" + }, + "input": "$$CURRENT" + } + } + })", + BSON("ignoredField" << expression->serialize())); } TEST(ExpressionGetFieldTest, GetFieldSerializesCorrectly) { @@ -3654,12 +3679,220 @@ TEST(ExpressionGetFieldTest, GetFieldSerializesCorrectly) { VariablesParseState vps = expCtx.variablesParseState; BSONObj expr = fromjson("{$meta: {\"field\": \"foo\", \"input\": {a: 1}}}"); auto expression = ExpressionGetField::parse(&expCtx, expr.firstElement(), vps); - ASSERT_BSONOBJ_EQ( - BSON("ignoredField" << BSON( - "$getField" << BSON("field" << BSON("$const" - << "foo") - << "input" << BSON("a" << BSON("$const" << 1))))), - BSON("ignoredField" << expression->serialize(false))); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "ignoredField": { + "$getField": { + "field": { + "$const": "foo" + }, + "input": { + "a": { + "$const": 1 + } + } + } + } + })", + BSON("ignoredField" << expression->serialize())); +} + +TEST(ExpressionGetFieldTest, GetFieldSerializesAndRedactsCorrectly) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto expCtx = ExpressionContextForTest{}; + VariablesParseState vps = expCtx.variablesParseState; + + BSONObj expressionBSON = BSON("$getField" << BSON("field" + << "a" + << "input" + << "$b")); + + auto expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"field":{"$getField":{"field":"HASH<a>","input":"$HASH<b>"}}})", + BSON("field" << expression->serialize(options))); + + // Test the shorthand syntax. + expressionBSON = BSON("$getField" + << "a"); + + expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({"field":{"$getField":{"field":"HASH<a>","input":"$$CURRENT"}}})", + BSON("field" << expression->serialize(options))); + + // Test a field with '.' characters. + expressionBSON = BSON("$getField" + << "a.b.c"); + + expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$getField": { + "field": "HASH<a>.HASH<b>.HASH<c>", + "input": "$$CURRENT" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Test a field with a '$' character. + expressionBSON = BSON("$getField" + << "a.$b.c"); + + expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$getField": { + "field": "HASH<a>.HASH<$b>.HASH<c>", + "input": "$$CURRENT" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Test a field with a trailing '.' character (invalid FieldPath). + expressionBSON = BSON("$getField" + << "a.b.c."); + + expression = ExpressionGetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$getField": { + "field": "HASH<invalidFieldPathPlaceholder>", + "input": "$$CURRENT" + } + } + })", + BSON("field" << expression->serialize(options))); +} + +TEST(ExpressionSetFieldTest, SetFieldRedactsCorrectly) { + SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST; + auto expCtx = ExpressionContextForTest{}; + VariablesParseState vps = expCtx.variablesParseState; + + // Test that a set field redacts properly. + BSONObj expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" + << "$b" + << "value" + << "$c")); + auto expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "$HASH<b>", + "value": "$HASH<c>" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Object as input. + expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" << BSON("a" << true) << "value" << 10)); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "?object", + "value": "?number" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Nested object as input. + expressionBSON = + BSON("$setField" << BSON("field" + << "a" + << "input" << BSON("a" << BSON("b" << 5)) << "value" << 10)); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "?object", + "value": "?number" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Object with field path in input. + expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" + << BSON("a" + << "$field") + << "value" << 10)); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": { + "HASH<a>": "$HASH<field>" + }, + "value": "?number" + } + } + })", + BSON("field" << expression->serialize(options))); + + // Object with field path in value. + expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" + << BSON("a" + << "b") + << "value" + << BSON("c" + << "$d"))); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "?object", + "value": { + "HASH<c>": "$HASH<d>" + } + } + } + })", + BSON("field" << expression->serialize(options))); + + // Array as input. + expressionBSON = BSON("$setField" << BSON("field" + << "a" + << "input" << BSON("a" << BSON_ARRAY(3 << 4 << 5)) + << "value" << 10)); + expression = ExpressionSetField::parse(&expCtx, expressionBSON.firstElement(), vps); + ASSERT_BSONOBJ_EQ_AUTO( // NOLINT + R"({ + "field": { + "$setField": { + "field": "HASH<a>", + "input": "?object", + "value": "?number" + } + } + })", + BSON("field" << expression->serialize(options))); } TEST(ExpressionSetFieldTest, SetFieldSerializesCorrectly) { @@ -3673,7 +3906,20 @@ TEST(ExpressionSetFieldTest, SetFieldSerializesCorrectly) { << "foo") << "input" << BSON("a" << BSON("$const" << 1)) << "value" << BSON("$const" << 24)))), - BSON("ignoredField" << expression->serialize(false))); + BSON("ignoredField" << expression->serialize())); +} + +TEST(ExpressionSetFieldTest, SetFieldRejectsNullCharInFieldArgument) { + auto expCtx = ExpressionContextForTest{}; + VariablesParseState vps = expCtx.variablesParseState; + auto fieldExpr = make_intrusive<ExpressionConstant>(&expCtx, Value("ab\0c"_sd)); + auto inputExpr = make_intrusive<ExpressionConstant>(&expCtx, Value(BSON("a" << 1))); + auto valueExpr = make_intrusive<ExpressionConstant>(&expCtx, Value(true)); + ASSERT_THROWS_CODE( + make_intrusive<ExpressionSetField>( + &expCtx, std::move(fieldExpr), std::move(inputExpr), std::move(valueExpr)), + AssertionException, + 9534700); } TEST(ExpressionIfNullTest, OptimizedExpressionIfNullShouldRemoveNullConstant) { @@ -3693,7 +3939,7 @@ TEST(ExpressionIfNullTest, auto expr = fromjson("{$ifNull: [null, \"$a\"]}"); auto exprIfNull = ExpressionIfNull::parse(&expCtx, expr.firstElement(), vps); auto optimizedNullRemoved = exprIfNull->optimize(); - ASSERT_VALUE_EQ(optimizedNullRemoved->serialize(false), Value("$a"_sd)); + ASSERT_VALUE_EQ(optimizedNullRemoved->serialize(), Value("$a"_sd)); } TEST(ExpressionIfNullTest, OptimizedExpressionIfNullShouldRemoveAllNullConstantsButLast) { @@ -4025,7 +4271,7 @@ TEST(ExpressionFLETest, TestBinData_RoundTrip) { ASSERT_VALUE_EQ(exprFle->evaluate({}, &expCtx.variables), Value(true)); // Verify it round trips - auto value = exprFle->serialize(false); + auto value = exprFle->serialize(); auto roundTripExpr = fromjson(R"({$_internalFleEq: { field: { @@ -4050,7 +4296,7 @@ TEST(ExpressionFLETest, TestBinData_RoundTrip) { "subType": "6" } } - } })"); + } })"); ASSERT_BSONOBJ_EQ(value.getDocument().toBson(), roundTripExpr); diff --git a/src/mongo/db/pipeline/expression_test_api_version.cpp b/src/mongo/db/pipeline/expression_test_api_version.cpp index 893ddf301be..08938f1afa8 100644 --- a/src/mongo/db/pipeline/expression_test_api_version.cpp +++ b/src/mongo/db/pipeline/expression_test_api_version.cpp @@ -89,7 +89,7 @@ boost::intrusive_ptr<Expression> ExpressionTestApiVersion::parse(ExpressionConte return new ExpressionTestApiVersion(expCtx, unstableField, deprecatedField); } -Value ExpressionTestApiVersion::serialize(bool explain) const { +Value ExpressionTestApiVersion::serialize(const SerializationOptions& options) const { return Value(Document{{"$_testApiVersion", Document{{"unstable", _unstable ? Value(_unstable) : Value()}, {"deprecated", _deprecated ? Value(_deprecated) : Value()}}}}); diff --git a/src/mongo/db/pipeline/expression_test_api_version.h b/src/mongo/db/pipeline/expression_test_api_version.h index cd5436d61b3..0b67556efbe 100644 --- a/src/mongo/db/pipeline/expression_test_api_version.h +++ b/src/mongo/db/pipeline/expression_test_api_version.h @@ -48,7 +48,7 @@ public: Value evaluate(const Document& root, Variables* variables) const final; - Value serialize(bool explain) const final; + Value serialize(const SerializationOptions& options) const final; void acceptVisitor(ExpressionMutableVisitor* visitor) final { return visitor->visit(this); diff --git a/src/mongo/db/pipeline/expression_test_api_version_test.cpp b/src/mongo/db/pipeline/expression_test_api_version_test.cpp index 05c9156ddf0..231a47fbc27 100644 --- a/src/mongo/db/pipeline/expression_test_api_version_test.cpp +++ b/src/mongo/db/pipeline/expression_test_api_version_test.cpp @@ -48,7 +48,7 @@ TEST_F(TestApiVersion, UnstableAcceptsBooleanValue) { expCtx->variablesParseState); ASSERT_VALUE_EQ(Value(DOC("$_testApiVersion" << DOC("unstable" << true))), - expression->serialize(false)); + expression->serialize()); } TEST_F(TestApiVersion, UnstableDoesNotAcceptNumericValue) { @@ -70,7 +70,7 @@ TEST_F(TestApiVersion, DeprecatedAcceptsBooleanValue) { expCtx->variablesParseState); ASSERT_VALUE_EQ(Value(DOC("$_testApiVersion" << DOC("deprecated" << true))), - expression->serialize(false)); + expression->serialize()); } TEST_F(TestApiVersion, DeprecatedDoesNotAcceptNumericValue) { diff --git a/src/mongo/db/pipeline/expression_trim_test.cpp b/src/mongo/db/pipeline/expression_trim_test.cpp index 620b371c5f7..bf5a5d63302 100644 --- a/src/mongo/db/pipeline/expression_trim_test.cpp +++ b/src/mongo/db/pipeline/expression_trim_test.cpp @@ -722,14 +722,16 @@ TEST(ExpressionTrimTest, DoesSerializeCorrectly) { BSON("$trim" << BSON("input" << " abc ")), expCtx.variablesParseState); - ASSERT_VALUE_EQ(trim->serialize(false), trim->serialize(true)); + ASSERT_VALUE_EQ(trim->serialize(), + trim->serialize(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})); ASSERT_VALUE_EQ( - trim->serialize(false), + trim->serialize(), Value(Document{{"$trim", Document{{"input", Document{{"$const", " abc "_sd}}}}}})); // Make sure we can re-parse it and evaluate it. auto reparsedTrim = Expression::parseExpression( - &expCtx, trim->serialize(false).getDocument().toBson(), expCtx.variablesParseState); + &expCtx, trim->serialize().getDocument().toBson(), expCtx.variablesParseState); ASSERT_VALUE_EQ(reparsedTrim->evaluate({}, &expCtx.variables), Value("abc"_sd)); // Use $ltrim, and specify the 'chars' option. @@ -740,12 +742,12 @@ TEST(ExpressionTrimTest, DoesSerializeCorrectly) { << "$$CURRENT.a")), expCtx.variablesParseState); ASSERT_VALUE_EQ( - trim->serialize(false), + trim->serialize(), Value(Document{{"$ltrim", Document{{"input", "$inputField"_sd}, {"chars", "$a"_sd}}}})); // Make sure we can re-parse it and evaluate it. reparsedTrim = Expression::parseExpression( - &expCtx, trim->serialize(false).getDocument().toBson(), expCtx.variablesParseState); + &expCtx, trim->serialize().getDocument().toBson(), expCtx.variablesParseState); ASSERT_VALUE_EQ(reparsedTrim->evaluate(Document{{"inputField", " , 4"_sd}, {"a", " ,"_sd}}, &expCtx.variables), Value("4"_sd)); diff --git a/src/mongo/db/pipeline/expression_walker_test.cpp b/src/mongo/db/pipeline/expression_walker_test.cpp index 36fbe505cc8..b03673c8df5 100644 --- a/src/mongo/db/pipeline/expression_walker_test.cpp +++ b/src/mongo/db/pipeline/expression_walker_test.cpp @@ -198,7 +198,7 @@ TEST_F(ExpressionWalkerTest, SubstitutePathOnlySubstitutesPrefix) { auto expression = parseExpression("{$concat: ['$a', '$b', '$a.a', '$b.a', '$$NOW']}"); walk<Expression>(expression.get(), &substituteWalker); ASSERT_BSONOBJ_EQ(fromjson("{$concat: ['$b', '$b', '$b.a', '$b.a', '$$NOW']}"), - expression->serialize(false).getDocument().toBson()); + expression->serialize().getDocument().toBson()); } TEST_F(ExpressionWalkerTest, SubstitutePathSubstitutesWhenThereAreDottedFields) { @@ -207,7 +207,7 @@ TEST_F(ExpressionWalkerTest, SubstitutePathSubstitutesWhenThereAreDottedFields) auto expression = parseExpression("{$concat: ['$a.b', '$a.b.c', '$c', '$d.e.f']}"); walk<Expression>(expression.get(), &substituteWalker); ASSERT_BSONOBJ_EQ(fromjson("{$concat: ['$a.b', '$x', '$q.r', '$y.f']}"), - expression->serialize(false).getDocument().toBson()); + expression->serialize().getDocument().toBson()); } TEST_F(ExpressionWalkerTest, SubstitutePathSubstitutesWhenExpressionIsNested) { @@ -217,7 +217,7 @@ TEST_F(ExpressionWalkerTest, SubstitutePathSubstitutesWhenExpressionIsNested) { parseExpression("{$multiply: [{$add: ['$a.b', '$c']}, {$ifNull: ['$a.b.c', '$d']}]}"); walk<Expression>(expression.get(), &substituteWalker); ASSERT_BSONOBJ_EQ(fromjson("{$multiply: [{$add: ['$x', '$y']}, {$ifNull: ['$x.c', '$d']}]}"), - expression->serialize(false).getDocument().toBson()); + expression->serialize().getDocument().toBson()); } TEST_F(ExpressionWalkerTest, SubstitutePathDoesNotSubstitutesWhenExpressionHasNoFieldPaths) { @@ -226,7 +226,7 @@ TEST_F(ExpressionWalkerTest, SubstitutePathDoesNotSubstitutesWhenExpressionHasNo auto expression = parseExpression("{$multiply: [1, 2, 3, 4]}"); walk<Expression>(expression.get(), &substituteWalker); ASSERT_BSONOBJ_EQ(fromjson("{$multiply: [{$const: 1}, {$const: 2}, {$const: 3}, {$const: 4}]}"), - expression->serialize(false).getDocument().toBson()); + expression->serialize().getDocument().toBson()); } } // namespace diff --git a/src/mongo/db/pipeline/field_path.cpp b/src/mongo/db/pipeline/field_path.cpp index 8839b906daf..28cd63e2c05 100644 --- a/src/mongo/db/pipeline/field_path.cpp +++ b/src/mongo/db/pipeline/field_path.cpp @@ -74,7 +74,7 @@ string FieldPath::getFullyQualifiedPath(StringData prefix, StringData suffix) { return str::stream() << prefix << "." << suffix; } -FieldPath::FieldPath(std::string inputPath) +FieldPath::FieldPath(std::string inputPath, bool validateFieldNames) : _fieldPath(std::move(inputPath)), _fieldPathDotPosition{string::npos}, _fieldHash{kHashUninitialized} { @@ -98,7 +98,10 @@ FieldPath::FieldPath(std::string inputPath) "FieldPath is too long", pathLength <= BSONDepth::getMaxAllowableDepth()); for (size_t i = 0; i < pathLength; ++i) { - uassertValidFieldName(getFieldName(i)); + const auto& fieldName = getFieldName(i); + if (validateFieldNames) { + uassertValidFieldName(fieldName); + } } } diff --git a/src/mongo/db/pipeline/field_path.h b/src/mongo/db/pipeline/field_path.h index d2ee93734e7..b8370ba7dac 100644 --- a/src/mongo/db/pipeline/field_path.h +++ b/src/mongo/db/pipeline/field_path.h @@ -69,9 +69,11 @@ public: * * Field names are validated using uassertValidFieldName(). */ - /* implicit */ FieldPath(std::string inputPath); - /* implicit */ FieldPath(StringData inputPath) : FieldPath(inputPath.toString()) {} - /* implicit */ FieldPath(const char* inputPath) : FieldPath(std::string(inputPath)) {} + /* implicit */ FieldPath(std::string inputPath, bool validateFieldNames = true); + /* implicit */ FieldPath(StringData inputPath, bool validateFieldNames = true) + : FieldPath(inputPath.toString(), validateFieldNames) {} + /* implicit */ FieldPath(const char* inputPath, bool validateFieldNames = true) + : FieldPath(std::string(inputPath), validateFieldNames) {} /** * Returns the number of path elements in the field path. @@ -134,11 +136,20 @@ public: } /** + * Returns the full path as StringData instead of const std::string&, not including the prefix + * 'FieldPath::prefix'. + */ + StringData fullPathStringData() const { + return _fieldPath; + } + + /** * Returns the full path, including the prefix 'FieldPath::prefix'. */ std::string fullPathWithPrefix() const { return prefix + _fieldPath; } + /** * A FieldPath like this but missing the first element (useful for recursion). * Precondition getPathLength() > 1. @@ -157,6 +168,14 @@ public: FieldPath concat(const FieldPath& tail) const; + bool isPrefixOf(const FieldPath& rhsPath) const { + auto lhsStr = fullPathStringData(); + auto rhsStr = rhsPath.fullPathStringData(); + return lhsStr.size() < rhsStr.size() + ? rhsStr.startsWith(lhsStr) && rhsStr[lhsStr.size()] == '.' + : lhsStr == rhsStr; + } + private: FieldPath(std::string string, std::vector<size_t> dots, std::vector<size_t> hashes) : _fieldPath(std::move(string)), diff --git a/src/mongo/db/pipeline/group_from_first_document_transformation.cpp b/src/mongo/db/pipeline/group_from_first_document_transformation.cpp index e1c711f30f3..c766bcb44d0 100644 --- a/src/mongo/db/pipeline/group_from_first_document_transformation.cpp +++ b/src/mongo/db/pipeline/group_from_first_document_transformation.cpp @@ -48,11 +48,11 @@ void GroupFromFirstDocumentTransformation::optimize() { } Document GroupFromFirstDocumentTransformation::serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const { - + boost::optional<ExplainOptions::Verbosity> explain, const SerializationOptions& options) const { MutableDocument newRoot(_accumulatorExprs.size()); + for (auto&& expr : _accumulatorExprs) { - newRoot.addField(expr.first, expr.second->serialize(static_cast<bool>(explain))); + newRoot.addField(expr.first, expr.second->serialize(options)); } return {{"newRoot", newRoot.freezeToValue()}}; diff --git a/src/mongo/db/pipeline/group_from_first_document_transformation.h b/src/mongo/db/pipeline/group_from_first_document_transformation.h index 541cd87967c..318a18cdb89 100644 --- a/src/mongo/db/pipeline/group_from_first_document_transformation.h +++ b/src/mongo/db/pipeline/group_from_first_document_transformation.h @@ -70,8 +70,8 @@ public: void optimize() final; - Document serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const final; + Document serializeTransformation(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options = {}) const final; DepsTracker::State addDependencies(DepsTracker* deps) const final; diff --git a/src/mongo/db/pipeline/javascript_execution.cpp b/src/mongo/db/pipeline/javascript_execution.cpp index f9ce429ba76..8b6d3bc6f9b 100644 --- a/src/mongo/db/pipeline/javascript_execution.cpp +++ b/src/mongo/db/pipeline/javascript_execution.cpp @@ -47,22 +47,37 @@ JsExecution* JsExecution::get(OperationContext* opCtx, StringData database, bool loadStoredProcedures, boost::optional<int> jsHeapLimitMB) { + // If a JsExecution object has already been created, return it. + JsExecution* jsExec = getCached(opCtx, loadStoredProcedures); + if (jsExec) { + return jsExec; + } + + // There is no cached JsExecution object, so create and cache one now. auto& exec = getExec(opCtx); - if (!exec) { - exec = std::make_unique<JsExecution>(opCtx, scope, jsHeapLimitMB); - exec->getScope()->setLocalDB(database); - if (loadStoredProcedures) { - exec->getScope()->loadStored(opCtx, true); - } - exec->_storedProceduresLoaded = loadStoredProcedures; - } else { - uassert(31438, - "A single operation cannot use both JavaScript aggregation expressions and $where.", - loadStoredProcedures == exec->_storedProceduresLoaded); + exec = std::make_unique<JsExecution>(opCtx, scope, jsHeapLimitMB); + exec->getScope()->setLocalDB(database); + if (loadStoredProcedures) { + exec->getScope()->loadStored(opCtx, true); } + exec->_storedProceduresLoaded = loadStoredProcedures; + return exec.get(); } +JsExecution* JsExecution::getCached(OperationContext* opCtx, bool loadStoredProcedures) { + auto& exec = getExec(opCtx); + if (exec) { + if (loadStoredProcedures == exec->_storedProceduresLoaded) { + return exec.get(); + } + tasserted( + 9136200, + "A single operation cannot use both JavaScript aggregation expressions and $where."); + } + return nullptr; +} + Value JsExecution::callFunction(ScriptingFunction func, const BSONObj& params, const BSONObj& thisObj) { diff --git a/src/mongo/db/pipeline/javascript_execution.h b/src/mongo/db/pipeline/javascript_execution.h index d95e90f6eea..a66dee6c6ca 100644 --- a/src/mongo/db/pipeline/javascript_execution.h +++ b/src/mongo/db/pipeline/javascript_execution.h @@ -58,6 +58,15 @@ public: StringData database, bool loadStoredProcedures, boost::optional<int> jsHeapLimitMB); + + /** + * Gets a pointer to the cached JsExecution instance if it exists, else returns nullptr. This + * allows the caller to skip passing all the arguments to the more general JsExecution::get() + * method, which avoids having to potentially construct and create a heap copy of a large scope + * object that will not be used because a cached JsExecution object already exists. + */ + static JsExecution* getCached(OperationContext* opCtx, bool loadStoredProcedures); + /** * Construct with a thread-local scope and initialize with the given scope variables. */ diff --git a/src/mongo/db/pipeline/lite_parsed_pipeline.h b/src/mongo/db/pipeline/lite_parsed_pipeline.h index 2c424a84ee3..279135ece43 100644 --- a/src/mongo/db/pipeline/lite_parsed_pipeline.h +++ b/src/mongo/db/pipeline/lite_parsed_pipeline.h @@ -37,6 +37,7 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/pipeline/aggregate_command_gen.h" #include "mongo/db/pipeline/lite_parsed_document_source.h" +#include "mongo/db/query/util/deferred.h" #include "mongo/db/read_concern_support_result.h" namespace mongo { @@ -66,14 +67,8 @@ public: /** * Returns all foreign namespaces referenced by stages within this pipeline, if any. */ - stdx::unordered_set<NamespaceString> getInvolvedNamespaces() const { - stdx::unordered_set<NamespaceString> involvedNamespaces; - for (auto&& spec : _stageSpecs) { - auto stagesInvolvedNamespaces = spec->getInvolvedNamespaces(); - involvedNamespaces.insert(stagesInvolvedNamespaces.begin(), - stagesInvolvedNamespaces.end()); - } - return involvedNamespaces; + const stdx::unordered_set<NamespaceString>& getInvolvedNamespaces() const { + return _involvedNamespaces.get(_stageSpecs); } /** @@ -133,9 +128,7 @@ public: * Returns true if the pipeline has a $changeStream stage. */ bool hasChangeStream() const { - return std::any_of(_stageSpecs.begin(), _stageSpecs.end(), [](auto&& spec) { - return spec->isChangeStream(); - }); + return _hasChangeStream.get(_stageSpecs); } /** @@ -226,7 +219,25 @@ public: void validate(const OperationContext* opCtx, bool performApiVersionChecks = true) const; private: + // This is logically const - any changes to _stageSpecs will invalidate cached copies of + // "_hasChangeStream" and "_involvedNamespaces" below. std::vector<std::unique_ptr<LiteParsedDocumentSource>> _stageSpecs; + Deferred<bool, const decltype(_stageSpecs)&> _hasChangeStream{[](const auto& stageSpecs) { + return std::any_of(stageSpecs.begin(), stageSpecs.end(), [](auto&& spec) { + return spec->isChangeStream(); + }); + }}; + + Deferred<stdx::unordered_set<NamespaceString>, const decltype(_stageSpecs)&> + _involvedNamespaces{[](const auto& stageSpecs) -> stdx::unordered_set<NamespaceString> { + stdx::unordered_set<NamespaceString> involvedNamespaces; + for (const auto& spec : stageSpecs) { + auto stagesInvolvedNamespaces = spec->getInvolvedNamespaces(); + involvedNamespaces.insert(stagesInvolvedNamespaces.begin(), + stagesInvolvedNamespaces.end()); + } + return involvedNamespaces; + }}; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp index b916c59728f..30dbcba6290 100644 --- a/src/mongo/db/pipeline/pipeline.cpp +++ b/src/mongo/db/pipeline/pipeline.cpp @@ -305,6 +305,11 @@ void Pipeline::validateCommon(bool alreadyOptimized) const { str::stream() << stage->getSourceName() << " can only be used once in the pipeline", !(constraints.canAppearOnlyOnceInPipeline && !singleUseStages.insert(stage->getSourceName()).second)); + + tassert(7355707, + "If a stage is broadcast to all shard servers then it must be a data source.", + constraints.hostRequirement != HostTypeRequirement::kAllShardServers || + !constraints.requiresInputDocSource); } } @@ -313,21 +318,29 @@ void Pipeline::optimizePipeline() { if (MONGO_unlikely(disablePipelineOptimization.shouldFail())) { return; } - optimizeContainer(&_sources); + optimizeEachStage(&_sources); } void Pipeline::optimizeContainer(SourceContainer* container) { - SourceContainer optimizedSources; - SourceContainer::iterator itr = container->begin(); try { while (itr != container->end()) { invariant((*itr).get()); itr = (*itr).get()->optimizeAt(itr, container); } + } catch (DBException& ex) { + ex.addContext("Failed to optimize pipeline"); + throw; + } - // Once we have reached our final number of stages, optimize each individually. + stitch(container); +} + +void Pipeline::optimizeEachStage(SourceContainer* container) { + SourceContainer optimizedSources; + try { + // We should have our final number of stages. Optimize each individually. for (auto&& source : *container) { if (auto out = source->optimize()) { optimizedSources.push_back(out); @@ -448,11 +461,19 @@ bool Pipeline::needsMongosMerger() const { }); } +bool Pipeline::needsAllShardServers() const { + return std::any_of(_sources.begin(), _sources.end(), [&](const auto& stage) { + return stage->constraints().resolvedHostTypeRequirement(pCtx) == + HostTypeRequirement::kAllShardServers; + }); +} + bool Pipeline::needsShard() const { return std::any_of(_sources.begin(), _sources.end(), [&](const auto& stage) { auto hostType = stage->constraints().resolvedHostTypeRequirement(pCtx); return (hostType == HostTypeRequirement::kAnyShard || - hostType == HostTypeRequirement::kPrimaryShard); + hostType == HostTypeRequirement::kPrimaryShard || + hostType == HostTypeRequirement::kAllShardServers); }); } @@ -498,20 +519,20 @@ stdx::unordered_set<NamespaceString> Pipeline::getInvolvedCollections() const { } vector<Value> Pipeline::serializeContainer(const SourceContainer& container, - boost::optional<ExplainOptions::Verbosity> explain) { + boost::optional<const SerializationOptions&> opts) { vector<Value> serializedSources; for (auto&& source : container) { - source->serializeToArray(serializedSources, explain); + source->serializeToArray(serializedSources, opts ? opts.get() : SerializationOptions()); } return serializedSources; } -vector<Value> Pipeline::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { - return serializeContainer(_sources, explain); + +vector<Value> Pipeline::serialize(boost::optional<const SerializationOptions&> opts) const { + return serializeContainer(_sources, opts); } -vector<BSONObj> Pipeline::serializeToBson( - boost::optional<ExplainOptions::Verbosity> explain) const { - const auto serialized = serialize(explain); +vector<BSONObj> Pipeline::serializeToBson(boost::optional<const SerializationOptions&> opts) const { + const auto serialized = serialize(opts); std::vector<BSONObj> asBson; asBson.reserve(serialized.size()); for (auto&& stage : serialized) { @@ -552,16 +573,16 @@ boost::optional<Document> Pipeline::getNext() { : boost::optional<Document>{nextResult.releaseDocument()}; } -vector<Value> Pipeline::writeExplainOps(ExplainOptions::Verbosity verbosity) const { +vector<Value> Pipeline::writeExplainOps(const SerializationOptions& opts) const { vector<Value> array; for (auto&& stage : _sources) { auto beforeSize = array.size(); - stage->serializeToArray(array, verbosity); + stage->serializeToArray(array, opts); auto afterSize = array.size(); // Append execution stats to the serialized stage if the specified verbosity is // 'executionStats' or 'allPlansExecution'. invariant(afterSize - beforeSize == 1u); - if (verbosity >= ExplainOptions::Verbosity::kExecStats) { + if (*opts.verbosity >= ExplainOptions::Verbosity::kExecStats) { auto serializedStage = array.back(); array.back() = appendCommonExecStats(serializedStage, stage->getCommonStats()); } @@ -652,7 +673,8 @@ Status Pipeline::_pipelineCanRunOnMongoS() const { auto hostRequirement = constraints.resolvedHostTypeRequirement(pCtx); const bool needsShard = (hostRequirement == HostTypeRequirement::kAnyShard || - hostRequirement == HostTypeRequirement::kPrimaryShard); + hostRequirement == HostTypeRequirement::kPrimaryShard || + hostRequirement == HostTypeRequirement::kAllShardServers); const bool mustWriteToDisk = (constraints.diskRequirement == DiskUseRequirement::kWritesPersistentData); @@ -739,17 +761,31 @@ boost::intrusive_ptr<DocumentSource> Pipeline::popFrontWithNameAndCriteria( return popFront(); } +void Pipeline::appendPipeline(std::unique_ptr<Pipeline, PipelineDeleter> otherPipeline) { + auto& otherPipelineSources = otherPipeline->getSources(); + while (!otherPipelineSources.empty()) { + _sources.push_back(std::move(otherPipelineSources.front())); + otherPipelineSources.pop_front(); + } + constexpr bool alreadyOptimized = false; + validateCommon(alreadyOptimized); + stitch(); +} + + std::unique_ptr<Pipeline, PipelineDeleter> Pipeline::makePipeline( const std::vector<BSONObj>& rawPipeline, const boost::intrusive_ptr<ExpressionContext>& expCtx, const MakePipelineOptions opts) { auto pipeline = Pipeline::parse(rawPipeline, expCtx, opts.validator); + bool alreadyOptimized = opts.alreadyOptimized; + if (opts.optimize) { pipeline->optimizePipeline(); + alreadyOptimized = true; } - constexpr bool alreadyOptimized = true; pipeline->validateCommon(alreadyOptimized); if (opts.attachCursorSource) { @@ -766,30 +802,13 @@ Pipeline::SourceContainer::iterator Pipeline::optimizeEndOfPipeline( // optimize, since otherwise calls to optimizeAt() will overrun these limits. auto endOfPipeline = Pipeline::SourceContainer(std::next(itr), container->end()); Pipeline::optimizeContainer(&endOfPipeline); + Pipeline::optimizeEachStage(&endOfPipeline); container->erase(std::next(itr), container->end()); container->splice(std::next(itr), endOfPipeline); return std::next(itr); } -Pipeline::SourceContainer::iterator Pipeline::optimizeAtEndOfPipeline( - Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container) { - if (itr == container->end()) { - return itr; - } - itr = std::next(itr); - try { - while (itr != container->end()) { - invariant((*itr).get()); - itr = (*itr).get()->optimizeAt(itr, container); - } - } catch (DBException& ex) { - ex.addContext("Failed to optimize pipeline"); - throw; - } - return itr; -} - std::unique_ptr<Pipeline, PipelineDeleter> Pipeline::makePipelineFromViewDefinition( const boost::intrusive_ptr<ExpressionContext>& subPipelineExpCtx, ExpressionContext::ResolvedNamespace resolvedNs, diff --git a/src/mongo/db/pipeline/pipeline.h b/src/mongo/db/pipeline/pipeline.h index a3108be8fa1..367b4382941 100644 --- a/src/mongo/db/pipeline/pipeline.h +++ b/src/mongo/db/pipeline/pipeline.h @@ -45,7 +45,6 @@ #include "mongo/db/query/explain_options.h" #include "mongo/db/query/query_knobs_gen.h" #include "mongo/executor/task_executor.h" -#include "mongo/s/query/async_results_merger_params_gen.h" #include "mongo/util/intrusive_counter.h" #include "mongo/util/timer.h" @@ -70,6 +69,10 @@ using PipelineValidatorCallback = std::function<void(const Pipeline&)>; struct MakePipelineOptions { bool optimize = true; + // It is assumed that the pipeline has already been optimized when we create the + // MakePipelineOptions. If this is not the case, the caller is responsible for setting + // alreadyOptimized to false. + bool alreadyOptimized = true; bool attachCursorSource = true; ShardTargetingPolicy shardTargetingPolicy = ShardTargetingPolicy::kAllowed; PipelineValidatorCallback validator = nullptr; @@ -171,16 +174,6 @@ public: static Pipeline::SourceContainer::iterator optimizeEndOfPipeline( Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container); - /** - * Applies optimizeAt() to all stages in the given pipeline after the stage that 'itr' points - * to. - * - * Returns a valid iterator that points to the new "end of the pipeline": i.e., the stage that - * comes after 'itr' in the newly optimized pipeline. - */ - static Pipeline::SourceContainer::iterator optimizeAtEndOfPipeline( - Pipeline::SourceContainer::iterator itr, Pipeline::SourceContainer* container); - static std::unique_ptr<Pipeline, PipelineDeleter> makePipelineFromViewDefinition( const boost::intrusive_ptr<ExpressionContext>& subPipelineExpCtx, ExpressionContext::ResolvedNamespace resolvedNs, @@ -269,6 +262,11 @@ public: bool needsShard() const; /** + * Returns 'true' if any stage in the pipeline requires being run on all shards. + */ + bool needsAllShardServers() const; + + /** * Returns true if the pipeline can run on mongoS, but is not obliged to; that is, it can run * either on mongoS or on a shard. */ @@ -286,7 +284,13 @@ public: void optimizePipeline(); /** - * Modifies the container, optimizing it by combining and swapping stages. + * Modifies the container, optimizes each stage individually. + */ + static void optimizeEachStage(SourceContainer* container); + + /** + * Modifies the container, optimizing it by combining, swapping, dropping and/or inserting + * stages. */ static void optimizeContainer(SourceContainer* container); @@ -301,11 +305,12 @@ public: * Helpers to serialize a pipeline. */ std::vector<Value> serialize( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const; + boost::optional<const SerializationOptions&> opts = boost::none) const; std::vector<BSONObj> serializeToBson( - boost::optional<ExplainOptions::Verbosity> explain = boost::none) const; + boost::optional<const SerializationOptions&> opts = boost::none) const; static std::vector<Value> serializeContainer( - const SourceContainer& container, boost::optional<ExplainOptions::Verbosity> = boost::none); + const SourceContainer& container, + boost::optional<const SerializationOptions&> opts = boost::none); /** * Serializes the pipeline into BSON for explain/debug logging purposes. @@ -326,7 +331,8 @@ public: * Write the pipeline's operators to a std::vector<Value>, providing the level of detail * specified by 'verbosity'. */ - std::vector<Value> writeExplainOps(ExplainOptions::Verbosity verbosity) const; + std::vector<Value> writeExplainOps( + const SerializationOptions& opts = SerializationOptions{}) const; /** * Returns the dependencies needed by this pipeline. 'unavailableMetadata' should reflect what @@ -398,6 +404,12 @@ public: StringData targetStageName, std::function<bool(const DocumentSource* const)> predicate); /** + * Appends another pipeline to the existing pipeline. + * NOTE: The other pipeline will be destroyed. + */ + void appendPipeline(std::unique_ptr<Pipeline, PipelineDeleter> otherPipeline); + + /** * Performs common validation for top-level or facet pipelines. Throws if the pipeline is * invalid. * diff --git a/src/mongo/db/pipeline/pipeline_metadata_tree_test.cpp b/src/mongo/db/pipeline/pipeline_metadata_tree_test.cpp index 8b8f6597596..b34a1cfc2c3 100644 --- a/src/mongo/db/pipeline/pipeline_metadata_tree_test.cpp +++ b/src/mongo/db/pipeline/pipeline_metadata_tree_test.cpp @@ -58,16 +58,6 @@ #include "mongo/unittest/temp_dir.h" #include "mongo/unittest/unittest.h" -#define ASSERT_DOES_NOT_THROW(EXPRESSION) \ - try { \ - EXPRESSION; \ - } catch (const AssertionException& e) { \ - ::mongo::str::stream err; \ - err << "Threw an exception incorrectly: " << e.toString() \ - << " Exception occured in: " << #EXPRESSION; \ - ::mongo::unittest::TestAssertionFailure(__FILE__, __LINE__, err).stream(); \ - } - namespace mongo { namespace { diff --git a/src/mongo/db/pipeline/pipeline_test.cpp b/src/mongo/db/pipeline/pipeline_test.cpp index 288769a3961..d36681531fe 100644 --- a/src/mongo/db/pipeline/pipeline_test.cpp +++ b/src/mongo/db/pipeline/pipeline_test.cpp @@ -42,6 +42,8 @@ #include "mongo/db/pipeline/document_source_change_stream.h" #include "mongo/db/pipeline/document_source_change_stream_add_post_image.h" #include "mongo/db/pipeline/document_source_change_stream_add_pre_image.h" +#include "mongo/db/pipeline/document_source_change_stream_ensure_resume_token_present.h" +#include "mongo/db/pipeline/document_source_change_stream_gen.h" #include "mongo/db/pipeline/document_source_change_stream_handle_topology_change.h" #include "mongo/db/pipeline/document_source_facet.h" #include "mongo/db/pipeline/document_source_graph_lookup.h" @@ -78,6 +80,8 @@ using std::vector; const NamespaceString kTestNss = NamespaceString("a.collection"); const NamespaceString kAdminCollectionlessNss = NamespaceString("admin.$cmd.aggregate"); +const auto kExplain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; constexpr size_t getChangeStreamStageSize() { return 6; @@ -89,6 +93,25 @@ void setMockReplicationCoordinatorOnOpCtx(OperationContext* opCtx) { std::make_unique<repl::ReplicationCoordinatorMock>(opCtx->getServiceContext())); } +DocumentSource* getStageAtPos(const Pipeline::SourceContainer& stages, int pos) { + if (pos >= 0) { + auto it = stages.begin(); + std::advance(it, pos); + return (*it).get(); + } else { + auto it = stages.rbegin(); + std::advance( + it, + -pos - 1); // Subtract 1 because rbegin() points to the element before the last one. + return (*it).get(); + } +} + +template <typename T> +void assertStageAtPos(const Pipeline::SourceContainer& stages, int pos) { + ASSERT(dynamic_cast<T*>(getStageAtPos(stages, pos))); +} + namespace Optimizations { namespace Local { @@ -102,7 +125,8 @@ class StubExplainInterface : public StubMongoProcessInterface { std::unique_ptr<Pipeline, PipelineDeleter> pipeline( ownedPipeline, PipelineDeleter(ownedPipeline->getContext()->opCtx)); BSONArrayBuilder bab; - auto pipelineVec = pipeline->writeExplainOps(verbosity); + auto opts = SerializationOptions{boost::make_optional(verbosity)}; + auto pipelineVec = pipeline->writeExplainOps(opts); for (auto&& stage : pipelineVec) { bab << stage; } @@ -149,7 +173,8 @@ void assertPipelineOptimizesAndSerializesTo(std::string inputPipeJson, auto outputPipe = Pipeline::parse(request.getPipeline(), ctx); outputPipe->optimizePipeline(); - ASSERT_VALUE_EQ(Value(outputPipe->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner)), + ASSERT_VALUE_EQ(Value(outputPipe->writeExplainOps(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})), Value(outputPipeExpected["pipeline"])); ASSERT_VALUE_EQ(Value(outputPipe->serialize()), Value(serializePipeExpected["pipeline"])); } @@ -497,6 +522,22 @@ TEST(PipelineOptimizationTest, SortDoesNotSwapBeforeUnwindBecauseUnwindPathEqual assertPipelineOptimizesAndSerializesTo(inputPipe, outputPipe, serializedPipe); } +TEST(PipelineOptimizationTest, SortDoesNotSwapBeforeUnwindBecauseArrayIndexField) { + std::string inputPipe = + "[{$unwind : {path: '$a', includeArrayIndex: 'i'}}" + ",{$sort : {i: 1}}" + "]"; + std::string outputPipe = + "[{$unwind : {path: '$a', includeArrayIndex: 'i'}}" + ",{$sort : {sortKey: {i: 1}}}" + "]"; + std::string serializedPipe = + "[{$unwind : {path: '$a', includeArrayIndex: 'i'}}" + ",{$sort : {i: 1}}" + "]"; + assertPipelineOptimizesAndSerializesTo(inputPipe, outputPipe, serializedPipe); +} + TEST(PipelineOptimizationTest, LookupShouldCoalesceWithUnwindOnAsSortDoesNotInterfere) { string inputPipe = "[{$lookup: {from : 'lookupColl', as : 'same', localField: 'left', foreignField: " @@ -2643,155 +2684,164 @@ TEST(PipelineOptimizationTest, MatchOnFmodShouldSwapWithAdjacentStage) { assertPipelineOptimizesTo(inputPipe, outputPipe); } -TEST(PipelineOptimizationTest, ChangeStreamLookupSwapsWithIndependentMatch) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); +class ChangeStreamPipelineOptimizationTest : public ServiceContextTest { +public: + ChangeStreamPipelineOptimizationTest() + : ChangeStreamPipelineOptimizationTest(false /* inMongos */) {} - intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); + ChangeStreamPipelineOptimizationTest(bool inMongos) { + _opCtx = _testServiceContext.makeOperationContext(); + _expCtx = make_intrusive<ExpressionContextForTest>(_opCtx.get(), kTestNss); + _expCtx->opCtx = _opCtx.get(); + _expCtx->uuid = UUID::gen(); + _expCtx->inMongos = inMongos; + setMockReplicationCoordinatorOnOpCtx(_expCtx->opCtx); + } + BSONObj changestreamStage(const std::string& stageStr) { + return fromjson("{$changeStream: " + stageStr + "}"); + } + BSONObj matchStage(const std::string& stageStr) { + return fromjson("{$match: " + stageStr + "}"); + } + BSONObj redactStage(const std::string& stageStr) { + return fromjson("{$redact: " + stageStr + "}"); + } + std::unique_ptr<Pipeline, PipelineDeleter> makePipeline( + const std::vector<BSONObj>& rawPipeline) { + auto pipeline = Pipeline::parse(rawPipeline, _expCtx); + return pipeline; + } - // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which - // filters out newly added events. - auto spec = BSON("$changeStream" << BSON( - "fullDocument" - << "updateLookup" - << DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName << true)); - auto stages = DocumentSourceChangeStream::createFromBson(spec.firstElement(), expCtx); - ASSERT_EQ(stages.size(), getChangeStreamStageSize()); - // Make sure the change lookup is at the end. - ASSERT(dynamic_cast<DocumentSourceChangeStreamAddPostImage*>(stages.back().get())); + static std::string generateEventResumeToken() { + ResumeTokenData resumeTokenDataIn{Timestamp{1001, 3}, + ResumeTokenData::kDefaultTokenVersion, + 0, + UUID::gen(), + Value(Document{{"operationType", "drop"_sd}})}; + return ResumeToken(resumeTokenDataIn).toBSON().toString(); + } - auto matchPredicate = BSON("extra" - << "predicate"); - stages.push_back(DocumentSourceMatch::create(matchPredicate, expCtx)); - auto pipeline = Pipeline::create(stages, expCtx); - pipeline->optimizePipeline(); +private: + QueryTestServiceContext _testServiceContext; + ServiceContext::UniqueOperationContext _opCtx; + boost::intrusive_ptr<ExpressionContextForTest> _expCtx; +}; - // Make sure the $match stage has swapped before the change look up. - ASSERT( - dynamic_cast<DocumentSourceChangeStreamAddPostImage*>(pipeline->getSources().back().get())); +TEST_F(ChangeStreamPipelineOptimizationTest, ChangeStreamLookUpSize) { + auto pipeline = makePipeline( + {changestreamStage("{fullDocument: 'updateLookup', showExpandedEvents: true}")}); + ASSERT_EQ(pipeline->getSources().size(), getChangeStreamStageSize()); + // Make sure the change lookup is at the end. + assertStageAtPos<DocumentSourceChangeStreamAddPostImage>(pipeline->getSources(), -1 /* pos */); } -TEST(PipelineOptimizationTest, ChangeStreamLookupDoesNotSwapWithMatchOnPostImage) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); - - intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); - +TEST_F(ChangeStreamPipelineOptimizationTest, ChangeStreamLookupSwapsWithIndependentMatch) { // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which // filters out newly added events. - auto spec = BSON("$changeStream" << BSON( - "fullDocument" - << "updateLookup" - << DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName << true)); - auto stages = DocumentSourceChangeStream::createFromBson(spec.firstElement(), expCtx); - ASSERT_EQ(stages.size(), getChangeStreamStageSize()); - // Make sure the change lookup is at the end. - ASSERT(dynamic_cast<DocumentSourceChangeStreamAddPostImage*>(stages.back().get())); - - stages.push_back(DocumentSourceMatch::create( - BSON(DocumentSourceChangeStreamAddPostImage::kFullDocumentFieldName << BSONNULL), expCtx)); - auto pipeline = Pipeline::create(stages, expCtx); + auto pipeline = + makePipeline({changestreamStage("{fullDocument: 'updateLookup', showExpandedEvents: true}"), + matchStage("{extra: 'predicate'}")}); pipeline->optimizePipeline(); + // Make sure the $match stage has swapped before the change look up. + assertStageAtPos<DocumentSourceChangeStreamAddPostImage>(pipeline->getSources(), -1 /* pos */); +} +TEST_F(ChangeStreamPipelineOptimizationTest, ChangeStreamLookupDoesNotSwapWithMatchOnPostImage) { + // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which + // filters out newly added eve + auto pipeline = + makePipeline({changestreamStage("{fullDocument: 'updateLookup', showExpandedEvents: true}"), + matchStage("{fullDocument: null}")}); + pipeline->optimizePipeline(); // Make sure the $match stage stays at the end. - ASSERT(dynamic_cast<DocumentSourceMatch*>(pipeline->getSources().back().get())); + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -1 /* pos */); } -TEST(PipelineOptimizationTest, FullDocumentBeforeChangeLookupSwapsWithIndependentMatch) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); - - intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); - +TEST_F(ChangeStreamPipelineOptimizationTest, FullDocumentBeforeChangeLookupSize) { // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which // filters out newly added events. - auto spec = BSON("$changeStream" << BSON( - "fullDocumentBeforeChange" - << "required" - << DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName << true)); - auto stages = DocumentSourceChangeStream::createFromBson(spec.firstElement(), expCtx); - ASSERT_EQ(stages.size(), getChangeStreamStageSize()); + auto pipeline = makePipeline( + {changestreamStage("{fullDocumentBeforeChange: 'required', showExpandedEvents: true}")}); + ASSERT_EQ(pipeline->getSources().size(), getChangeStreamStageSize()); // Make sure the pre-image lookup is at the end. - ASSERT(dynamic_cast<DocumentSourceChangeStreamAddPreImage*>(stages.back().get())); + assertStageAtPos<DocumentSourceChangeStreamAddPreImage>(pipeline->getSources(), -1 /* pos */); +} - auto matchPredicate = BSON("extra" - << "predicate"); - stages.push_back(DocumentSourceMatch::create(matchPredicate, expCtx)); - auto pipeline = Pipeline::create(stages, expCtx); +TEST_F(ChangeStreamPipelineOptimizationTest, + FullDocumentBeforeChangeLookupSwapsWithIndependentMatch) { + // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which + // filters out newly added events. + auto pipeline = makePipeline( + {changestreamStage("{fullDocumentBeforeChange: 'required', showExpandedEvents: true}"), + matchStage("{extra: 'predicate'}")}); pipeline->optimizePipeline(); - // Make sure the $match stage has swapped before the change look up. - ASSERT( - dynamic_cast<DocumentSourceChangeStreamAddPreImage*>(pipeline->getSources().back().get())); + assertStageAtPos<DocumentSourceChangeStreamAddPreImage>(pipeline->getSources(), -1 /* pos */); } -TEST(PipelineOptimizationTest, FullDocumentBeforeChangeDoesNotSwapWithMatchOnPreImage) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); - - intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); - +TEST_F(ChangeStreamPipelineOptimizationTest, + FullDocumentBeforeChangeDoesNotSwapWithMatchOnPreImage) { // We enable the 'showExpandedEvents' flag to avoid injecting an additional $match stage which // filters out newly added events. - auto spec = BSON("$changeStream" << BSON( - "fullDocumentBeforeChange" - << "required" - << DocumentSourceChangeStreamSpec::kShowExpandedEventsFieldName << true)); - auto stages = DocumentSourceChangeStream::createFromBson(spec.firstElement(), expCtx); - ASSERT_EQ(stages.size(), getChangeStreamStageSize()); - // Make sure the pre-image lookup is at the end. - ASSERT(dynamic_cast<DocumentSourceChangeStreamAddPreImage*>(stages.back().get())); - - stages.push_back(DocumentSourceMatch::create( - BSON(DocumentSourceChangeStreamAddPreImage::kFullDocumentBeforeChangeFieldName << BSONNULL), - expCtx)); - auto pipeline = Pipeline::create(stages, expCtx); + auto pipeline = makePipeline( + {changestreamStage("{fullDocumentBeforeChange: 'required', showExpandedEvents: true}"), + matchStage("{fullDocumentBeforeChange: null}")}); pipeline->optimizePipeline(); - // Make sure the $match stage stays at the end. - ASSERT(dynamic_cast<DocumentSourceMatch*>(pipeline->getSources().back().get())); + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -1 /* pos */); } -TEST(PipelineOptimizationTest, ChangeStreamHandleTopologyChangeSwapsWithRedact) { - QueryTestServiceContext testServiceContext; - auto opCtx = testServiceContext.makeOperationContext(); +TEST_F(ChangeStreamPipelineOptimizationTest, + ChangeStreamEnsureResumeTokenSwapsWithJsonSchemaMatch) { + auto pipeline = makePipeline( + {changestreamStage("{resumeAfter: " + generateEventResumeToken() + "}"), + matchStage( + "{$jsonSchema: {properties: {documentKey: {properties: {_id: {enum: [1, 2]}}}}}}")}); + + // Assert $match is the last stage before optimization. + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -1); + + pipeline->optimizePipeline(); + + // Assert that $match swaps with $_internalChangeStreamHandleTopologyChange after optimization. + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -2); + assertStageAtPos<DocumentSourceChangeStreamEnsureResumeTokenPresent>(pipeline->getSources(), + -1); +} - boost::intrusive_ptr<ExpressionContext> expCtx(new ExpressionContextForTest(kTestNss)); - expCtx->opCtx = opCtx.get(); - expCtx->uuid = UUID::gen(); - expCtx->inMongos = true; // To enforce the $_internalChangeStreamHandleTopologyChange stage. - setMockReplicationCoordinatorOnOpCtx(expCtx->opCtx); +// To enforce the $_internalChangeStreamHandleTopologyChange stage. +class ChangeStreamPipelineOptimizationTestWithMongoS : public ChangeStreamPipelineOptimizationTest { +public: + ChangeStreamPipelineOptimizationTestWithMongoS() + : ChangeStreamPipelineOptimizationTest(true /* inMongos */) {} +}; - auto stages = DocumentSourceChangeStream::createFromBson( - fromjson("{$changeStream: {showExpandedEvents: true}}").firstElement(), expCtx); +TEST_F(ChangeStreamPipelineOptimizationTestWithMongoS, + ChangeStreamHandleTopologyChangeSwapsWithRedact) { + auto pipeline = + makePipeline({changestreamStage("{showExpandedEvents: true}"), redactStage("'$$PRUNE'")}); + pipeline->optimizePipeline(); + // Assert that $redact swaps with $_internalChangeStreamHandleTopologyChange after optimization. + assertStageAtPos<DocumentSourceRedact>(pipeline->getSources(), -2 /* pos */); + assertStageAtPos<DocumentSourceChangeStreamHandleTopologyChange>(pipeline->getSources(), + -1 /* pos */); +} - // Assert that the last stage is $_internalChangeStreamHandleTopologyChange. - ASSERT(dynamic_cast<DocumentSourceChangeStreamHandleTopologyChange*>(stages.back().get())); +TEST_F(ChangeStreamPipelineOptimizationTestWithMongoS, + ChangeStreamHandleTopologyChangeSwapsWithJsonSchemaMatch) { + auto pipeline = makePipeline( + {changestreamStage("{}"), + matchStage( + "{$jsonSchema: {properties: {documentKey: {properties: {_id: {enum: [1, 2]}}}}}}")}); - // Add $redact as the last stage. - stages.push_back(DocumentSourceRedact::createFromBson( - fromjson("{$redact: '$$PRUNE'}").firstElement(), expCtx)); + // Assert $match is the last stage before optimization. + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -1); - auto pipeline = Pipeline::create(stages, expCtx); pipeline->optimizePipeline(); - // Assert that $redact swaps with $_internalChangeStreamHandleTopologyChange after optimization. - ASSERT(dynamic_cast<DocumentSourceRedact*>( - std::prev(std::prev(pipeline->getSources().end()))->get())); - ASSERT(dynamic_cast<DocumentSourceChangeStreamHandleTopologyChange*>( - pipeline->getSources().back().get())); + // Assert that $match swaps with $_internalChangeStreamHandleTopologyChange after optimization. + assertStageAtPos<DocumentSourceMatch>(pipeline->getSources(), -2); + assertStageAtPos<DocumentSourceChangeStreamHandleTopologyChange>(pipeline->getSources(), -1); } TEST(PipelineOptimizationTest, SortLimProjLimBecomesTopKSortProj) { @@ -3194,7 +3244,8 @@ void assertTwoPipelinesOptimizeAndMergeTo(const std::string inputPipe1, } pipeline1->optimizePipeline(); - ASSERT_VALUE_EQ(Value(pipeline1->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner)), + ASSERT_VALUE_EQ(Value(pipeline1->writeExplainOps(SerializationOptions{ + boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})), Value(outputBson["pipeline"])); } @@ -3312,12 +3363,11 @@ public: mergePipe->optimizePipeline(); auto splitPipeline = sharded_agg_helpers::splitPipeline(std::move(mergePipe)); - - ASSERT_VALUE_EQ(Value(splitPipeline.shardsPipeline->writeExplainOps( - ExplainOptions::Verbosity::kQueryPlanner)), + const auto explain = + SerializationOptions{boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)}; + ASSERT_VALUE_EQ(Value(splitPipeline.shardsPipeline->writeExplainOps(explain)), Value(shardPipeExpected["pipeline"])); - ASSERT_VALUE_EQ(Value(splitPipeline.mergePipeline->writeExplainOps( - ExplainOptions::Verbosity::kQueryPlanner)), + ASSERT_VALUE_EQ(Value(splitPipeline.mergePipeline->writeExplainOps(explain)), Value(mergePipeExpected["pipeline"])); shardPipe = std::move(splitPipeline.shardsPipeline); @@ -3860,7 +3910,7 @@ class Out : public ShardMergerBase { return "[]"; } string mergePipeJson() { - return "[{$out: {db: 'a', coll: 'outColl'}}]"; + return "[{$out: {coll: 'outColl', db: 'a'}}]"; } }; diff --git a/src/mongo/db/pipeline/plan_executor_pipeline.cpp b/src/mongo/db/pipeline/plan_executor_pipeline.cpp index 958c8a653af..7e39571b589 100644 --- a/src/mongo/db/pipeline/plan_executor_pipeline.cpp +++ b/src/mongo/db/pipeline/plan_executor_pipeline.cpp @@ -37,6 +37,7 @@ #include "mongo/db/pipeline/plan_explainer_pipeline.h" #include "mongo/db/pipeline/resume_token.h" #include "mongo/db/repl/speculative_majority_read_info.h" +#include "mongo/util/duration.h" namespace mongo { namespace { diff --git a/src/mongo/db/pipeline/plan_executor_pipeline.h b/src/mongo/db/pipeline/plan_executor_pipeline.h index 52139d1b34d..577a2d3fe3f 100644 --- a/src/mongo/db/pipeline/plan_executor_pipeline.h +++ b/src/mongo/db/pipeline/plan_executor_pipeline.h @@ -29,12 +29,14 @@ #pragma once +#include "mongo/util/duration.h" #include <queue> #include "mongo/db/exec/document_value/document.h" #include "mongo/db/pipeline/pipeline.h" #include "mongo/db/pipeline/plan_explainer_pipeline.h" #include "mongo/db/query/plan_executor.h" +#include "mongo/db/query/query_shape/serialization_options.h" namespace mongo { @@ -155,7 +157,9 @@ public: * providing the level of detail specified by 'verbosity'. */ std::vector<Value> writeExplainOps(ExplainOptions::Verbosity verbosity) const { - return _pipeline->writeExplainOps(verbosity); + auto opts = SerializationOptions{}; + opts.verbosity = verbosity; + return _pipeline->writeExplainOps(opts); } void enableSaveRecoveryUnitAcrossCommandsIfSupported() override {} diff --git a/src/mongo/db/pipeline/plan_explainer_pipeline.h b/src/mongo/db/pipeline/plan_explainer_pipeline.h index ae2dca8239f..e6c1a689a16 100644 --- a/src/mongo/db/pipeline/plan_explainer_pipeline.h +++ b/src/mongo/db/pipeline/plan_explainer_pipeline.h @@ -31,6 +31,7 @@ #include "mongo/db/pipeline/pipeline.h" #include "mongo/db/query/plan_explainer.h" +#include "mongo/util/duration.h" namespace mongo { /** diff --git a/src/mongo/db/pipeline/process_interface/mongo_process_interface.h b/src/mongo/db/pipeline/process_interface/mongo_process_interface.h index f4fe2f54c4a..9a980594737 100644 --- a/src/mongo/db/pipeline/process_interface/mongo_process_interface.h +++ b/src/mongo/db/pipeline/process_interface/mongo_process_interface.h @@ -84,7 +84,7 @@ public: * 2. write_ops::UpdateModification - either the new document we want to upsert or insert into * the collection (i.e. a 'classic' replacement update), or the pipeline to run to compute * the new document. - * 3. boost::optional<BSONObj> - for pipeline-style updated, specifies variables that can be + * 3. boost::optional<BSONObj> - for pipeline-style updates, specifies variables that can be * referred to in the pipeline performing the custom update. */ using BatchObject = @@ -172,6 +172,20 @@ public: virtual bool isSharded(OperationContext* opCtx, const NamespaceString& ns) = 0; /** + * TODO SERVER-79508 validate callers of this function remain correct. + * + * Returns false if the current request only handles parsing and validating queries. In other + * words, we are not executing queries. Examples include query analysis for queryable + * encryption, executing pipeline-style operations in the Update system, and creating a Query + * Shape. This function only returns false when the process interface is of type + * 'StubMongoProcessInterface'. + * + */ + virtual bool isExpectedToExecuteQueries() { + return true; + } + + /** * Advances the proxied write time associated with the client in ReplClientInfo to * be at least as high as the one tracked by the OperationTimeTracker associated with the * given operation context. diff --git a/src/mongo/db/pipeline/process_interface/non_shardsvr_process_interface.cpp b/src/mongo/db/pipeline/process_interface/non_shardsvr_process_interface.cpp index 63fbfe7c7ec..2a1ce64792a 100644 --- a/src/mongo/db/pipeline/process_interface/non_shardsvr_process_interface.cpp +++ b/src/mongo/db/pipeline/process_interface/non_shardsvr_process_interface.cpp @@ -205,6 +205,7 @@ BSONObj NonShardServerProcessInterface::preparePipelineAndExplain( Pipeline* ownedPipeline, ExplainOptions::Verbosity verbosity) { std::vector<Value> pipelineVec; auto firstStage = ownedPipeline->peekFront(); + auto opts = SerializationOptions{verbosity}; // If the pipeline already has a cursor explain with that one, otherwise attach a new one like // we would for a normal execution and explain that. if (firstStage && typeid(*firstStage) == typeid(DocumentSourceCursor)) { @@ -212,7 +213,7 @@ BSONObj NonShardServerProcessInterface::preparePipelineAndExplain( // extracted the necessary information and won't need it again. std::unique_ptr<Pipeline, PipelineDeleter> managedPipeline( ownedPipeline, PipelineDeleter(ownedPipeline->getContext()->opCtx)); - pipelineVec = managedPipeline->writeExplainOps(verbosity); + pipelineVec = managedPipeline->writeExplainOps(opts); ownedPipeline = nullptr; } else { auto pipelineWithCursor = attachCursorSourceToPipelineForLocalRead(ownedPipeline); @@ -221,7 +222,7 @@ BSONObj NonShardServerProcessInterface::preparePipelineAndExplain( while (pipelineWithCursor->getNext()) { } } - pipelineVec = pipelineWithCursor->writeExplainOps(verbosity); + pipelineVec = pipelineWithCursor->writeExplainOps(opts); } BSONArrayBuilder bab; for (auto&& stage : pipelineVec) { diff --git a/src/mongo/db/pipeline/process_interface/shardsvr_process_interface.cpp b/src/mongo/db/pipeline/process_interface/shardsvr_process_interface.cpp index 9755fcc7dbd..aa6914f88c3 100644 --- a/src/mongo/db/pipeline/process_interface/shardsvr_process_interface.cpp +++ b/src/mongo/db/pipeline/process_interface/shardsvr_process_interface.cpp @@ -110,7 +110,9 @@ Status ShardServerProcessInterface::insert( BatchedCommandRequest batchInsertCommand(std::move(insertCommand)); - batchInsertCommand.setWriteConcern(wc.toBSON()); + const auto originalWC = expCtx->opCtx->getWriteConcern(); + ScopeGuard resetWCGuard([&] { expCtx->opCtx->setWriteConcern(originalWC); }); + expCtx->opCtx->setWriteConcern(wc); cluster::write(expCtx->opCtx, batchInsertCommand, &stats, &response, targetEpoch); @@ -129,7 +131,10 @@ StatusWith<MongoProcessInterface::UpdateResult> ShardServerProcessInterface::upd BatchWriteExecStats stats; BatchedCommandRequest batchUpdateCommand(std::move(updateCommand)); - batchUpdateCommand.setWriteConcern(wc.toBSON()); + + const auto originalWC = expCtx->opCtx->getWriteConcern(); + ScopeGuard resetWCGuard([&] { expCtx->opCtx->setWriteConcern(originalWC); }); + expCtx->opCtx->setWriteConcern(wc); cluster::write(expCtx->opCtx, batchUpdateCommand, &stats, &response, targetEpoch); diff --git a/src/mongo/db/pipeline/process_interface/shardsvr_process_interface_test.cpp b/src/mongo/db/pipeline/process_interface/shardsvr_process_interface_test.cpp index 5c3f7eebf97..d60af845ccc 100644 --- a/src/mongo/db/pipeline/process_interface/shardsvr_process_interface_test.cpp +++ b/src/mongo/db/pipeline/process_interface/shardsvr_process_interface_test.cpp @@ -28,9 +28,11 @@ */ #include "mongo/db/concurrency/lock_state.h" +#include "mongo/db/cursor_id.h" #include "mongo/db/pipeline/document_source_out.h" #include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/process_interface/shardsvr_process_interface.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/s/query/sharded_agg_test_fixture.h" #include "mongo/unittest/unittest.h" diff --git a/src/mongo/db/pipeline/process_interface/stub_mongo_process_interface.h b/src/mongo/db/pipeline/process_interface/stub_mongo_process_interface.h index 0e984abb1b7..d69d5af7809 100644 --- a/src/mongo/db/pipeline/process_interface/stub_mongo_process_interface.h +++ b/src/mongo/db/pipeline/process_interface/stub_mongo_process_interface.h @@ -79,6 +79,10 @@ public: return std::make_unique<StubWriteSizeEstimator>(); } + bool isExpectedToExecuteQueries() override { + return false; + } + bool isSharded(OperationContext* opCtx, const NamespaceString& ns) override { return false; } diff --git a/src/mongo/db/pipeline/resume_token.cpp b/src/mongo/db/pipeline/resume_token.cpp index 8cc4fde72a0..23553fc5881 100644 --- a/src/mongo/db/pipeline/resume_token.cpp +++ b/src/mongo/db/pipeline/resume_token.cpp @@ -309,8 +309,26 @@ ResumeTokenData ResumeToken::getData() const { return result; } -Document ResumeToken::toDocument() const { - return Document{{kDataFieldName, _hexKeyString}, {kTypeBitsFieldName, _typeBits}}; +Document ResumeToken::toDocument(const SerializationOptions& options) const { + // This is our default resume token for the representative query shape. + static const auto kDefaultTokenQueryStats = makeHighWaterMarkToken(Timestamp(), 1); + + return Document{ + {kDataFieldName, + options.serializeLiteral(_hexKeyString, Value(kDefaultTokenQueryStats._hexKeyString))}, + + // When serializing with 'kToDebugTypeString' 'serializeLiteral' will return an + // incorrect result. Therefore, we prefer to always exclude '_typeBits' when serializing + // the debug string by passing an empty value, since '_typeBits' is rarely set and will + // always be either missing or of type BinData. + {kTypeBitsFieldName, + options.literalPolicy == LiteralSerializationPolicy::kToDebugTypeString + ? Value() + : options.serializeLiteral(_typeBits, kDefaultTokenQueryStats._typeBits)}}; +} + +BSONObj ResumeToken::toBSON(const SerializationOptions& options) const { + return toDocument(options).toBson(); } ResumeToken ResumeToken::parse(const Document& resumeDoc) { diff --git a/src/mongo/db/pipeline/resume_token.h b/src/mongo/db/pipeline/resume_token.h index c4e8b5d06fd..74b2dd9ddef 100644 --- a/src/mongo/db/pipeline/resume_token.h +++ b/src/mongo/db/pipeline/resume_token.h @@ -36,6 +36,7 @@ #include "mongo/bson/timestamp.h" #include "mongo/db/exec/document_value/document.h" #include "mongo/db/exec/document_value/value.h" +#include "mongo/db/query/query_shape/serialization_options.h" #include "mongo/util/uuid.h" namespace mongo { @@ -165,11 +166,18 @@ public: */ explicit ResumeToken(const ResumeTokenData& resumeValue); - Document toDocument() const; + /** + * Convenience method to represent the ResumeToken as a Document. + * Provides support for specifying SerializationOptions, as this method is used to service the + * toBSON(). + */ + Document toDocument(const SerializationOptions& options = {}) const; - BSONObj toBSON() const { - return toDocument().toBson(); - } + /** + * Serialization to BSONObj. Provides support for specifying SerializationOptions, + * as ResumeToken requires a "query_shape: custom" specification in its IDL uses. + */ + BSONObj toBSON(const SerializationOptions& options = {}) const; ResumeTokenData getData() const; diff --git a/src/mongo/db/pipeline/resume_token.idl b/src/mongo/db/pipeline/resume_token.idl new file mode 100644 index 00000000000..ee20d401dd0 --- /dev/null +++ b/src/mongo/db/pipeline/resume_token.idl @@ -0,0 +1,46 @@ +# Copyright (C) 2023-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo" + cpp_includes: + - "mongo/db/pipeline/resume_token.h" + +imports: + - "mongo/idl/basic_types.idl" + +types: + # A resume token is an opaque document we return to the user that contains all the information + # needed to resume a stream where they left off. It also provides the ordering of streams + # from multiple shards. + resumeToken: + bson_serialization_type: object + description: An object representing a resume token for a change stream + cpp_type: ResumeToken + serializer: ResumeToken::toBSON + deserializer: ResumeToken::parse diff --git a/src/mongo/db/pipeline/sharded_agg_helpers.cpp b/src/mongo/db/pipeline/sharded_agg_helpers.cpp index b2a0392ef15..2fe2307b0a6 100644 --- a/src/mongo/db/pipeline/sharded_agg_helpers.cpp +++ b/src/mongo/db/pipeline/sharded_agg_helpers.cpp @@ -67,6 +67,9 @@ #include "mongo/util/fail_point.h" #include "mongo/util/visit_helper.h" +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery + + namespace mongo { namespace sharded_agg_helpers { namespace { @@ -193,11 +196,12 @@ BSONObj genericTransformForShards(MutableDocument&& cmdForShards, std::vector<RemoteCursor> establishShardCursors(OperationContext* opCtx, std::shared_ptr<executor::TaskExecutor> executor, const NamespaceString& nss, - bool mustRunOnAll, + bool mustRunOnAllShards, boost::optional<ChunkManager>& cm, const std::set<ShardId>& shardIds, const BSONObj& cmdObj, - const ReadPreferenceSetting& readPref) { + const ReadPreferenceSetting& readPref, + bool targetEveryShardServer) { LOGV2_DEBUG(20904, 1, "Dispatching command {cmdObj} to establish cursors on shards", @@ -206,9 +210,27 @@ std::vector<RemoteCursor> establishShardCursors(OperationContext* opCtx, std::vector<std::pair<ShardId, BSONObj>> requests; // If we don't need to run on all shards, then we should always have a valid routing table. - invariant(cm || mustRunOnAll); + invariant(cm || mustRunOnAllShards); + + if (targetEveryShardServer) { + uassert(7355703, + "Cannot target all hosts if the pipeline is not run on all shards.", + mustRunOnAllShards); + if (MONGO_unlikely(shardedAggregateHangBeforeEstablishingShardCursors.shouldFail())) { + LOGV2( + 7355704, + "shardedAggregateHangBeforeEstablishingShardCursors fail point enabled. Blocking " + "until fail point is disabled."); + while ( + MONGO_unlikely(shardedAggregateHangBeforeEstablishingShardCursors.shouldFail())) { + sleepsecs(1); + } + } + return establishCursorsOnAllHosts( + opCtx, std::move(executor), nss, shardIds, cmdObj, false, getDesiredRetryPolicy(opCtx)); + } - if (mustRunOnAll) { + if (mustRunOnAllShards) { // The pipeline contains a stage which must be run on all shards. Skip versioning and // enqueue the raw command objects. for (const auto& shardId : shardIds) { @@ -1057,15 +1079,17 @@ DispatchShardPipelineResults dispatchShardPipeline( : expCtx->getCollatorBSON(); // Determine whether we can run the entire aggregation on a single shard. - const bool mustRunOnAll = mustRunOnAllShards(expCtx->ns, hasChangeStream, startsWithDocuments); + const bool mustRunOnAllShards = + checkIfMustRunOnAllShards(expCtx->ns, hasChangeStream, startsWithDocuments); std::set<ShardId> shardIds = getTargetedShards( - expCtx, mustRunOnAll, executionNsRoutingInfo, shardQuery, shardTargetingCollation); + expCtx, mustRunOnAllShards, executionNsRoutingInfo, shardQuery, shardTargetingCollation); + bool targetEveryShardServer = pipeline->needsAllShardServers(); // Don't need to split the pipeline if we are only targeting a single shard, unless: // - There is a stage that needs to be run on the primary shard and the single target shard // is not the primary. // - The pipeline contains one or more stages which must always merge on mongoS. - const bool needsSplit = (shardIds.size() > 1u || needsMongosMerge || + const bool needsSplit = (shardIds.size() > 1u || needsMongosMerge || targetEveryShardServer || (needsPrimaryShardMerge && executionNsRoutingInfo && *(shardIds.begin()) != executionNsRoutingInfo->dbPrimary())); @@ -1134,8 +1158,11 @@ DispatchShardPipelineResults dispatchShardPipeline( if (hasChangeStream) { Grid::get(opCtx)->shardRegistry()->reload(opCtx); // Rebuild the set of shards as the shard registry might have changed. - shardIds = getTargetedShards( - expCtx, mustRunOnAll, executionNsRoutingInfo, shardQuery, shardTargetingCollation); + shardIds = getTargetedShards(expCtx, + mustRunOnAllShards, + executionNsRoutingInfo, + shardQuery, + shardTargetingCollation); } // If there were no shards when we began execution, we wouldn't have run this aggregation in the @@ -1147,7 +1174,7 @@ DispatchShardPipelineResults dispatchShardPipeline( // Explain does not produce a cursor, so instead we scatter-gather commands to the shards. if (explain) { - if (mustRunOnAll) { + if (mustRunOnAllShards) { // Some stages (such as $currentOp) need to be broadcast to all shards, and // should not participate in the shard version protocol. shardResults = @@ -1176,11 +1203,12 @@ DispatchShardPipelineResults dispatchShardPipeline( cursors = establishShardCursors(opCtx, expCtx->mongoProcessInterface->taskExecutor, expCtx->ns, - mustRunOnAll, + mustRunOnAllShards, executionNsRoutingInfo, shardIds, targetedCommand, - ReadPreferenceSetting::get(opCtx)); + ReadPreferenceSetting::get(opCtx), + targetEveryShardServer); } catch (const ExceptionFor<ErrorCodes::StaleConfig>& e) { // Check to see if the command failed because of a stale shard version or something @@ -1407,9 +1435,11 @@ Status appendExplainResults(DispatchShardPipelineResults&& dispatchResults, MutableDocument pipelinesDoc; // We specify "queryPlanner" verbosity when building the output for "shardsPart" because // execution stats are reported by each shard individually. - pipelinesDoc.addField("shardsPart", - Value(dispatchResults.splitPipeline->shardsPipeline->writeExplainOps( - ExplainOptions::Verbosity::kQueryPlanner))); + auto opts = SerializationOptions{}; + opts.verbosity = boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner); + pipelinesDoc.addField( + "shardsPart", + Value(dispatchResults.splitPipeline->shardsPipeline->writeExplainOps(opts))); if (dispatchResults.exchangeSpec) { BSONObjBuilder bob; dispatchResults.exchangeSpec->exchangeSpec.serialize(&bob); @@ -1418,7 +1448,7 @@ Status appendExplainResults(DispatchShardPipelineResults&& dispatchResults, } // We specify "queryPlanner" verbosity because execution stats are not currently // supported when building the output for "mergerPart". - auto explainOps = mergePipeline->writeExplainOps(ExplainOptions::Verbosity::kQueryPlanner); + auto explainOps = mergePipeline->writeExplainOps(opts); // No cursors to remote shards are established for an explain, and the $mergeCursors // aggregation stage which is normally built in addMergeCursorsSource() requires vectors of @@ -1538,9 +1568,9 @@ Shard::RetryPolicy getDesiredRetryPolicy(OperationContext* opCtx) { return Shard::RetryPolicy::kIdempotent; } -bool mustRunOnAllShards(const NamespaceString& nss, - bool hasChangeStream, - bool startsWithDocuments) { +bool checkIfMustRunOnAllShards(const NamespaceString& nss, + bool hasChangeStream, + bool startsWithDocuments) { // The following aggregations must be routed to all shards: // - Any collectionless aggregation, such as non-localOps $currentOp. // - Any aggregation which begins with a $changeStream stage. diff --git a/src/mongo/db/pipeline/sharded_agg_helpers.h b/src/mongo/db/pipeline/sharded_agg_helpers.h index 12945397b59..c23ad646bbd 100644 --- a/src/mongo/db/pipeline/sharded_agg_helpers.h +++ b/src/mongo/db/pipeline/sharded_agg_helpers.h @@ -195,7 +195,9 @@ StatusWith<ChunkManager> getExecutionNsRoutingInfo(OperationContext* opCtx, /** * Returns true if an aggregation over 'nss' must run on all shards. */ -bool mustRunOnAllShards(const NamespaceString& nss, bool hasChangeStream, bool startsWithDocuments); +bool checkIfMustRunOnAllShards(const NamespaceString& nss, + bool hasChangeStream, + bool startsWithDocuments); /** * Retrieves the desired retry policy based on whether the default writeConcern is set on 'opCtx'. diff --git a/src/mongo/db/pipeline/sharded_union_test.cpp b/src/mongo/db/pipeline/sharded_union_test.cpp index 79863fc7f14..71336e4dfb9 100644 --- a/src/mongo/db/pipeline/sharded_union_test.cpp +++ b/src/mongo/db/pipeline/sharded_union_test.cpp @@ -35,6 +35,7 @@ #include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/document_source_union_with.h" #include "mongo/db/pipeline/process_interface/shardsvr_process_interface.h" +#include "mongo/db/query/cursor_response.h" #include "mongo/db/repl/read_concern_args.h" #include "mongo/db/views/resolved_view.h" #include "mongo/s/query/sharded_agg_test_fixture.h" diff --git a/src/mongo/db/pipeline/stage_constraints.h b/src/mongo/db/pipeline/stage_constraints.h index 74bd32fbaa8..456f3cfc65f 100644 --- a/src/mongo/db/pipeline/stage_constraints.h +++ b/src/mongo/db/pipeline/stage_constraints.h @@ -77,6 +77,10 @@ struct StageConstraints { kAnyShard, // Indicates that the stage can only run on mongoS. kMongoS, + // Indicates that the stage should run on all data-bearing nodes, primary and seconday, for + // the participating shards. This is useful for stages like $currentOp which generate + // node-specific metadata. + kAllShardServers, }; /** @@ -190,7 +194,8 @@ struct StageConstraints { // shard, since it needs to be able to run on mongoS in a cluster. invariant(!(changeStreamRequirement == ChangeStreamRequirement::kAllowlist && (hostRequirement == HostTypeRequirement::kAnyShard || - hostRequirement == HostTypeRequirement::kPrimaryShard))); + hostRequirement == HostTypeRequirement::kPrimaryShard || + hostRequirement == HostTypeRequirement::kAllShardServers))); // A stage which is allowlisted for $changeStream cannot have a position requirement. invariant(!(changeStreamRequirement == ChangeStreamRequirement::kAllowlist && @@ -209,6 +214,13 @@ struct StageConstraints { if (diskRequirement == DiskUseRequirement::kWritesPersistentData) { invariant(!isAllowedInTransaction()); } + + tassert( + 7355706, + "Stage can only broadcast to all shard servers if it must be the first stage in the " + "pipeline.", + hostRequirement != HostTypeRequirement::kAllShardServers || + (requiredPosition == PositionRequirement::kFirst)); } /** diff --git a/src/mongo/db/pipeline/storage_stats_spec.idl b/src/mongo/db/pipeline/storage_stats_spec.idl index 8052f09de51..c0cc0a4ce6a 100644 --- a/src/mongo/db/pipeline/storage_stats_spec.idl +++ b/src/mongo/db/pipeline/storage_stats_spec.idl @@ -36,18 +36,23 @@ structs: StorageStatsSpec: description: Represents the 'storageStats' argument to the $collStats stage. strict: false + query_shape_component: true fields: scale: description: A number to use as a scaling factor applied to reported metrics. type: safeInt optional: true validator: { gte: 1 } + query_shape: literal verbose: type: optionalBool default: false + query_shape: parameter waitForLock: type: optionalBool default: true + query_shape: parameter numericOnly: type: optionalBool default: false + query_shape: parameter diff --git a/src/mongo/db/pipeline/transformer_interface.h b/src/mongo/db/pipeline/transformer_interface.h index 6be98af2570..6057a2e328c 100644 --- a/src/mongo/db/pipeline/transformer_interface.h +++ b/src/mongo/db/pipeline/transformer_interface.h @@ -65,8 +65,8 @@ public: * Returns a document describing this transformation. For example, this function will return * {_id: 0, x: 1} for the stage parsed from {$project: {_id: 0, x: 1}}. */ - virtual Document serializeTransformation( - boost::optional<ExplainOptions::Verbosity> explain) const = 0; + virtual Document serializeTransformation(boost::optional<ExplainOptions::Verbosity> explain, + const SerializationOptions& options = {}) const = 0; /** * Method used by inclusion and add fields projecton executors to extract computed projections diff --git a/src/mongo/db/pipeline/variables.cpp b/src/mongo/db/pipeline/variables.cpp index 26793962efd..279b8630ce6 100644 --- a/src/mongo/db/pipeline/variables.cpp +++ b/src/mongo/db/pipeline/variables.cpp @@ -40,6 +40,21 @@ namespace mongo { +namespace { + +// We need to be careful when serializing values, e.g. to populate the 'let' parameter of a command +// to be sent over the wire. First, missing values should be serialied as $$REMOVE, otherwise they +// might be incorrectly omitted or serialized as empty objects ({}). Also, we should wrap values in +// $literal to avoid a scenario like the following: suppose we had a user-defined 'let' specified as +// {let: {a: {$literal: "$notAFieldName"}}}. On mongos, this will be evaluated to the string +// "$notAFieldName". When we serialize it again for the shard commands, it must appear as {$literal: +// "$notAFieldName"}, not simply "$notAFieldName", since the latter will be treated as a field name +// by mongods. +Value serializeValue(Value val) { + return val.missing() ? Value("$$REMOVE"_sd) : Value(DOC("$literal" << val)); +} +} // namespace + using namespace std::string_literals; constexpr Variables::Id Variables::kRootId; @@ -330,7 +345,8 @@ LegacyRuntimeConstants Variables::transitionalExtractRuntimeConstants() const { } Variables::Id VariablesParseState::defineVariable(StringData name) { - // Caller should have validated before hand by using variableValidationvalidateNameForUserWrite. + // Caller should have validated before hand by using + // variableValidation::validateNameForUserWrite. massert(17275, "Can't redefine a non-user-writable variable", Variables::kBuiltinVarNameToId.find(name) == Variables::kBuiltinVarNameToId.end()); @@ -374,8 +390,9 @@ std::set<Variables::Id> VariablesParseState::getDefinedVariableIDs() const { BSONObj VariablesParseState::serialize(const Variables& vars) const { auto bob = BSONObjBuilder{}; for (auto&& [var_name, id] : _variables) - if (vars.hasValue(id)) - bob << var_name << Value(DOC("$literal" << vars.getValue(id))); + if (vars.hasValue(id)) { + bob << var_name << serializeValue(vars.getValue(id)); + } // System variables have to be added separately since the variable IDs are reserved and not // allocated like normal variables, and so not present in '_variables'. @@ -387,8 +404,9 @@ std::pair<LegacyRuntimeConstants, BSONObj> VariablesParseState::transitionalComp const Variables& vars) const { auto bob = BSONObjBuilder{}; for (auto&& [var_name, id] : _variables) - if (vars.hasValue(id)) - bob << var_name << Value(DOC("$literal" << vars.getValue(id))); + if (vars.hasValue(id)) { + bob << var_name << serializeValue(vars.getValue(id)); + } return {vars.transitionalExtractRuntimeConstants(), bob.obj()}; } diff --git a/src/mongo/db/pipeline/variables.h b/src/mongo/db/pipeline/variables.h index d7a79c523b9..d69d9d46831 100644 --- a/src/mongo/db/pipeline/variables.h +++ b/src/mongo/db/pipeline/variables.h @@ -190,6 +190,13 @@ public: MONGO_UNREACHABLE_TASSERT(5858104); } + /** + * Return true if the passed-in variable ID belongs to a builtin variable. + */ + static auto isBuiltin(Variables::Id variable) { + return kIdToBuiltinVarName.find(variable) != kIdToBuiltinVarName.end(); + } + private: struct ValueAndState { ValueAndState() = default; diff --git a/src/mongo/db/pipeline/visitors/document_source_visitor.h b/src/mongo/db/pipeline/visitors/document_source_visitor.h index a0158147e38..9358b6267f9 100644 --- a/src/mongo/db/pipeline/visitors/document_source_visitor.h +++ b/src/mongo/db/pipeline/visitors/document_source_visitor.h @@ -67,6 +67,7 @@ class DocumentSourceSingleDocumentTransformation; class DocumentSourceSkip; class DocumentSourceSort; class DocumentSourceTeeConsumer; +class DocumentSourceQueryStats; class DocumentSourceUnionWith; class DocumentSourceUnwind; @@ -125,6 +126,7 @@ public: virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceSkip> source) = 0; virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceSort> source) = 0; virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceTeeConsumer> source) = 0; + virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceQueryStats> source) = 0; virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceUnionWith> source) = 0; virtual void visit(tree_walker::MaybeConstPtr<IsConst, DocumentSourceUnwind> source) = 0; }; diff --git a/src/mongo/db/pipeline/visitors/document_source_walker.cpp b/src/mongo/db/pipeline/visitors/document_source_walker.cpp index b0ea004cae9..5f271814e1c 100644 --- a/src/mongo/db/pipeline/visitors/document_source_walker.cpp +++ b/src/mongo/db/pipeline/visitors/document_source_walker.cpp @@ -54,6 +54,7 @@ #include "mongo/db/pipeline/document_source_operation_metrics.h" #include "mongo/db/pipeline/document_source_out.h" #include "mongo/db/pipeline/document_source_plan_cache_stats.h" +#include "mongo/db/pipeline/document_source_query_stats.h" #include "mongo/db/pipeline/document_source_queue.h" #include "mongo/db/pipeline/document_source_redact.h" #include "mongo/db/pipeline/document_source_sample.h" @@ -126,6 +127,7 @@ void DocumentSourceWalker::walk(const Pipeline& pipeline) { visitHelper<DocumentSourceSingleDocumentTransformation>(ds) || visitHelper<DocumentSourceSkip>(ds) || visitHelper<DocumentSourceSort>(ds) || visitHelper<DocumentSourceTeeConsumer>(ds) || + visitHelper<DocumentSourceQueryStats>(ds) || visitHelper<DocumentSourceUnionWith>(ds) || visitHelper<DocumentSourceUnwind>(ds) // TODO: uncomment after fixing dependency //&& visitHelper<DocumentSourceUpdateOnAddShard>(ds) diff --git a/src/mongo/db/pipeline/window_function/partition_iterator.cpp b/src/mongo/db/pipeline/window_function/partition_iterator.cpp index 83925b862b3..6188845d516 100644 --- a/src/mongo/db/pipeline/window_function/partition_iterator.cpp +++ b/src/mongo/db/pipeline/window_function/partition_iterator.cpp @@ -205,15 +205,6 @@ optional<int> numericBound(WindowBounds::Bound<int> bound) { bound); } -// Assumes both arguments are numeric, and performs Decimal128 addition on them. -Value decimalAdd(const Value& left, const Value& right) { - // Widening to Decimal128 is a convenient way to avoid having many cases for different numeric - // types. The 'threshold' values we compute are only used to choose a set of documents; the - // user can't observe the type. - return Value(left.coerceToDecimal().add(right.coerceToDecimal())); -} - - } // namespace optional<std::pair<int, int>> PartitionIterator::getEndpointsRangeBased( @@ -247,7 +238,18 @@ optional<std::pair<int, int>> PartitionIterator::getEndpointsRangeBased( dateAdd(base.coerceToDate(), *range.unit, delta.coerceToInt(), TimeZone())}; } else { tassert(5429406, "Range-based bounds are specified as a number", delta.numeric()); - return decimalAdd(base, delta); + if (base.getType() == BSONType::NumberDouble) { + // When we compare a double and a Decimal128, we convert the Decimal128 to double + // and compare two double values. Since converting a double to Decimal128 is + // expensive and since during the comparison we will convert the Decimal128 to + // double, we compute the threshold as double from the beginning when the base + // value is already a double. + return Value(base.getDouble() + delta.coerceToDouble()); + } + // Widening to Decimal128 is a convenient way to avoid having many cases for different + // numeric types. The 'threshold' values we compute are only used to choose a set of + // documents; the user can't observe the type. + return Value(base.coerceToDecimal().add(delta.coerceToDecimal())); } }; auto hasExpectedType = [&](const Value& v) -> bool { diff --git a/src/mongo/db/pipeline/window_function/window_bounds.cpp b/src/mongo/db/pipeline/window_function/window_bounds.cpp index dd082135e02..7ede49d0065 100644 --- a/src/mongo/db/pipeline/window_function/window_bounds.cpp +++ b/src/mongo/db/pipeline/window_function/window_bounds.cpp @@ -63,12 +63,19 @@ WindowBounds::Bound<T> parseBound(ExpressionContext* expCtx, } template <class T> -Value serializeBound(const WindowBounds::Bound<T>& bound) { +Value serializeBound(const WindowBounds::Bound<T>& bound, + const SerializationOptions& opts, + const Value& representativeValue) { return stdx::visit( visit_helper::Overloaded{ - [](const WindowBounds::Unbounded&) { return Value(WindowBounds::kValUnbounded); }, - [](const WindowBounds::Current&) { return Value(WindowBounds::kValCurrent); }, - [](const T& n) { return Value(n); }, + [&](const WindowBounds::Unbounded&) { return Value(WindowBounds::kValUnbounded); }, + [&](const WindowBounds::Current&) { return Value(WindowBounds::kValCurrent); }, + [&](const T& n) { + // If not "unbounded" or "current", n must be a literal constant + // The upper bound must be greater than the lower bound. We override the + // representative value to meet this constraint. + return opts.serializeLiteral(n, representativeValue); + }, }, bound); } @@ -215,22 +222,31 @@ WindowBounds WindowBounds::parse(BSONObj args, uassert(5339902, "Range-based bounds require sortBy a single field", sortBy && sortBy->size() == 1); + const SortPattern::SortPatternPart& part = *sortBy->begin(); + uassert(8947400, + "Range-based bounds require a non-expression sortBy", + part.fieldPath && !part.expression); + uassert(8947401, "Range-based bounds require an ascending sortBy", part.isAscending); return bounds; } } -void WindowBounds::serialize(MutableDocument& args) const { +void WindowBounds::serialize(MutableDocument& args, const SerializationOptions& opts) const { stdx::visit( visit_helper::Overloaded{ [&](const DocumentBased& docBounds) { args[kArgDocuments] = Value{std::vector<Value>{ - serializeBound(docBounds.lower), - serializeBound(docBounds.upper), + serializeBound( + docBounds.lower, opts, /* representative value, if needed */ Value(0LL)), + serializeBound( + docBounds.upper, opts, /* representative value, if needed */ Value(1LL)), }}; }, [&](const RangeBased& rangeBounds) { args[kArgRange] = Value{std::vector<Value>{ - serializeBound(rangeBounds.lower), - serializeBound(rangeBounds.upper), + serializeBound( + rangeBounds.lower, opts, /* representative value, if needed */ Value(0LL)), + serializeBound( + rangeBounds.upper, opts, /* representative value, if needed */ Value(1LL)), }}; if (rangeBounds.unit) { args[kArgUnit] = Value{serializeTimeUnit(*rangeBounds.unit)}; diff --git a/src/mongo/db/pipeline/window_function/window_bounds.h b/src/mongo/db/pipeline/window_function/window_bounds.h index 6999f8fcdbc..90d0adf0371 100644 --- a/src/mongo/db/pipeline/window_function/window_bounds.h +++ b/src/mongo/db/pipeline/window_function/window_bounds.h @@ -121,7 +121,7 @@ struct WindowBounds { const boost::optional<SortPattern>& sortBy, ExpressionContext* expCtx); - void serialize(MutableDocument& args) const; + void serialize(MutableDocument& args, const SerializationOptions& opts) const; }; } // namespace mongo diff --git a/src/mongo/db/pipeline/window_function/window_function_expression.cpp b/src/mongo/db/pipeline/window_function/window_function_expression.cpp index 0d43689c977..a46bac72dfa 100644 --- a/src/mongo/db/pipeline/window_function/window_function_expression.cpp +++ b/src/mongo/db/pipeline/window_function/window_function_expression.cpp @@ -310,12 +310,12 @@ boost::intrusive_ptr<Expression> ExpressionFirstLast::parse( template <typename WindowFunctionN, typename AccumulatorNType> Value ExpressionN<WindowFunctionN, AccumulatorNType>::serialize( - boost::optional<ExplainOptions::Verbosity> explain) const { + const SerializationOptions& opts) const { auto acc = buildAccumulatorOnly(); - MutableDocument result(acc->serialize(nExpr, _input, static_cast<bool>(explain))); + MutableDocument result(acc->serialize(nExpr, _input, opts)); MutableDocument windowField; - _bounds.serialize(windowField); + _bounds.serialize(windowField, opts); result[kWindowArg] = windowField.freezeToValue(); return result.freezeToValue(); } diff --git a/src/mongo/db/pipeline/window_function/window_function_expression.h b/src/mongo/db/pipeline/window_function/window_function_expression.h index 261d4212fe5..b14a81d1ae3 100644 --- a/src/mongo/db/pipeline/window_function/window_function_expression.h +++ b/src/mongo/db/pipeline/window_function/window_function_expression.h @@ -189,17 +189,16 @@ public: } }; - virtual Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const { + virtual Value serialize(const SerializationOptions& opts) const { MutableDocument args; - args[_accumulatorName] = _input->serialize(static_cast<bool>(explain)); + args[_accumulatorName] = _input->serialize(opts); MutableDocument windowField; - _bounds.serialize(windowField); + _bounds.serialize(windowField, opts); args[kWindowArg] = windowField.freezeToValue(); return args.freezeToValue(); } - protected: ExpressionContext* _expCtx; std::string _accumulatorName; @@ -326,9 +325,9 @@ public: << " is not supported as a removable window function"); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument args; - args.addField(_accumulatorName, Value(_input->serialize(static_cast<bool>(explain)))); + args.addField(_accumulatorName, Value(_input->serialize(opts))); return args.freezeToValue(); } }; @@ -444,7 +443,7 @@ public: << " is not supported with a removable window"); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument args; args.addField(_accumulatorName, Value(Document())); return args.freezeToValue(); @@ -493,15 +492,17 @@ public: << " is not supported with a removable window"); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument subObj; tassert(5433604, "ExpMovingAvg neither N nor alpha was set", _N || _alpha); if (_N) { - subObj[kNArg] = Value(_N.get()); + subObj[kNArg] = opts.serializeLiteral(_N.get()); } else { - subObj[kAlphaArg] = Value(_alpha.get()); + // Alpha must be between zero and one (exclusive), so choose a legal representative + // value if applicable. + subObj[kAlphaArg] = opts.serializeLiteral(_alpha.get(), Value(0.1)); } - subObj[kInputArg] = _input->serialize(static_cast<bool>(explain)); + subObj[kInputArg] = _input->serialize(opts); MutableDocument outerObj; outerObj[kAccName] = subObj.freezeToValue(); return outerObj.freezeToValue(); @@ -528,15 +529,15 @@ public: return _unit; } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument result; - result[_accumulatorName][kArgInput] = _input->serialize(static_cast<bool>(explain)); + result[_accumulatorName][kArgInput] = _input->serialize(opts); if (_unit) { result[_accumulatorName][kArgUnit] = Value(serializeTimeUnit(*_unit)); } MutableDocument windowField; - _bounds.serialize(windowField); + _bounds.serialize(windowField, opts); result[kWindowArg] = windowField.freezeToValue(); return result.freezeToValue(); } @@ -810,9 +811,9 @@ public: MONGO_UNREACHABLE_TASSERT(5490705); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final { + Value serialize(const SerializationOptions& opts) const final { MutableDocument args; - args.addField(_accumulatorName, Value(_input->serialize(static_cast<bool>(explain)))); + args.addField(_accumulatorName, Value(_input->serialize(opts))); return args.freezeToValue(); } }; @@ -904,7 +905,7 @@ public: nExpr(std::move(nExpr)), sortPattern(std::move(sortPattern)) {} - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts) const final; boost::intrusive_ptr<AccumulatorState> buildAccumulatorOnly() const final; diff --git a/src/mongo/db/pipeline/window_function/window_function_shift.cpp b/src/mongo/db/pipeline/window_function/window_function_shift.cpp index c74424e74e7..0ae15c4e3f0 100644 --- a/src/mongo/db/pipeline/window_function/window_function_shift.cpp +++ b/src/mongo/db/pipeline/window_function/window_function_shift.cpp @@ -119,12 +119,12 @@ boost::intrusive_ptr<Expression> ExpressionShift::parse(BSONObj obj, return shiftExpr; } -Value ExpressionShift::serialize(boost::optional<ExplainOptions::Verbosity> explain) const { +Value ExpressionShift::serialize(const SerializationOptions& opts) const { MutableDocument args; - args.addField(kByArg, Value(_offset)); - args.addField(kOutputArg, _input->serialize(static_cast<bool>(explain))); - args.addField(kDefaultArg, _defaultVal.get_value_or(mongo::Value(BSONNULL))); - + args.addField(kByArg, opts.serializeLiteral(_offset)); + args.addField(kOutputArg, _input->serialize(opts)); + args.addField(kDefaultArg, + opts.serializeLiteral(_defaultVal.get_value_or(mongo::Value(BSONNULL)))); MutableDocument windowFun; windowFun.addField(_accumulatorName, args.freezeToValue()); return windowFun.freezeToValue(); diff --git a/src/mongo/db/pipeline/window_function/window_function_shift.h b/src/mongo/db/pipeline/window_function/window_function_shift.h index 99a45dd3f84..649e90ce7e4 100644 --- a/src/mongo/db/pipeline/window_function/window_function_shift.h +++ b/src/mongo/db/pipeline/window_function/window_function_shift.h @@ -68,7 +68,7 @@ public: MONGO_UNREACHABLE_TASSERT(5424302); } - Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; + Value serialize(const SerializationOptions& opts) const final; private: static boost::intrusive_ptr<Expression> parseShiftArgs(BSONObj obj, |