summaryrefslogtreecommitdiff
path: root/src/mongo/db/query
diff options
context:
space:
mode:
authorLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
committerLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
commit4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch)
tree1682a647d4463397c119183369ae6f750d5fdcff /src/mongo/db/query
parentaa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff)
parent8f0827553e09872941945a093b647a4211a9db7f (diff)
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0' with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'src/mongo/db/query')
-rw-r--r--src/mongo/db/query/SConscript69
-rw-r--r--src/mongo/db/query/canonical_query.cpp436
-rw-r--r--src/mongo/db/query/canonical_query.h78
-rw-r--r--src/mongo/db/query/canonical_query_encoder.cpp28
-rw-r--r--src/mongo/db/query/canonical_query_encoder.h32
-rw-r--r--src/mongo/db/query/canonical_query_test.cpp55
-rw-r--r--src/mongo/db/query/classic_plan_cache.h7
-rw-r--r--src/mongo/db/query/collection_query_info.cpp22
-rw-r--r--src/mongo/db/query/datetime/date_time_support.cpp2
-rw-r--r--src/mongo/db/query/datetime/date_time_support_test.cpp25
-rw-r--r--src/mongo/db/query/explain.cpp2
-rw-r--r--src/mongo/db/query/find.cpp19
-rw-r--r--src/mongo/db/query/find.h3
-rw-r--r--src/mongo/db/query/find_common.cpp12
-rw-r--r--src/mongo/db/query/find_common.h28
-rw-r--r--src/mongo/db/query/find_common_test.cpp74
-rw-r--r--src/mongo/db/query/get_executor.cpp97
-rw-r--r--src/mongo/db/query/index_bounds.cpp14
-rw-r--r--src/mongo/db/query/index_bounds.h10
-rw-r--r--src/mongo/db/query/index_bounds_builder.cpp144
-rw-r--r--src/mongo/db/query/index_bounds_builder.h23
-rw-r--r--src/mongo/db/query/index_bounds_builder_eq_null_test.cpp14
-rw-r--r--src/mongo/db/query/index_tag.cpp131
-rw-r--r--src/mongo/db/query/indexability.h34
-rw-r--r--src/mongo/db/query/internal_plans.cpp36
-rw-r--r--src/mongo/db/query/internal_plans.h10
-rw-r--r--src/mongo/db/query/lru_key_value.h79
-rw-r--r--src/mongo/db/query/lru_key_value_test.cpp75
-rw-r--r--src/mongo/db/query/mock_yield_policies.h18
-rw-r--r--src/mongo/db/query/multiple_collection_accessor.h11
-rw-r--r--src/mongo/db/query/parsed_find_command.cpp381
-rw-r--r--src/mongo/db/query/parsed_find_command.h127
-rw-r--r--src/mongo/db/query/partitioned_cache.h242
-rw-r--r--src/mongo/db/query/plan_cache.h199
-rw-r--r--src/mongo/db/query/plan_cache_indexability.cpp13
-rw-r--r--src/mongo/db/query/plan_cache_indexability.h32
-rw-r--r--src/mongo/db/query/plan_cache_indexability_test.cpp148
-rw-r--r--src/mongo/db/query/plan_cache_key_factory.cpp37
-rw-r--r--src/mongo/db/query/plan_cache_key_info_test.cpp133
-rw-r--r--src/mongo/db/query/plan_cache_size_parameter.cpp (renamed from src/mongo/db/query/query_stats/aggregated_metric.h)74
-rw-r--r--src/mongo/db/query/plan_cache_size_parameter.h (renamed from src/mongo/db/query/util/memory_util.h)27
-rw-r--r--src/mongo/db/query/plan_cache_size_parameter_test.cpp79
-rw-r--r--src/mongo/db/query/plan_enumerator.cpp52
-rw-r--r--src/mongo/db/query/plan_enumerator.h2
-rw-r--r--src/mongo/db/query/plan_executor.cpp4
-rw-r--r--src/mongo/db/query/plan_executor.h15
-rw-r--r--src/mongo/db/query/plan_executor_factory.cpp8
-rw-r--r--src/mongo/db/query/plan_executor_factory.h1
-rw-r--r--src/mongo/db/query/plan_executor_impl.cpp47
-rw-r--r--src/mongo/db/query/plan_executor_impl.h1
-rw-r--r--src/mongo/db/query/plan_executor_sbe.cpp1
-rw-r--r--src/mongo/db/query/plan_executor_sbe.h1
-rw-r--r--src/mongo/db/query/plan_explainer.h2
-rw-r--r--src/mongo/db/query/plan_explainer_factory.cpp2
-rw-r--r--src/mongo/db/query/plan_explainer_factory.h1
-rw-r--r--src/mongo/db/query/plan_explainer_impl.cpp4
-rw-r--r--src/mongo/db/query/plan_explainer_impl.h3
-rw-r--r--src/mongo/db/query/plan_explainer_sbe.cpp5
-rw-r--r--src/mongo/db/query/plan_explainer_sbe.h1
-rw-r--r--src/mongo/db/query/plan_insert_listener.cpp5
-rw-r--r--src/mongo/db/query/plan_ranker.cpp302
-rw-r--r--src/mongo/db/query/plan_ranker.h16
-rw-r--r--src/mongo/db/query/plan_ranker_index_prefix_test.cpp422
-rw-r--r--src/mongo/db/query/plan_ranker_util.h142
-rw-r--r--src/mongo/db/query/plan_summary_stats.h1
-rw-r--r--src/mongo/db/query/plan_yield_policy.cpp77
-rw-r--r--src/mongo/db/query/plan_yield_policy.h14
-rw-r--r--src/mongo/db/query/plan_yield_policy_impl.cpp5
-rw-r--r--src/mongo/db/query/plan_yield_policy_sbe.h12
-rw-r--r--src/mongo/db/query/planner_access.cpp320
-rw-r--r--src/mongo/db/query/planner_access.h11
-rw-r--r--src/mongo/db/query/planner_access_test.cpp4
-rw-r--r--src/mongo/db/query/planner_analysis.cpp6
-rw-r--r--src/mongo/db/query/planner_ixselect.cpp67
-rw-r--r--src/mongo/db/query/planner_ixselect.h27
-rw-r--r--src/mongo/db/query/planner_ixselect_test.cpp114
-rw-r--r--src/mongo/db/query/projection.cpp9
-rw-r--r--src/mongo/db/query/projection.h17
-rw-r--r--src/mongo/db/query/projection_ast.h2
-rw-r--r--src/mongo/db/query/projection_ast_test.cpp59
-rw-r--r--src/mongo/db/query/projection_ast_util.cpp156
-rw-r--r--src/mongo/db/query/projection_ast_util.h3
-rw-r--r--src/mongo/db/query/projection_parser.cpp17
-rw-r--r--src/mongo/db/query/query_feature_flags.idl30
-rw-r--r--src/mongo/db/query/query_knobs.idl105
-rw-r--r--src/mongo/db/query/query_planner.cpp195
-rw-r--r--src/mongo/db/query/query_planner_array_test.cpp4
-rw-r--r--src/mongo/db/query/query_planner_common.h14
-rw-r--r--src/mongo/db/query/query_planner_index_test.cpp14
-rw-r--r--src/mongo/db/query/query_planner_params.h5
-rw-r--r--src/mongo/db/query/query_planner_test_lib.cpp10
-rw-r--r--src/mongo/db/query/query_planner_tree_test.cpp110
-rw-r--r--src/mongo/db/query/query_planner_wildcard_index_test.cpp17
-rw-r--r--src/mongo/db/query/query_request_helper.cpp42
-rw-r--r--src/mongo/db/query/query_request_helper.h6
-rw-r--r--src/mongo/db/query/query_request_test.cpp15
-rw-r--r--src/mongo/db/query/query_shape.cpp292
-rw-r--r--src/mongo/db/query/query_shape/README.md69
-rw-r--r--src/mongo/db/query/query_shape/SConscript42
-rw-r--r--src/mongo/db/query/query_shape/agg_cmd_shape.cpp125
-rw-r--r--src/mongo/db/query/query_shape/agg_cmd_shape.h103
-rw-r--r--src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp266
-rw-r--r--src/mongo/db/query/query_shape/cmd_with_let_shape.cpp107
-rw-r--r--src/mongo/db/query/query_shape/cmd_with_let_shape.h109
-rw-r--r--src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp78
-rw-r--r--src/mongo/db/query/query_shape/find_cmd_shape.cpp227
-rw-r--r--src/mongo/db/query/query_shape/find_cmd_shape.h130
-rw-r--r--src/mongo/db/query/query_shape/find_cmd_shape_test.cpp238
-rw-r--r--src/mongo/db/query/query_shape/query_shape.cpp103
-rw-r--r--src/mongo/db/query/query_shape/query_shape.h165
-rw-r--r--src/mongo/db/query/query_shape/query_shape.idl50
-rw-r--r--src/mongo/db/query/query_shape/query_shape_test.cpp767
-rw-r--r--src/mongo/db/query/query_shape/query_shape_test.idl91
-rw-r--r--src/mongo/db/query/query_shape/serialization_options.cpp515
-rw-r--r--src/mongo/db/query/query_shape/serialization_options.h236
-rw-r--r--src/mongo/db/query/query_shape/shape_helpers.cpp108
-rw-r--r--src/mongo/db/query/query_shape/shape_helpers.h101
-rw-r--r--src/mongo/db/query/query_solution.cpp23
-rw-r--r--src/mongo/db/query/query_solution.h20
-rw-r--r--src/mongo/db/query/query_stats/README.md200
-rw-r--r--src/mongo/db/query/query_stats/SConscript121
-rw-r--r--src/mongo/db/query/query_stats/agg_key.cpp174
-rw-r--r--src/mongo/db/query/query_stats/agg_key.h129
-rw-r--r--src/mongo/db/query/query_stats/agg_key_test.cpp204
-rw-r--r--src/mongo/db/query/query_stats/aggregate_key_generator.cpp185
-rw-r--r--src/mongo/db/query/query_stats/find_key.cpp69
-rw-r--r--src/mongo/db/query/query_stats/find_key.h152
-rw-r--r--src/mongo/db/query/query_stats/find_key_test.cpp133
-rw-r--r--src/mongo/db/query/query_stats/key.cpp223
-rw-r--r--src/mongo/db/query/query_stats/key.h304
-rw-r--r--src/mongo/db/query/query_stats/key_test.cpp177
-rw-r--r--src/mongo/db/query/query_stats/query_stats.cpp441
-rw-r--r--src/mongo/db/query/query_stats/query_stats.h211
-rw-r--r--src/mongo/db/query/query_stats/query_stats_entry.cpp54
-rw-r--r--src/mongo/db/query/query_stats/query_stats_entry.h95
-rw-r--r--src/mongo/db/query/query_stats/query_stats_helpers.h52
-rw-r--r--src/mongo/db/query/query_stats/query_stats_on_parameter_change.cpp97
-rw-r--r--src/mongo/db/query/query_stats/query_stats_on_parameter_change.h76
-rw-r--r--src/mongo/db/query/query_stats/query_stats_store_test.cpp1406
-rw-r--r--src/mongo/db/query/query_stats/query_stats_test.cpp175
-rw-r--r--src/mongo/db/query/query_stats/rate_limiting.cpp96
-rw-r--r--src/mongo/db/query/query_stats/rate_limiting.h126
-rw-r--r--src/mongo/db/query/query_stats/rate_limiting_bm.cpp144
-rw-r--r--src/mongo/db/query/query_stats/rate_limiting_test.cpp77
-rw-r--r--src/mongo/db/query/query_stats/shapifying_bm.cpp142
-rw-r--r--src/mongo/db/query/query_stats/transform_algorithm.idl37
-rw-r--r--src/mongo/db/query/record_id_bound.h11
-rw-r--r--src/mongo/db/query/record_id_range.cpp109
-rw-r--r--src/mongo/db/query/record_id_range.h117
-rw-r--r--src/mongo/db/query/record_id_range_test.cpp119
-rw-r--r--src/mongo/db/query/sbe_cached_solution_planner.cpp1
-rw-r--r--src/mongo/db/query/sbe_multi_planner.cpp1
-rw-r--r--src/mongo/db/query/sbe_plan_cache.cpp91
-rw-r--r--src/mongo/db/query/sbe_plan_cache.h9
-rw-r--r--src/mongo/db/query/sbe_plan_cache_on_parameter_change.cpp4
-rw-r--r--src/mongo/db/query/sbe_plan_cache_on_parameter_change.h6
-rw-r--r--src/mongo/db/query/sbe_stage_builder.cpp236
-rw-r--r--src/mongo/db/query/sbe_stage_builder.h4
-rw-r--r--src/mongo/db/query/sbe_stage_builder_accumulator.cpp445
-rw-r--r--src/mongo/db/query/sbe_stage_builder_accumulator.h30
-rw-r--r--src/mongo/db/query/sbe_stage_builder_accumulator_test.cpp727
-rw-r--r--src/mongo/db/query/sbe_stage_builder_expression.cpp212
-rw-r--r--src/mongo/db/query/sbe_stage_builder_helpers.cpp70
-rw-r--r--src/mongo/db/query/sbe_stage_builder_helpers.h5
-rw-r--r--src/mongo/db/query/sbe_stage_builder_index_scan.cpp100
-rw-r--r--src/mongo/db/query/sbe_stage_builder_index_scan.h4
-rw-r--r--src/mongo/db/query/sbe_stage_builder_lookup.cpp36
-rw-r--r--src/mongo/db/query/sbe_stage_builder_lookup_test.cpp2
-rw-r--r--src/mongo/db/query/sbe_utils.cpp4
-rw-r--r--src/mongo/db/query/sort_pattern.cpp13
-rw-r--r--src/mongo/db/query/sort_pattern.h6
-rw-r--r--src/mongo/db/query/sort_pattern_test.cpp97
-rw-r--r--src/mongo/db/query/util/deferred.h118
-rw-r--r--src/mongo/db/query/util/deferred_test.cpp98
-rw-r--r--src/mongo/db/query/util/memory_util.cpp128
-rw-r--r--src/mongo/db/query/util/memory_util_test.cpp73
-rw-r--r--src/mongo/db/query/wildcard_multikey_paths.cpp2
177 files changed, 1947 insertions, 15801 deletions
diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript
index d6da5af72e4..025fd11d118 100644
--- a/src/mongo/db/query/SConscript
+++ b/src/mongo/db/query/SConscript
@@ -1,9 +1,6 @@
# -*- mode: python -*-
-Import([
- "env",
- "get_option",
-])
+Import("env")
env = env.Clone()
@@ -13,8 +10,6 @@ env.SConscript(
"collation",
"datetime",
'optimizer',
- 'query_stats',
- 'query_shape',
],
exports=[
'env'
@@ -24,11 +19,8 @@ env.SConscript(
env.Library(
target='canonical_query',
source=[
- 'canonical_query.cpp',
- 'canonical_query_encoder.cpp',
- 'parsed_find_command.cpp',
- 'query_shape/find_cmd_shape.cpp',
- 'query_stats/find_key.cpp',
+ "canonical_query.cpp",
+ "canonical_query_encoder.cpp",
],
LIBDEPS=[
"$BUILD_DIR/mongo/crypto/encrypted_field_config",
@@ -61,7 +53,6 @@ env.Library(
"query_planner.cpp",
"query_settings.cpp",
"query_solution.cpp",
- "record_id_range.cpp",
"stage_types.cpp",
],
LIBDEPS=[
@@ -70,6 +61,7 @@ env.Library(
"$BUILD_DIR/mongo/db/commands/server_status_core",
"$BUILD_DIR/mongo/db/exec/sbe/query_sbe_plan_stats",
"$BUILD_DIR/mongo/db/index/expression_params",
+ "$BUILD_DIR/mongo/db/index/key_generator",
"$BUILD_DIR/mongo/db/index_names",
"canonical_query",
"query_index_bounds",
@@ -78,7 +70,6 @@ env.Library(
],
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/db/fts/base_fts',
- '$BUILD_DIR/mongo/db/index/index_access_method',
"$BUILD_DIR/mongo/db/record_id_helpers",
"$BUILD_DIR/mongo/idl/server_parameter",
],
@@ -99,17 +90,6 @@ env.Library(
)
env.Library(
- target='memory_util',
- source=[
- 'util/memory_util.cpp',
- ],
- LIBDEPS=[
- '$BUILD_DIR/mongo/util/processinfo',
- '$BUILD_DIR/mongo/util/regex_util',
- ],
-)
-
-env.Library(
target="query_plan_cache",
source=[
"classic_plan_cache.cpp",
@@ -121,7 +101,6 @@ env.Library(
"$BUILD_DIR/mongo/base",
"$BUILD_DIR/mongo/db/exec/sbe/query_sbe",
"canonical_query",
- "memory_util",
]
)
@@ -139,9 +118,6 @@ env.Library(
"$BUILD_DIR/mongo/db/matcher/expressions",
"$BUILD_DIR/mongo/db/storage/execution_context",
],
- LIBDEPS_PRIVATE=[
- '$BUILD_DIR/mongo/db/catalog/health_log_interface',
- ],
)
env.Library(
@@ -282,18 +258,17 @@ env.Library(
env.Library(
target="query_knobs",
source=[
+ 'plan_cache_size_parameter.cpp',
'query_feature_flags.idl',
'query_knobs.idl',
- 'query_stats/query_stats_on_parameter_change.cpp',
'sbe_plan_cache_on_parameter_change.cpp',
],
LIBDEPS_PRIVATE=[
- '$BUILD_DIR/mongo/db/service_context',
+ "$BUILD_DIR/mongo/db/service_context",
'$BUILD_DIR/mongo/idl/feature_flag',
'$BUILD_DIR/mongo/idl/server_parameter',
'$BUILD_DIR/third_party/shim_pcrecpp',
- 'memory_util',
- ],
+ ]
)
env.Library(
@@ -367,10 +342,10 @@ env.Library(
'$BUILD_DIR/mongo/util/fail_point',
],
LIBDEPS_PRIVATE=[
- '$BUILD_DIR/mongo/db/concurrency/exception_util',
+ '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception',
'$BUILD_DIR/mongo/db/storage/recovery_unit_base',
],
-)
+ )
env.CppUnitTest(
target="db_query_test",
@@ -381,7 +356,6 @@ env.CppUnitTest(
"classic_stage_builder_test.cpp",
"count_command_test.cpp",
"cursor_response_test.cpp",
- "find_common_test.cpp",
"get_executor_test.cpp",
"getmore_request_test.cpp",
"hint_parser_test.cpp",
@@ -397,12 +371,13 @@ env.CppUnitTest(
"interval_test.cpp",
"killcursors_request_test.cpp",
"lru_key_value_test.cpp",
+ 'map_reduce_output_format_test.cpp',
"parsed_distinct_test.cpp",
"plan_cache_indexability_test.cpp",
+ "plan_cache_size_parameter_test.cpp",
"plan_cache_key_info_test.cpp",
"plan_cache_test.cpp",
"plan_ranker_test.cpp",
- "plan_ranker_index_prefix_test.cpp",
"planner_access_test.cpp",
"planner_analysis_test.cpp",
"planner_ixselect_test.cpp",
@@ -412,52 +387,41 @@ env.CppUnitTest(
"query_planner_collation_test.cpp",
"query_planner_columnar_test.cpp",
"query_planner_geo_test.cpp",
+ "query_planner_pipeline_pushdown_test.cpp",
"query_planner_hashed_index_test.cpp",
+ "query_planner_partialidx_test.cpp",
"query_planner_index_test.cpp",
"query_planner_operator_test.cpp",
"query_planner_options_test.cpp",
- "query_planner_partialidx_test.cpp",
- "query_planner_pipeline_pushdown_test.cpp",
- "query_planner_text_test.cpp",
"query_planner_tree_test.cpp",
+ "query_planner_text_test.cpp",
"query_planner_wildcard_index_test.cpp",
"query_request_test.cpp",
"query_settings_test.cpp",
- 'query_shape/agg_cmd_shape_test.cpp',
- 'query_shape/cmd_with_let_shape_test.cpp',
- "query_shape/find_cmd_shape_test.cpp",
"query_solution_test.cpp",
- "record_id_range_test.cpp",
"sbe_and_hash_test.cpp",
"sbe_and_sorted_test.cpp",
- "sbe_shard_filter_test.cpp",
"sbe_stage_builder_accumulator_test.cpp",
"sbe_stage_builder_lookup_test.cpp",
- "sbe_stage_builder_test.cpp",
"sbe_stage_builder_test_fixture.cpp",
+ "sbe_stage_builder_test.cpp",
+ "sbe_shard_filter_test.cpp",
"shard_filterer_factory_mock.cpp",
- "sort_pattern_test.cpp",
- "util/deferred_test.cpp",
- "util/memory_util_test.cpp",
"view_response_formatter_test.cpp",
- 'map_reduce_output_format_test.cpp',
],
LIBDEPS=[
"$BUILD_DIR/mongo/db/auth/authmocks",
"$BUILD_DIR/mongo/db/concurrency/lock_manager",
- "$BUILD_DIR/mongo/db/exec/document_value/document_value_test_util",
"$BUILD_DIR/mongo/db/exec/sbe/sbe_plan_stage_test",
"$BUILD_DIR/mongo/db/multitenancy",
"$BUILD_DIR/mongo/db/pipeline/aggregation_request_helper",
"$BUILD_DIR/mongo/db/pipeline/document_source_mock",
"$BUILD_DIR/mongo/db/query_exec",
- '$BUILD_DIR/mongo/db/record_id_helpers',
"$BUILD_DIR/mongo/db/repl/replmocks",
"$BUILD_DIR/mongo/db/repl/storage_interface_impl",
"$BUILD_DIR/mongo/db/service_context_d_test_fixture",
"$BUILD_DIR/mongo/db/service_context_test_fixture",
"$BUILD_DIR/mongo/dbtests/mocklib",
- "$BUILD_DIR/mongo/idl/idl_parser",
"$BUILD_DIR/mongo/rpc/rpc",
"$BUILD_DIR/mongo/util/clock_source_mock",
"collation/collator_factory_mock",
@@ -469,7 +433,6 @@ env.CppUnitTest(
"query_planner",
"query_planner_test_fixture",
"query_request",
- "query_shape/query_shape",
"query_test_service_context",
],
)
diff --git a/src/mongo/db/query/canonical_query.cpp b/src/mongo/db/query/canonical_query.cpp
index 32ebde51cfb..865cd10b245 100644
--- a/src/mongo/db/query/canonical_query.cpp
+++ b/src/mongo/db/query/canonical_query.cpp
@@ -36,6 +36,7 @@
#include "mongo/crypto/encryption_fields_gen.h"
#include "mongo/db/catalog/collection.h"
#include "mongo/db/commands/test_commands_enabled.h"
+#include "mongo/db/cst/cst_parser.h"
#include "mongo/db/jsobj.h"
#include "mongo/db/matcher/expression_array.h"
#include "mongo/db/namespace_string.h"
@@ -44,158 +45,162 @@
#include "mongo/db/query/collation/collator_factory_interface.h"
#include "mongo/db/query/fle/server_rewrite.h"
#include "mongo/db/query/indexability.h"
-#include "mongo/db/query/parsed_find_command.h"
#include "mongo/db/query/projection_parser.h"
#include "mongo/db/query/query_planner_common.h"
#include "mongo/logv2/log.h"
namespace mongo {
+namespace {
+
+bool parsingCanProduceNoopMatchNodes(const ExtensionsCallback& extensionsCallback,
+ MatchExpressionParser::AllowedFeatureSet allowedFeatures) {
+ return extensionsCallback.hasNoopExtensions() &&
+ (allowedFeatures & MatchExpressionParser::AllowedFeatures::kText ||
+ allowedFeatures & MatchExpressionParser::AllowedFeatures::kJavascript);
+}
+
+} // namespace
// static
StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize(
OperationContext* opCtx,
std::unique_ptr<FindCommandRequest> findCommand,
bool explain,
- const boost::intrusive_ptr<ExpressionContext>& givenExpCtx,
+ const boost::intrusive_ptr<ExpressionContext>& expCtx,
const ExtensionsCallback& extensionsCallback,
MatchExpressionParser::AllowedFeatureSet allowedFeatures,
const ProjectionPolicies& projectionPolicies,
std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) {
+ tassert(5746107,
+ "ntoreturn should not be set on the findCommand",
+ findCommand->getNtoreturn() == boost::none);
- if (givenExpCtx) {
- // Caller provided an ExpressionContext, let's go ahead and use that.
- auto swParsedFind = parsed_find_command::parse(givenExpCtx,
- std::move(findCommand),
- extensionsCallback,
- allowedFeatures,
- projectionPolicies);
- if (!swParsedFind.isOK()) {
- return swParsedFind.getStatus();
+ auto status = query_request_helper::validateFindCommandRequest(*findCommand);
+ if (!status.isOK()) {
+ return status;
+ }
+
+ std::unique_ptr<CollatorInterface> collator;
+ if (!findCommand->getCollation().isEmpty()) {
+ auto statusWithCollator = CollatorFactoryInterface::get(opCtx->getServiceContext())
+ ->makeFromBSON(findCommand->getCollation());
+ if (!statusWithCollator.isOK()) {
+ return statusWithCollator.getStatus();
}
- return canonicalize(std::move(givenExpCtx),
- std::move(swParsedFind.getValue()),
- explain,
- std::move(pipeline));
+ collator = std::move(statusWithCollator.getValue());
+ }
+
+ // Make MatchExpression.
+ boost::intrusive_ptr<ExpressionContext> newExpCtx;
+ if (!expCtx.get()) {
+ invariant(findCommand->getNamespaceOrUUID().nss());
+ newExpCtx = make_intrusive<ExpressionContext>(opCtx,
+ std::move(collator),
+ *findCommand->getNamespaceOrUUID().nss(),
+ findCommand->getLegacyRuntimeConstants(),
+ findCommand->getLet());
} else {
- // No ExpressionContext provided, let's call the override that makes one for us.
- auto swResults = parsed_find_command::parse(
- opCtx, std::move(findCommand), extensionsCallback, allowedFeatures, projectionPolicies);
- if (!swResults.isOK()) {
- return swResults.getStatus();
+ newExpCtx = expCtx;
+ // A collator can enter through both the FindCommandRequest and ExpressionContext arguments.
+ // This invariant ensures that both collators are the same because downstream we
+ // pull the collator from only one of the ExpressionContext carrier.
+ if (collator.get() && expCtx->getCollator()) {
+ invariant(CollatorInterface::collatorsMatch(collator.get(), expCtx->getCollator()));
}
- auto&& [expCtx, parsedFind] = std::move(swResults.getValue());
- return canonicalize(std::move(expCtx), std::move(parsedFind), explain, std::move(pipeline));
}
-}
-
-// static
-StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize(
- boost::intrusive_ptr<ExpressionContext> expCtx,
- std::unique_ptr<ParsedFindCommand> parsedFind,
- bool explain,
- std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) {
// Make the CQ we'll hopefully return.
- auto cq = std::make_unique<CanonicalQuery>();
+ std::unique_ptr<CanonicalQuery> cq(new CanonicalQuery());
cq->setExplain(explain);
- if (auto initStatus = cq->init(std::move(expCtx),
- std::move(parsedFind),
- std::move(pipeline),
- true /*optimizeMatchExpression*/);
- !initStatus.isOK()) {
+
+ StatusWithMatchExpression statusWithMatcher = [&]() -> StatusWithMatchExpression {
+ if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) {
+ try {
+ return cst::parseToMatchExpression(
+ findCommand->getFilter(), newExpCtx, extensionsCallback);
+ } catch (const DBException& ex) {
+ return ex.toStatus();
+ }
+ } else {
+ return MatchExpressionParser::parse(
+ findCommand->getFilter(), newExpCtx, extensionsCallback, allowedFeatures);
+ }
+ }();
+ if (!statusWithMatcher.isOK()) {
+ return statusWithMatcher.getStatus();
+ }
+
+ // Stop counting expressions after they have been parsed to exclude expressions created
+ // during optimization and other processing steps.
+ newExpCtx->stopExpressionCounters();
+
+ std::unique_ptr<MatchExpression> me = std::move(statusWithMatcher.getValue());
+
+ Status initStatus =
+ cq->init(opCtx,
+ std::move(newExpCtx),
+ std::move(findCommand),
+ parsingCanProduceNoopMatchNodes(extensionsCallback, allowedFeatures),
+ std::move(me),
+ projectionPolicies,
+ std::move(pipeline));
+
+ if (!initStatus.isOK()) {
return initStatus;
}
- return {std::move(cq)};
+ return std::move(cq);
}
// static
-StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::makeForSubplanner(
- OperationContext* opCtx, const CanonicalQuery& baseQuery, size_t i) {
- tassert(8401301,
- "expected MatchExpression with rooted $or",
- baseQuery.root()->matchType() == MatchExpression::OR);
- tassert(8401302,
- "attempted to get out of bounds child of $or",
- baseQuery.root()->numChildren() > i);
- auto root = baseQuery.root()->getChild(i);
+StatusWith<std::unique_ptr<CanonicalQuery>> CanonicalQuery::canonicalize(
+ OperationContext* opCtx, const CanonicalQuery& baseQuery, MatchExpression* root) {
auto findCommand = std::make_unique<FindCommandRequest>(baseQuery.nss());
- findCommand->setFilter(root->serialize());
+ BSONObjBuilder builder;
+ root->serialize(&builder, true);
+ findCommand->setFilter(builder.obj());
findCommand->setProjection(baseQuery.getFindCommandRequest().getProjection().getOwned());
findCommand->setSort(baseQuery.getFindCommandRequest().getSort().getOwned());
findCommand->setCollation(baseQuery.getFindCommandRequest().getCollation().getOwned());
+ auto status = query_request_helper::validateFindCommandRequest(*findCommand);
+ if (!status.isOK()) {
+ return status;
+ }
// Make the CQ we'll hopefully return.
- auto cq = std::make_unique<CanonicalQuery>();
+ std::unique_ptr<CanonicalQuery> cq(new CanonicalQuery());
cq->setExplain(baseQuery.getExplain());
- auto swParsedFind = ParsedFindCommand::withExistingFilter(
- baseQuery.getExpCtx(),
- baseQuery.getCollator() ? baseQuery.getCollator()->clone() : nullptr,
- root->shallowClone(),
- std::move(findCommand));
- if (!swParsedFind.isOK()) {
- return swParsedFind.getStatus();
- }
- // Note: we do not optimize the MatchExpression representing the branch of the top-level $or
- // that we are currently examining. This is because repeated invocations of
- // MatchExpression::optimize() may change the order of predicates in the MatchExpression, due to
- // new rewrites being unlocked by previous ones. We need to preserve the order of predicates to
- // allow index tagging to work properly. See SERVER-84013 for more details.
- Status initStatus = cq->init(baseQuery.getExpCtx(),
- std::move(swParsedFind.getValue()),
- {} /* an empty pipeline */,
- false /*optimizeMatchExpression*/);
-
- invariant(initStatus.isOK());
- return {std::move(cq)};
+ Status initStatus = cq->init(opCtx,
+ baseQuery.getExpCtx(),
+ std::move(findCommand),
+ baseQuery.canHaveNoopMatchNodes(),
+ root->shallowClone(),
+ ProjectionPolicies::findProjectionPolicies(),
+ {} /* an empty pipeline */);
+
+ if (!initStatus.isOK()) {
+ return initStatus;
+ }
+ return std::move(cq);
}
-Status CanonicalQuery::init(boost::intrusive_ptr<ExpressionContext> expCtx,
- std::unique_ptr<ParsedFindCommand> parsedFind,
- std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline,
- bool optimizeMatchExpression) {
+Status CanonicalQuery::init(OperationContext* opCtx,
+ boost::intrusive_ptr<ExpressionContext> expCtx,
+ std::unique_ptr<FindCommandRequest> findCommand,
+ bool canHaveNoopMatchNodes,
+ std::unique_ptr<MatchExpression> root,
+ const ProjectionPolicies& projectionPolicies,
+ std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline) {
_expCtx = expCtx;
- _findCommand = std::move(parsedFind->findCommandRequest);
- _canHaveNoopMatchNodes = parsedFind->canHaveNoopMatchNodes;
+ _findCommand = std::move(findCommand);
+ _canHaveNoopMatchNodes = canHaveNoopMatchNodes;
_forceClassicEngine = internalQueryForceClassicEngine.load();
- if (optimizeMatchExpression) {
- _root = MatchExpression::normalize(std::move(parsedFind->filter));
- } else {
- _root = std::move(parsedFind->filter);
- }
-
- if (parsedFind->proj) {
- // The projection will be optimized only if the query is not compatible with SBE or there's
- // no user-specified "let" variable. This is to prevent the user-defined variable being
- // optimized out. We will optimize the projection later after we are certain that the query
- // is ineligible for SBE.
- bool shouldOptimizeProj = !expCtx->sbeCompatible || !_findCommand->getLet();
- if (parsedFind->proj->requiresMatchDetails()) {
- // Sadly, in some cases the match details cannot be generated from the unoptimized
- // MatchExpression. For example, a rooted-$or of equalities won't work to produce the
- // details, but if you optimize that query to an $in, it will work. If we were starting
- // from scratch, we may disallow this. But it has already been released as working so we
- // will keep it so, and here have to re-parse the projection using the new, normalized
- // MatchExpression, before we save this projection for later execution.
- _proj.emplace(projection_ast::parseAndAnalyze(expCtx,
- _findCommand->getProjection(),
- _root.get(),
- _findCommand->getFilter(),
- *parsedFind->savedProjectionPolicies,
- shouldOptimizeProj));
- } else {
- _proj.emplace(std::move(*parsedFind->proj));
- if (shouldOptimizeProj) {
- _proj->optimize();
- }
- }
- }
- if (parsedFind->sort) {
- _sortPattern = std::move(parsedFind->sort);
+ auto validStatus = isValid(root.get(), *_findCommand);
+ if (!validStatus.isOK()) {
+ return validStatus.getStatus();
}
- _pipeline = std::move(pipeline);
-
- // Perform auto-parameterization only if the query is SBE-compatible and caching is enabled.
+ auto unavailableMetadata = validStatus.getValue();
+ _root = MatchExpression::normalize(std::move(root));
if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) {
const bool hasNoTextNodes =
!QueryPlannerCommon::hasNode(_root.get(), MatchExpression::TEXT);
@@ -212,45 +217,89 @@ Status CanonicalQuery::init(boost::intrusive_ptr<ExpressionContext> expCtx,
}
}
// The tree must always be valid after normalization.
- dassert(parsed_find_command::isValid(_root.get(), *_findCommand).isOK());
+ dassert(isValid(_root.get(), *_findCommand).isOK());
if (auto status = isValidNormalized(_root.get()); !status.isOK()) {
return status;
}
- if (_proj) {
- _metadataDeps = _proj->metadataDeps();
+ // Validate the projection if there is one.
+ if (!_findCommand->getProjection().isEmpty()) {
+ try {
+ _proj.emplace(projection_ast::parseAndAnalyze(expCtx,
+ _findCommand->getProjection(),
+ _root.get(),
+ _findCommand->getFilter(),
+ projectionPolicies,
+ true /* Should optimize? */));
- if (_proj->metadataDeps()[DocumentMetadataFields::kSortKey] &&
- _findCommand->getSort().isEmpty()) {
- return {ErrorCodes::BadValue, "cannot use sortKey $meta projection without a sort"};
+ // Fail if any of the projection's dependencies are unavailable.
+ DepsTracker{unavailableMetadata}.requestMetadata(_proj->metadataDeps());
+ } catch (const DBException& e) {
+ return e.toStatus();
}
+
+ _metadataDeps = _proj->metadataDeps();
}
- if (_sortPattern) {
- // Be sure to track and add any metadata dependencies from the sort (e.g. text score).
- _metadataDeps |= _sortPattern->metadataDeps(parsedFind->unavailableMetadata);
+ _pipeline = std::move(pipeline);
- // If the results of this query might have to be merged on a remote node, then that node
- // might need the sort key metadata. Request that the plan generates this metadata.
- if (_expCtx->needsMerge) {
- _metadataDeps.set(DocumentMetadataFields::kSortKey);
- }
+ if (_proj && _proj->metadataDeps()[DocumentMetadataFields::kSortKey] &&
+ _findCommand->getSort().isEmpty()) {
+ return Status(ErrorCodes::BadValue, "cannot use sortKey $meta projection without a sort");
+ }
+
+ // If there is a sort, parse it and add any metadata dependencies it induces.
+ try {
+ initSortPattern(unavailableMetadata);
+ } catch (const DBException& ex) {
+ return ex.toStatus();
}
// If the 'returnKey' option is set, then the plan should produce index key metadata.
if (_findCommand->getReturnKey()) {
_metadataDeps.set(DocumentMetadataFields::kIndexKey);
}
+
return Status::OK();
}
+void CanonicalQuery::initSortPattern(QueryMetadataBitSet unavailableMetadata) {
+ if (_findCommand->getSort().isEmpty()) {
+ return;
+ }
+
+ // A $natural sort is really a hint, and should be handled as such. Furthermore, the downstream
+ // sort handling code may not expect a $natural sort.
+ //
+ // We have already validated that if there is a $natural sort and a hint, that the hint
+ // also specifies $natural with the same direction. Therefore, it is safe to clear the $natural
+ // sort and rewrite it as a $natural hint.
+ if (_findCommand->getSort()[query_request_helper::kNaturalSortField]) {
+ _findCommand->setHint(_findCommand->getSort().getOwned());
+ _findCommand->setSort(BSONObj{});
+ }
+
+ if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) {
+ _sortPattern = cst::parseToSortPattern(_findCommand->getSort(), _expCtx);
+ } else {
+ _sortPattern = SortPattern{_findCommand->getSort(), _expCtx};
+ }
+ _metadataDeps |= _sortPattern->metadataDeps(unavailableMetadata);
+
+ // If the results of this query might have to be merged on a remote node, then that node might
+ // need the sort key metadata. Request that the plan generates this metadata.
+ if (_expCtx->needsMerge) {
+ _metadataDeps.set(DocumentMetadataFields::kSortKey);
+ }
+}
+
void CanonicalQuery::setCollator(std::unique_ptr<CollatorInterface> collator) {
auto collatorRaw = collator.get();
// We must give the ExpressionContext the same collator.
_expCtx->setCollator(std::move(collator));
- // The collator associated with the match expression tree is now invalid, since we have
- // reset the collator owned by the ExpressionContext.
+ // The collator associated with the match expression tree is now invalid, since we have reset
+ // the collator owned by the ExpressionContext.
_root->setCollator(collatorRaw);
}
@@ -284,9 +333,138 @@ bool CanonicalQuery::isSimpleIdQuery(const BSONObj& query) {
return hasID;
}
+size_t CanonicalQuery::countNodes(const MatchExpression* root, MatchExpression::MatchType type) {
+ size_t sum = 0;
+ if (type == root->matchType()) {
+ sum = 1;
+ }
+ for (size_t i = 0; i < root->numChildren(); ++i) {
+ sum += countNodes(root->getChild(i), type);
+ }
+ return sum;
+}
+
+/**
+ * Does 'root' have a subtree of type 'subtreeType' with a node of type 'childType' inside?
+ */
+bool hasNodeInSubtree(const MatchExpression* root,
+ MatchExpression::MatchType childType,
+ MatchExpression::MatchType subtreeType) {
+ if (subtreeType == root->matchType()) {
+ return QueryPlannerCommon::hasNode(root, childType);
+ }
+ for (size_t i = 0; i < root->numChildren(); ++i) {
+ if (hasNodeInSubtree(root->getChild(i), childType, subtreeType)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+StatusWith<QueryMetadataBitSet> CanonicalQuery::isValid(const MatchExpression* root,
+ const FindCommandRequest& findCommand) {
+ QueryMetadataBitSet unavailableMetadata{};
+
+ // There can only be one TEXT. If there is a TEXT, it cannot appear inside a NOR.
+ //
+ // Note that the query grammar (as enforced by the MatchExpression parser) forbids TEXT
+ // inside of value-expression clauses like NOT, so we don't check those here.
+ size_t numText = countNodes(root, MatchExpression::TEXT);
+ if (numText > 1) {
+ return Status(ErrorCodes::BadValue, "Too many text expressions");
+ } else if (1 == numText) {
+ if (hasNodeInSubtree(root, MatchExpression::TEXT, MatchExpression::NOR)) {
+ return Status(ErrorCodes::BadValue, "text expression not allowed in nor");
+ }
+ } else {
+ // Text metadata is not available.
+ unavailableMetadata.set(DocumentMetadataFields::kTextScore);
+ }
+
+ // There can only be one NEAR. If there is a NEAR, it must be either the root or the root
+ // must be an AND and its child must be a NEAR.
+ size_t numGeoNear = countNodes(root, MatchExpression::GEO_NEAR);
+ if (numGeoNear > 1) {
+ return Status(ErrorCodes::BadValue, "Too many geoNear expressions");
+ } else if (1 == numGeoNear) {
+ // Do nothing, we will perform extra checks in CanonicalQuery::isValidNormalized.
+ } else {
+ // Geo distance and geo point metadata are unavailable.
+ unavailableMetadata |= DepsTracker::kAllGeoNearData;
+ }
+
+ const BSONObj& sortObj = findCommand.getSort();
+ BSONElement sortNaturalElt = sortObj["$natural"];
+ const BSONObj& hintObj = findCommand.getHint();
+ BSONElement hintNaturalElt = hintObj["$natural"];
+
+ if (sortNaturalElt && sortObj.nFields() != 1) {
+ return Status(ErrorCodes::BadValue,
+ str::stream() << "Cannot include '$natural' in compound sort: " << sortObj);
+ }
+
+ if (hintNaturalElt && hintObj.nFields() != 1) {
+ return Status(ErrorCodes::BadValue,
+ str::stream() << "Cannot include '$natural' in compound hint: " << hintObj);
+ }
+
+ // NEAR cannot have a $natural sort or $natural hint.
+ if (numGeoNear > 0) {
+ if (sortNaturalElt) {
+ return Status(ErrorCodes::BadValue,
+ "geoNear expression not allowed with $natural sort order");
+ }
+
+ if (hintNaturalElt) {
+ return Status(ErrorCodes::BadValue,
+ "geoNear expression not allowed with $natural hint");
+ }
+ }
+
+ // TEXT and NEAR cannot both be in the query.
+ if (numText > 0 && numGeoNear > 0) {
+ return Status(ErrorCodes::BadValue, "text and geoNear not allowed in same query");
+ }
+
+ // TEXT and {$natural: ...} sort order cannot both be in the query.
+ if (numText > 0 && sortNaturalElt) {
+ return Status(ErrorCodes::BadValue, "text expression not allowed with $natural sort order");
+ }
+
+ // TEXT and hint cannot both be in the query.
+ if (numText > 0 && !hintObj.isEmpty()) {
+ return Status(ErrorCodes::BadValue, "text and hint not allowed in same query");
+ }
+
+ // TEXT and tailable are incompatible.
+ if (numText > 0 && findCommand.getTailable()) {
+ return Status(ErrorCodes::BadValue, "text and tailable cursor not allowed in same query");
+ }
+
+ // NEAR and tailable are incompatible.
+ if (numGeoNear > 0 && findCommand.getTailable()) {
+ return Status(ErrorCodes::BadValue,
+ "Tailable cursors and geo $near cannot be used together");
+ }
+
+ // $natural sort order must agree with hint.
+ if (sortNaturalElt) {
+ if (!hintObj.isEmpty() && !hintNaturalElt) {
+ return Status(ErrorCodes::BadValue, "index hint not allowed with $natural sort order");
+ }
+ if (hintNaturalElt) {
+ if (hintNaturalElt.numberInt() != sortNaturalElt.numberInt()) {
+ return Status(ErrorCodes::BadValue,
+ "$natural hint must be in the same direction as $natural sort order");
+ }
+ }
+ }
+
+ return unavailableMetadata;
+}
+
Status CanonicalQuery::isValidNormalized(const MatchExpression* root) {
- if (auto numGeoNear = QueryPlannerCommon::countNodes(root, MatchExpression::GEO_NEAR);
- numGeoNear > 0) {
+ if (auto numGeoNear = countNodes(root, MatchExpression::GEO_NEAR); numGeoNear > 0) {
tassert(5705300, "Only one geo $near expression is expected", numGeoNear == 1);
auto topLevel = false;
diff --git a/src/mongo/db/query/canonical_query.h b/src/mongo/db/query/canonical_query.h
index 570b71b6fae..98780fb00a1 100644
--- a/src/mongo/db/query/canonical_query.h
+++ b/src/mongo/db/query/canonical_query.h
@@ -37,7 +37,6 @@
#include "mongo/db/matcher/extensions_callback_noop.h"
#include "mongo/db/pipeline/inner_pipeline_stage_interface.h"
#include "mongo/db/query/collation/collator_interface.h"
-#include "mongo/db/query/parsed_find_command.h"
#include "mongo/db/query/projection.h"
#include "mongo/db/query/projection_policies.h"
#include "mongo/db/query/query_request_helper.h"
@@ -77,25 +76,19 @@ public:
std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline = {});
/**
- * Creates a CanonicalQuery from a ParsedFindCommand. Uses 'expCtx->opCtx', which must be valid.
- */
- static StatusWith<std::unique_ptr<CanonicalQuery>> canonicalize(
- boost::intrusive_ptr<ExpressionContext> expCtx,
- std::unique_ptr<ParsedFindCommand> parsedFind,
- bool explain = false,
- std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline = {});
-
- /**
* For testing or for internal clients to use.
*/
/**
- * Construct a 'CanonicalQuery' for a subquery of the given query. This function should only be
- * invoked by the subplanner. 'baseQuery' must contain a MatchExpression with rooted $or. This
- * function returns a 'CanonicalQuery' housing a copy of the i'th child of the root.
+ * Used for creating sub-queries from an existing CanonicalQuery.
+ *
+ * 'root' must be an expression in baseQuery.root().
+ *
+ * Does not take ownership of 'root'.
*/
- static StatusWith<std::unique_ptr<CanonicalQuery>> makeForSubplanner(
- OperationContext* opCtx, const CanonicalQuery& baseQuery, size_t i);
+ static StatusWith<std::unique_ptr<CanonicalQuery>> canonicalize(OperationContext* opCtx,
+ const CanonicalQuery& baseQuery,
+ MatchExpression* root);
/**
* Returns true if "query" describes an exact-match query on _id.
@@ -103,22 +96,32 @@ public:
static bool isSimpleIdQuery(const BSONObj& query);
/**
- * Perform validation checks on the normalized 'root' which could not be checked before
- * normalization - those should happen in parsed_find_command::isValid().
+ * Validates the match expression 'root' as well as the query specified by 'request', checking
+ * for illegal combinations of operators. Returns a non-OK status if any such illegal
+ * combination is found.
+ *
+ * This method can be called both on normalized and non-normalized 'root'. However, some checks
+ * can only be performed once the match expressions is normalized. To perform these checks one
+ * can call 'isValidNormalized()'.
+ *
+ * On success, returns a bitset indicating which types of metadata are *unavailable*. For
+ * example, if 'root' does not contain a $text predicate, then the returned metadata bitset will
+ * indicate that text score metadata is unavailable. This means that if subsequent
+ * $meta:"textScore" expressions are found during analysis of the query, we should raise in an
+ * error.
*/
- static Status isValidNormalized(const MatchExpression* root);
+ static StatusWith<QueryMetadataBitSet> isValid(const MatchExpression* root,
+ const FindCommandRequest& findCommand);
/**
- * For internal use only - but public for accessibility for make_unique(). You must go through
- * canonicalize to create a CanonicalQuery.
+ * Perform additional validation checks on the normalized 'root'.
*/
- CanonicalQuery() {}
+ static Status isValidNormalized(const MatchExpression* root);
const NamespaceString nss() const {
invariant(_findCommand->getNamespaceOrUUID().nss());
return *_findCommand->getNamespaceOrUUID().nss();
}
-
const std::string ns() const {
return nss().ns();
}
@@ -197,6 +200,11 @@ public:
std::string toStringShort() const;
/**
+ * Returns a count of 'type' nodes in expression tree.
+ */
+ static size_t countNodes(const MatchExpression* root, MatchExpression::MatchType type);
+
+ /**
* Returns true if this canonical query may have converted extensions such as $where and $text
* into no-ops during parsing. This will be the case if it allowed $where and $text in parsing,
* but parsed using an ExtensionsCallbackNoop. This does not guarantee that a $where or $text
@@ -257,17 +265,23 @@ public:
return _pipeline;
}
- void optimizeProjection() {
- if (_proj) {
- _proj->optimize();
- }
- }
-
private:
- Status init(boost::intrusive_ptr<ExpressionContext> expCtx,
- std::unique_ptr<ParsedFindCommand> parsedFind,
- std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline,
- bool optimizeMatchExpression);
+ // You must go through canonicalize to create a CanonicalQuery.
+ CanonicalQuery() {}
+
+ Status init(OperationContext* opCtx,
+ boost::intrusive_ptr<ExpressionContext> expCtx,
+ std::unique_ptr<FindCommandRequest> findCommand,
+ bool canHaveNoopMatchNodes,
+ std::unique_ptr<MatchExpression> root,
+ const ProjectionPolicies& projectionPolicies,
+ std::vector<std::unique_ptr<InnerPipelineStageInterface>> pipeline);
+
+ // Initializes '_sortPattern', adding any metadata dependencies implied by the sort.
+ //
+ // Throws a UserException if the sort is illegal, or if any metadata type in
+ // 'unavailableMetadata' is required.
+ void initSortPattern(QueryMetadataBitSet unavailableMetadata);
boost::intrusive_ptr<ExpressionContext> _expCtx;
diff --git a/src/mongo/db/query/canonical_query_encoder.cpp b/src/mongo/db/query/canonical_query_encoder.cpp
index c8a8cd29d48..ccbcce36635 100644
--- a/src/mongo/db/query/canonical_query_encoder.cpp
+++ b/src/mongo/db/query/canonical_query_encoder.cpp
@@ -74,6 +74,32 @@ bool isQueryNegatingEqualToNull(const mongo::MatchExpression* tree) {
namespace {
+// Delimiters for cache key encoding.
+const char kEncodeChildrenBegin = '[';
+const char kEncodeChildrenEnd = ']';
+const char kEncodeChildrenSeparator = ',';
+const char kEncodeCollationSection = '#';
+const char kEncodeProjectionSection = '|';
+const char kEncodeProjectionRequirementSeparator = '-';
+const char kEncodeRegexFlagsSeparator = '/';
+const char kEncodeSortSection = '~';
+const char kEncodeEngineSection = '@';
+
+// These special bytes are used in the encoding of auto-parameterized match expressions in the SBE
+// plan cache key.
+
+// Precedes the id number of a parameter marker.
+const char kEncodeParamMarker = '?';
+// Precedes the encoding of a constant when that constant has not been auto-paramterized. The
+// constant is typically encoded as a BSON type byte followed by a BSON value (without the
+// BSONElement's field name).
+const char kEncodeConstantLiteralMarker = ':';
+// Precedes a byte which encodes the bounds tightness associated with a predicate. The structure of
+// the plan (i.e. presence of filters) is affected by bounds tightness. Therefore, if different
+// parameter values can result in different tightnesses, this must be explicitly encoded into the
+// plan cache key.
+const char kEncodeBoundsTightnessDiscriminator = ':';
+
/**
* AppendChar provides the compiler with a type for a "appendChar(...)" member function.
*/
@@ -558,7 +584,7 @@ void encodeKeyForProj(const projection_ast::Projection* proj, StringBuilder* key
return;
}
- auto requiredFields = proj->getRequiredFields();
+ std::set<std::string> requiredFields = proj->getRequiredFields();
// If the only requirement is that $sortKey be included with some value, we just act as if the
// entire document is needed.
diff --git a/src/mongo/db/query/canonical_query_encoder.h b/src/mongo/db/query/canonical_query_encoder.h
index cf124655681..3164ddbec67 100644
--- a/src/mongo/db/query/canonical_query_encoder.h
+++ b/src/mongo/db/query/canonical_query_encoder.h
@@ -33,38 +33,6 @@
namespace mongo {
-// Delimiters for canonical query portion of cache key encoding.
-inline constexpr char kEncodeChildrenBegin = '[';
-inline constexpr char kEncodeChildrenEnd = ']';
-inline constexpr char kEncodeChildrenSeparator = ',';
-inline constexpr char kEncodeCollationSection = '#';
-inline constexpr char kEncodeProjectionSection = '|';
-inline constexpr char kEncodeProjectionRequirementSeparator = '-';
-inline constexpr char kEncodeRegexFlagsSeparator = '/';
-inline constexpr char kEncodeSortSection = '~';
-inline constexpr char kEncodeEngineSection = '@';
-
-// These special bytes are used in the encoding of auto-parameterized match expressions in the SBE
-// plan cache key.
-
-// Precedes the id number of a parameter marker.
-inline constexpr char kEncodeParamMarker = '?';
-// Precedes the encoding of a constant when that constant has not been auto-paramterized. The
-// constant is typically encoded as a BSON type byte followed by a BSON value (without the
-// BSONElement's field name).
-inline constexpr char kEncodeConstantLiteralMarker = ':';
-// Precedes a byte which encodes the bounds tightness associated with a predicate. The structure of
-// the plan (i.e. presence of filters) is affected by bounds tightness. Therefore, if different
-// parameter values can result in different tightnesses, this must be explicitly encoded into the
-// plan cache key.
-inline constexpr char kEncodeBoundsTightnessDiscriminator = ':';
-
-// Delimiters for the discriminator portion of the cache key encoding.
-inline constexpr char kEncodeDiscriminatorsBegin = '<';
-inline constexpr char kEncodeDiscriminatorsEnd = '>';
-inline constexpr char kEncodeGlobalDiscriminatorsBegin = '(';
-inline constexpr char kEncodeGlobalDiscriminatorsEnd = ')';
-
/**
* Returns true if the query predicate involves a negation of an EQ, LTE, or GTE comparison to
* 'null'.
diff --git a/src/mongo/db/query/canonical_query_test.cpp b/src/mongo/db/query/canonical_query_test.cpp
index 5cba0bcc3f4..dbb2fd929da 100644
--- a/src/mongo/db/query/canonical_query_test.cpp
+++ b/src/mongo/db/query/canonical_query_test.cpp
@@ -279,52 +279,11 @@ TEST(CanonicalQueryTest, CanonicalizeFromBaseQuery) {
assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(findCommand), isExplain));
MatchExpression* firstClauseExpr = baseCq->root()->getChild(0);
- auto childCq = assertGet(CanonicalQuery::makeForSubplanner(opCtx.get(), *baseCq, 0));
+ auto childCq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), *baseCq, firstClauseExpr));
- ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getFilter(), firstClauseExpr->serialize());
-
- ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getProjection(),
- baseCq->getFindCommandRequest().getProjection());
- ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getSort(),
- baseCq->getFindCommandRequest().getSort());
- ASSERT_TRUE(childCq->getExplain());
-}
-
-TEST(CanonicalQueryTest, CanonicalizeFromBaseQueryWithSpecialFeature) {
- // Like the above test, but use $text which is a 'special feature' not always allowed. This is
- // meant to reproduce SERVER-XYZ.
- QueryTestServiceContext serviceContext;
- auto opCtx = serviceContext.makeOperationContext();
-
- const bool isExplain = true;
- const std::string cmdStr = R"({
- find:'bogusns',
- filter: {
- $or:[
- {a: 'foo'},
- {$text: {$search: 'bar'}}
- ]
- },
- projection: {a:1},
- sort: {b:1},
- $db: 'test'
- })";
- auto findCommand = query_request_helper::makeFromFindCommandForTests(fromjson(cmdStr));
- auto baseCq =
- assertGet(CanonicalQuery::canonicalize(opCtx.get(),
- std::move(findCommand),
- isExplain,
- nullptr,
- ExtensionsCallbackNoop(),
- MatchExpressionParser::kAllowAllSpecialFeatures));
-
- // Note: be sure to use the second child to get $text, since we 'normalize' and sort the
- // MatchExpression tree as part of canonicalization. This will put the text search clause
- // second.
- MatchExpression* secondClauseExpr = baseCq->root()->getChild(1);
- auto childCq = assertGet(CanonicalQuery::makeForSubplanner(opCtx.get(), *baseCq, 1));
-
- ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getFilter(), secondClauseExpr->serialize());
+ BSONObjBuilder expectedFilter;
+ firstClauseExpr->serialize(&expectedFilter);
+ ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getFilter(), expectedFilter.obj());
ASSERT_BSONOBJ_EQ(childCq->getFindCommandRequest().getProjection(),
baseCq->getFindCommandRequest().getProjection());
@@ -361,7 +320,8 @@ TEST(CanonicalQueryTest, CanonicalQueryFromBaseQueryWithNoCollation) {
auto findCommand = std::make_unique<FindCommandRequest>(nss);
findCommand->setFilter(fromjson("{$or:[{a:1,b:1},{a:1,c:1}]}"));
auto baseCq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(findCommand)));
- auto childCq = assertGet(CanonicalQuery::makeForSubplanner(opCtx.get(), *baseCq, 0));
+ MatchExpression* firstClauseExpr = baseCq->root()->getChild(0);
+ auto childCq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), *baseCq, firstClauseExpr));
ASSERT_TRUE(baseCq->getCollator() == nullptr);
ASSERT_TRUE(childCq->getCollator() == nullptr);
}
@@ -375,7 +335,8 @@ TEST(CanonicalQueryTest, CanonicalQueryFromBaseQueryWithCollation) {
findCommand->setCollation(BSON("locale"
<< "reverse"));
auto baseCq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(findCommand)));
- auto childCq = assertGet(CanonicalQuery::makeForSubplanner(opCtx.get(), *baseCq, 0));
+ MatchExpression* firstClauseExpr = baseCq->root()->getChild(0);
+ auto childCq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), *baseCq, firstClauseExpr));
ASSERT(baseCq->getCollator());
ASSERT(childCq->getCollator());
ASSERT_TRUE(*(childCq->getCollator()) == *(baseCq->getCollator()));
diff --git a/src/mongo/db/query/classic_plan_cache.h b/src/mongo/db/query/classic_plan_cache.h
index e510b48dd6d..8d2e6889bfb 100644
--- a/src/mongo/db/query/classic_plan_cache.h
+++ b/src/mongo/db/query/classic_plan_cache.h
@@ -235,12 +235,7 @@ using PlanCacheEntry = PlanCacheEntryBase<SolutionCacheData, plan_cache_debug_in
using CachedSolution = CachedPlanHolder<SolutionCacheData, plan_cache_debug_info::DebugInfo>;
struct BudgetEstimator {
- /**
- * This estimator function is called when an entry is added or removed to LRU cache in order to
- * make sure the total plan cache size does not exceed the maximum size.
- */
-
- size_t operator()(const PlanCacheKey&, const std::shared_ptr<const PlanCacheEntry>&) {
+ size_t operator()(const std::shared_ptr<const PlanCacheEntry>&) {
return 1;
}
};
diff --git a/src/mongo/db/query/collection_query_info.cpp b/src/mongo/db/query/collection_query_info.cpp
index ff4e430f6a1..e624389b642 100644
--- a/src/mongo/db/query/collection_query_info.cpp
+++ b/src/mongo/db/query/collection_query_info.cpp
@@ -90,13 +90,12 @@ CollectionQueryInfo::PlanCacheState::PlanCacheState(OperationContext* opCtx,
// TODO We shouldn't need to include unfinished indexes, but we must here because the index
// catalog may be in an inconsistent state. SERVER-18346.
- auto ii = collection->getIndexCatalog()->getIndexIterator(
- opCtx, IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished);
+ const bool includeUnfinishedIndexes = true;
+ std::unique_ptr<IndexCatalog::IndexIterator> ii =
+ collection->getIndexCatalog()->getIndexIterator(opCtx, includeUnfinishedIndexes);
while (ii->more()) {
const IndexCatalogEntry* ice = ii->next();
- if (ice->accessMethod()) {
- indexCores.emplace_back(indexInfoFromIndexCatalogEntry(*ice));
- }
+ indexCores.emplace_back(indexInfoFromIndexCatalogEntry(*ice));
}
planCacheIndexabilityState.updateDiscriminators(indexCores);
@@ -118,8 +117,8 @@ const UpdateIndexData& CollectionQueryInfo::getIndexKeys(OperationContext* opCtx
void CollectionQueryInfo::computeIndexKeys(OperationContext* opCtx, const CollectionPtr& coll) {
_indexedPaths.clear();
- auto it = coll->getIndexCatalog()->getIndexIterator(
- opCtx, IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished);
+ std::unique_ptr<IndexCatalog::IndexIterator> it =
+ coll->getIndexCatalog()->getIndexIterator(opCtx, true);
while (it->more()) {
const IndexCatalogEntry* entry = it->next();
const IndexDescriptor* descriptor = entry->descriptor();
@@ -176,10 +175,10 @@ void CollectionQueryInfo::computeIndexKeys(OperationContext* opCtx, const Collec
// handle partial indexes
const MatchExpression* filter = entry->getFilterExpression();
if (filter) {
- RelevantFieldIndexMap paths;
+ stdx::unordered_set<std::string> paths;
QueryPlannerIXSelect::getFields(filter, &paths);
for (auto it = paths.begin(); it != paths.end(); ++it) {
- _indexedPaths.addPath(FieldRef(it->first));
+ _indexedPaths.addPath(FieldRef(*it));
}
}
}
@@ -239,8 +238,9 @@ void CollectionQueryInfo::updatePlanCacheIndexEntries(OperationContext* opCtx,
}
void CollectionQueryInfo::init(OperationContext* opCtx, const CollectionPtr& coll) {
- auto ii =
- coll->getIndexCatalog()->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady);
+ const bool includeUnfinishedIndexes = false;
+ std::unique_ptr<IndexCatalog::IndexIterator> ii =
+ coll->getIndexCatalog()->getIndexIterator(opCtx, includeUnfinishedIndexes);
while (ii->more()) {
const IndexDescriptor* desc = ii->next()->descriptor();
CollectionIndexUsageTrackerDecoration::get(coll->getSharedDecorations())
diff --git a/src/mongo/db/query/datetime/date_time_support.cpp b/src/mongo/db/query/datetime/date_time_support.cpp
index 93859935c98..1fe0ecd81d0 100644
--- a/src/mongo/db/query/datetime/date_time_support.cpp
+++ b/src/mongo/db/query/datetime/date_time_support.cpp
@@ -842,7 +842,7 @@ StringData serializeTimeUnit(TimeUnit unit) {
case TimeUnit::millisecond:
return "millisecond"_sd;
}
- MONGO_UNREACHABLE_TASSERT(5339903);
+ MONGO_UNREACHABLE_TASSERT(5339900);
}
DayOfWeek parseDayOfWeek(StringData dayOfWeek) {
diff --git a/src/mongo/db/query/datetime/date_time_support_test.cpp b/src/mongo/db/query/datetime/date_time_support_test.cpp
index df6e97c979a..d2b2e3b2d44 100644
--- a/src/mongo/db/query/datetime/date_time_support_test.cpp
+++ b/src/mongo/db/query/datetime/date_time_support_test.cpp
@@ -2629,10 +2629,7 @@ TEST(DateAdd, DateAddWithTimezoneDST) {
{europeAmsterdamZone.createFromDateParts(2020, 10, 24, 2, 0, 1, 0),
TimeUnit::day,
1,
- europeAmsterdamZone.createFromDateParts(2020, 10, 25, 1, 59, 59, 0) +
- Milliseconds{2000}}, // as this date is ambiguous (it could in both timezones, with or
- // without DST) and the computation is expected to return the
- // "with DST" one, obtain it via a computation
+ europeAmsterdamZone.createFromDateParts(2020, 10, 25, 2, 0, 1, 0)},
{europeAmsterdamZone.createFromDateParts(2020, 10, 24, 3, 0, 1, 0),
TimeUnit::day,
1,
@@ -2724,13 +2721,10 @@ TEST(DateAdd, DateAddWithTimezoneDST) {
TimeUnit::day,
1,
newYorkZone.createFromDateParts(2020, 11, 2, 1, 30, 0, 0)},
- {newYorkZone.createFromDateParts(2020, 10, 31, 1, 0, 1, 0),
+ {newYorkZone.createFromDateParts(2020, 10, 31, 1, 30, 0, 0),
TimeUnit::day,
1,
- newYorkZone.createFromDateParts(2020, 11, 1, 0, 59, 59, 0) +
- Milliseconds{2000}}, // as this date is ambiguous (it could in both timezones, with or
- // without DST) and the computation is expected to return the
- // "with DST" one, obtain it via a computation
+ newYorkZone.createFromDateParts(2020, 11, 1, 1, 30, 0, 0)},
{newYorkZone.createFromDateParts(2020, 11, 1, 3, 0, 0, 0),
TimeUnit::day,
-1,
@@ -2791,20 +2785,15 @@ TEST(DateAdd, DateAdd_LordHoweTimezoneDST) {
auto australiaLordHoweZone = kDefaultTimeZoneDatabase.getTimeZone("Australia/Lord_Howe");
std::vector<TestCase> tests{
// DST to Standard change: 2021-04-04T02:00:00 -> 2021-04-04T01:30:00 Lord Howe timezone.
- {australiaLordHoweZone.createFromDateParts(2021, 4, 4, 1, 29, 59, 0) +
- Milliseconds{1000}, // as this date is ambiguous (it could in both timezones, with or
- // without DST) and the computation is expected to start from the
- // "with DST" one, obtain it via a computation
+ {australiaLordHoweZone.createFromDateParts(2021, 4, 4, 1, 30, 0, 0),
TimeUnit::day,
1,
australiaLordHoweZone.createFromDateParts(2021, 4, 5, 1, 30, 0, 0)},
- {australiaLordHoweZone.createFromDateParts(2021, 4, 3, 1, 30, 1, 0),
+ {australiaLordHoweZone.createFromDateParts(2021, 4, 3, 1, 45, 0, 0),
TimeUnit::day,
1,
- australiaLordHoweZone.createFromDateParts(2021, 4, 4, 1, 29, 59, 0) +
- Milliseconds{2000}}, // as this date is ambiguous (it could in both timezones, with or
- // without DST) and the computation is expected to return the
- // "with DST" one, obtain it via a computation
+ // Computed time falls into the repeated 1/2 hour.
+ australiaLordHoweZone.createFromDateParts(2021, 4, 4, 1, 45, 0, 0)},
{australiaLordHoweZone.createFromDateParts(2021, 4, 5, 1, 0, 0, 0),
TimeUnit::day,
-1,
diff --git a/src/mongo/db/query/explain.cpp b/src/mongo/db/query/explain.cpp
index 796875bca68..414badb8332 100644
--- a/src/mongo/db/query/explain.cpp
+++ b/src/mongo/db/query/explain.cpp
@@ -124,7 +124,7 @@ void generatePlannerInfo(PlanExecutor* exec,
auto query = exec->getCanonicalQuery();
if (nullptr != query) {
BSONObjBuilder parsedQueryBob(plannerBob.subobjStart("parsedQuery"));
- query->root()->serialize(&parsedQueryBob, {});
+ query->root()->serialize(&parsedQueryBob);
parsedQueryBob.doneFast();
if (query->getCollator()) {
diff --git a/src/mongo/db/query/find.cpp b/src/mongo/db/query/find.cpp
index cd8e21bf6c7..d2f36155e13 100644
--- a/src/mongo/db/query/find.cpp
+++ b/src/mongo/db/query/find.cpp
@@ -107,28 +107,19 @@ void endQueryOp(OperationContext* opCtx,
const CollectionPtr& collection,
const PlanExecutor& exec,
long long numResults,
- boost::optional<ClientCursorPin&> cursor,
- const BSONObj& cmdObj) {
+ CursorId cursorId) {
auto curOp = CurOp::get(opCtx);
- // Fill out basic CurOp query exec properties. More metrics (nreturned and executionTime)
- // are collected within collectQueryStatsMongod.
- curOp->debug().cursorid = (cursor.has_value() ? cursor->getCursor()->cursorid() : -1);
- curOp->debug().cursorExhausted = !cursor.has_value();
- curOp->debug().additiveMetrics.nBatches = 1;
+ // Fill out basic CurOp query exec properties.
+ curOp->debug().nreturned = numResults;
+ curOp->debug().cursorid = (0 == cursorId ? -1 : cursorId);
+ curOp->debug().cursorExhausted = (0 == cursorId);
// Fill out CurOp based on explain summary statistics.
PlanSummaryStats summaryStats;
auto&& explainer = exec.getPlanExplainer();
explainer.getSummaryStats(&summaryStats);
curOp->debug().setPlanSummaryMetrics(summaryStats);
- curOp->setEndOfOpMetrics(numResults);
-
- if (cursor) {
- collectQueryStatsMongod(opCtx, *cursor);
- } else {
- collectQueryStatsMongod(opCtx, std::move(curOp->debug().queryStatsInfo.key));
- }
if (collection) {
CollectionQueryInfo::get(collection).notifyOfQuery(opCtx, collection, summaryStats);
diff --git a/src/mongo/db/query/find.h b/src/mongo/db/query/find.h
index 235ebf91ed8..2b45efcd602 100644
--- a/src/mongo/db/query/find.h
+++ b/src/mongo/db/query/find.h
@@ -73,7 +73,6 @@ void endQueryOp(OperationContext* opCtx,
const CollectionPtr& collection,
const PlanExecutor& exec,
long long numResults,
- boost::optional<ClientCursorPin&> cursor,
- const BSONObj& cmdObj);
+ CursorId cursorId);
} // namespace mongo
diff --git a/src/mongo/db/query/find_common.cpp b/src/mongo/db/query/find_common.cpp
index 0078533cc60..57f9f3954b2 100644
--- a/src/mongo/db/query/find_common.cpp
+++ b/src/mongo/db/query/find_common.cpp
@@ -133,17 +133,5 @@ std::size_t FindCommon::getBytesToReserveForGetMoreReply(bool isTailable,
// command metadata to the reply.
return kMaxBytesToReturnToClientAtOnce;
}
-bool FindCommon::BSONArrayResponseSizeTracker::haveSpaceForNext(const BSONObj& document) {
- return FindCommon::haveSpaceForNext(document, _numberOfDocuments, _bsonArraySizeInBytes);
-}
-void FindCommon::BSONArrayResponseSizeTracker::add(const BSONObj& document) {
- dassert(haveSpaceForNext(document));
- ++_numberOfDocuments;
- _bsonArraySizeInBytes += (document.objsize() + kPerDocumentOverheadBytesUpperBound);
-}
-// Upper bound of BSON array element overhead. The overhead is 1 byte/doc for the type + 1 byte/doc
-// for the field name's null terminator + 1 byte per digit of the maximum array index value.
-const size_t FindCommon::BSONArrayResponseSizeTracker::kPerDocumentOverheadBytesUpperBound{
- 2 + std::to_string(BSONObjMaxUserSize / BSONObj::kMinBSONLength).length()};
} // namespace mongo
diff --git a/src/mongo/db/query/find_common.h b/src/mongo/db/query/find_common.h
index d38d580bdd2..45f60d2fd51 100644
--- a/src/mongo/db/query/find_common.h
+++ b/src/mongo/db/query/find_common.h
@@ -89,7 +89,7 @@ public:
// This max may be exceeded by epsilon for output documents that approach the maximum user
// document size. That is, if we must return a BSONObjMaxUserSize document, then the total
// response size will be BSONObjMaxUserSize plus the amount of size required for the message
- // header and the cursor response "envelope". (The envelope contains namespace and cursor id
+ // header and the cursor response "envelope". (The envolope contains namespace and cursor id
// info.)
static const size_t kMaxBytesToReturnToClientAtOnce;
@@ -148,32 +148,6 @@ public:
static std::size_t getBytesToReserveForGetMoreReply(bool isTailable,
size_t firstResultSize,
size_t batchSize);
-
- /**
- * Tracker of a size of a server response presented as a BSON array. Facilitates limiting the
- * server response size to 16MB + certain epsilon. Accounts for array element and it's overhead
- * size. Does not account for response "envelope" size.
- */
- class BSONArrayResponseSizeTracker {
- // Upper bound of BSON array element overhead.
- static const size_t kPerDocumentOverheadBytesUpperBound;
-
- public:
- /**
- * Returns true only if 'document' can be added to the BSON array without violating the
- * overall response size limit or if it is the first document.
- */
- bool haveSpaceForNext(const BSONObj& document);
-
- /**
- * Records that 'document' was added to the response.
- */
- void add(const BSONObj& document);
-
- private:
- std::size_t _numberOfDocuments{0};
- std::size_t _bsonArraySizeInBytes{0};
- };
};
} // namespace mongo
diff --git a/src/mongo/db/query/find_common_test.cpp b/src/mongo/db/query/find_common_test.cpp
deleted file mode 100644
index d7dfc10d950..00000000000
--- a/src/mongo/db/query/find_common_test.cpp
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/platform/basic.h"
-
-#include <string>
-
-#include "mongo/bson/bsonobj.h"
-#include "mongo/bson/bsonobjbuilder.h"
-#include "mongo/db/query/find_common.h"
-
-#include "mongo/unittest/unittest.h"
-
-namespace {
-
-using namespace mongo;
-
-TEST(BSONArrayResponseSizeTrackerTest, AddLargeNumberOfElements) {
- BSONObjBuilder bsonObjBuilder;
- {
- FindCommon::BSONArrayResponseSizeTracker sizeTracker;
- BSONArrayBuilder arrayBuilder{bsonObjBuilder.subarrayStart("a")};
- BSONObj emptyObject;
- while (sizeTracker.haveSpaceForNext(emptyObject)) {
- sizeTracker.add(emptyObject);
- arrayBuilder.append(emptyObject);
- }
- }
- // If the BSON object is successfully constructed, then space accounting was correct.
- bsonObjBuilder.obj();
-}
-TEST(BSONArrayResponseSizeTrackerTest, CanAddAtLeastOneDocument) {
- auto largeObject = BSON("a" << std::string(16 * 1024 * 1024, 'A'));
- BSONObj emptyObject;
- BSONObjBuilder bsonObjBuilder;
- {
- FindCommon::BSONArrayResponseSizeTracker sizeTracker;
- BSONArrayBuilder arrayBuilder{bsonObjBuilder.subarrayStart("a")};
- // Add an object that is larger than 16MB.
- ASSERT(sizeTracker.haveSpaceForNext(largeObject));
- sizeTracker.add(largeObject);
- arrayBuilder.append(largeObject);
- ASSERT(!sizeTracker.haveSpaceForNext(emptyObject));
- }
- // If the BSON object is successfully constructed, then space accounting was correct.
- bsonObjBuilder.obj();
-}
-} // namespace
diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp
index 602be15dbc2..db04d6a276a 100644
--- a/src/mongo/db/query/get_executor.cpp
+++ b/src/mongo/db/query/get_executor.cpp
@@ -29,13 +29,10 @@
#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
-#include "mongo/db/curop.h"
#include "mongo/platform/basic.h"
#include "mongo/db/query/get_executor.h"
-#include "mongo/util/duration.h"
-#include "mongo/util/tick_source.h"
#include <boost/optional.hpp>
#include <limits>
#include <memory>
@@ -110,7 +107,6 @@
#include "mongo/logv2/log.h"
#include "mongo/scripting/engine.h"
#include "mongo/util/str.h"
-#include "mongo/util/timer.h"
namespace mongo {
MONGO_FAIL_POINT_DEFINE(includeFakeColumnarIndex);
@@ -220,15 +216,10 @@ IndexEntry indexEntryFromIndexCatalogEntry(OperationContext* opCtx,
MultikeyMetadataAccessStats mkAccessStats;
if (canonicalQuery) {
- RelevantFieldIndexMap fieldIndexProps;
- QueryPlannerIXSelect::getFields(canonicalQuery->root(), &fieldIndexProps);
- stdx::unordered_set<std::string> projectedFields;
- for (auto&& [fieldName, _] : fieldIndexProps) {
- if (projection_executor_utils::applyProjectionToOneField(
- wildcardProjection->exec(), fieldName)) {
- projectedFields.insert(fieldName);
- }
- }
+ stdx::unordered_set<std::string> fields;
+ QueryPlannerIXSelect::getFields(canonicalQuery->root(), &fields);
+ const auto projectedFields = projection_executor_utils::applyProjectionToFields(
+ wildcardProjection->exec(), fields);
multikeyPathSet =
getWildcardMultikeyPathSet(wam, opCtx, projectedFields, &mkAccessStats);
@@ -293,8 +284,7 @@ void fillOutIndexEntries(OperationContext* opCtx,
const CanonicalQuery* canonicalQuery,
const CollectionPtr& collection,
std::vector<IndexEntry>& entries) {
- auto ii = collection->getIndexCatalog()->getIndexIterator(
- opCtx, IndexCatalog::InclusionPolicy::kReady);
+ auto ii = collection->getIndexCatalog()->getIndexIterator(opCtx, false);
while (ii->more()) {
const IndexCatalogEntry* ice = ii->next();
@@ -607,8 +597,6 @@ public:
StatusWith<std::unique_ptr<ResultType>> prepare() {
const auto& mainColl = getMainCollection();
-
- ON_BLOCK_EXIT([&] { CurOp::get(_opCtx)->stopQueryPlanningTimer(); });
if (!mainColl) {
LOGV2_DEBUG(20921,
2,
@@ -708,8 +696,10 @@ public:
"Only one plan is available",
"query"_attr = redact(_cq->toStringShort()),
"planSummary"_attr = result->getPlanSummary());
+
return std::move(result);
}
+
return buildMultiPlan(std::move(solutions));
}
@@ -1278,12 +1268,12 @@ std::unique_ptr<sbe::RuntimePlanner> makeRuntimePlannerIfNeeded(
return nullptr;
}
-std::unique_ptr<PlanYieldPolicySBE> makeSbeYieldPolicy(OperationContext* opCtx,
- PlanYieldPolicy::YieldPolicy policy,
- const Yieldable* yieldable,
- NamespaceString nss) {
- return std::make_unique<PlanYieldPolicySBE>(opCtx,
- policy,
+std::unique_ptr<PlanYieldPolicySBE> makeSbeYieldPolicy(
+ OperationContext* opCtx,
+ PlanYieldPolicy::YieldPolicy requestedYieldPolicy,
+ const Yieldable* yieldable,
+ NamespaceString nss) {
+ return std::make_unique<PlanYieldPolicySBE>(requestedYieldPolicy,
opCtx->getServiceContext()->getFastClockSource(),
internalQueryExecYieldIterations.load(),
Milliseconds{internalQueryExecYieldPeriodMS.load()},
@@ -1321,16 +1311,16 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getSlotBasedExe
auto&& [roots, solutions] = planningResult->extractResultData();
// In some circumstances (e.g. when have multiple candidate plans or using a cached one), we
// might need to execute the plan(s) to pick the best one or to confirm the choice.
- if (auto runTimePlanner = makeRuntimePlannerIfNeeded(opCtx,
- collections,
- cq.get(),
- solutions.size(),
- planningResult->decisionWorks(),
- planningResult->needsSubplanning(),
- yieldPolicy.get(),
- plannerParams.options)) {
+ if (auto planner = makeRuntimePlannerIfNeeded(opCtx,
+ collections,
+ cq.get(),
+ solutions.size(),
+ planningResult->decisionWorks(),
+ planningResult->needsSubplanning(),
+ yieldPolicy.get(),
+ plannerParams.options)) {
// Do the runtime planning and pick the best candidate plan.
- auto candidates = runTimePlanner->plan(std::move(solutions), std::move(roots));
+ auto candidates = planner->plan(std::move(solutions), std::move(roots));
return plan_executor_factory::make(opCtx,
std::move(cq),
@@ -1401,11 +1391,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutor(
}
}
- // There's a special case of the projection optimization being skipped when a query has any
- // user-defined "let" variable and the query may be run with SBE. Here we make sure the
- // projection is optimized for the classic engine.
- canonicalQuery->optimizeProjection();
-
return getClassicExecutor(
opCtx, mainColl, std::move(canonicalQuery), yieldPolicy, plannerParams);
}
@@ -1418,7 +1403,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutor(
PlanYieldPolicy::YieldPolicy yieldPolicy,
size_t plannerOptions) {
MultipleCollectionAccessor multi{collection};
-
return getExecutor(opCtx,
multi,
std::move(canonicalQuery),
@@ -1438,8 +1422,9 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorFind
std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages,
bool permitYield,
QueryPlannerParams plannerParams) {
- auto yieldPolicy = permitYield ? PlanYieldPolicy::YieldPolicy::YIELD_AUTO
- : PlanYieldPolicy::YieldPolicy::INTERRUPT_ONLY;
+ auto yieldPolicy = (permitYield && !opCtx->inMultiDocumentTransaction())
+ ? PlanYieldPolicy::YieldPolicy::YIELD_AUTO
+ : PlanYieldPolicy::YieldPolicy::INTERRUPT_ONLY;
if (OperationShardingState::isComingFromRouter(opCtx)) {
plannerParams.options |= QueryPlannerParams::INCLUDE_SHARD_FILTER;
@@ -1460,7 +1445,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorFind
std::function<void(CanonicalQuery*)> extractAndAttachPipelineStages,
bool permitYield,
size_t plannerOptions) {
-
MultipleCollectionAccessor multi{*coll};
return getExecutorFind(opCtx,
multi,
@@ -1542,13 +1526,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorDele
expCtx->setIsCappedDelete();
}
- // If the parsed delete does not have a user-specified collation, set it from the collection
- // default.
- if (collection && parsedDelete->getRequest()->getCollation().isEmpty() &&
- collection->getDefaultCollator()) {
- parsedDelete->setCollator(collection->getDefaultCollator()->clone());
- }
-
if (collection && collection->isCapped() && opCtx->inMultiDocumentTransaction()) {
// This check is duplicated from CollectionImpl::deleteDocument() for two reasons:
// - Performing a remove on an empty capped collection would not call
@@ -1676,7 +1653,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorDele
ClassicPrepareExecutionHelper helper{
opCtx, collection, ws.get(), cq.get(), nullptr, defaultPlannerOptions};
auto executionResult = helper.prepare();
-
if (!executionResult.isOK()) {
return executionResult.getStatus();
}
@@ -1686,9 +1662,9 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorDele
deleteStageParams->canonicalQuery = cq.get();
const bool batchDelete =
- (deleteStageParams->isMulti && !opCtx->inMultiDocumentTransaction() &&
- !deleteStageParams->fromMigrate && !deleteStageParams->returnDeleted &&
- deleteStageParams->sort.isEmpty() && !deleteStageParams->numStatsForDoc) &&
+ (deleteStageParams->isMulti && !deleteStageParams->fromMigrate &&
+ !deleteStageParams->returnDeleted && deleteStageParams->sort.isEmpty() &&
+ !deleteStageParams->numStatsForDoc) &&
((gInternalBatchUserMultiDeletesForTest.load() &&
nss.ns() == "__internalBatchedDeletesTesting.Collection0") ||
(batchDeletesByDefault.shouldFail()));
@@ -1864,7 +1840,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorUpda
ClassicPrepareExecutionHelper helper{
opCtx, collection, ws.get(), cq.get(), nullptr, defaultPlannerOptions};
auto executionResult = helper.prepare();
-
if (!executionResult.isOK()) {
return executionResult.getStatus();
}
@@ -2138,8 +2113,8 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorCoun
OperationContext* opCtx = expCtx->opCtx;
std::unique_ptr<WorkingSet> ws = std::make_unique<WorkingSet>();
- auto findCommand = std::make_unique<FindCommandRequest>(nss);
+ auto findCommand = std::make_unique<FindCommandRequest>(nss);
findCommand->setFilter(request.getQuery());
auto collation = request.getCollation().value_or(BSONObj());
findCommand->setCollation(collation);
@@ -2160,7 +2135,9 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorCoun
}
std::unique_ptr<CanonicalQuery> cq = std::move(statusWithCQ.getValue());
- const auto yieldPolicy = PlanYieldPolicy::YieldPolicy::YIELD_AUTO;
+ const auto yieldPolicy = opCtx->inMultiDocumentTransaction()
+ ? PlanYieldPolicy::YieldPolicy::INTERRUPT_ONLY
+ : PlanYieldPolicy::YieldPolicy::YIELD_AUTO;
const auto skip = request.getSkip().value_or(0);
const auto limit = request.getLimit().value_or(0);
@@ -2213,7 +2190,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorCoun
if (!executionResult.isOK()) {
return executionResult.getStatus();
}
-
auto [root, querySolution] = executionResult.getValue()->extractResultData();
invariant(root);
@@ -2222,7 +2198,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorCoun
expCtx.get(), collection, limit, skip, ws.get(), root.release());
// We must have a tree of stages in order to have a valid plan executor, but the query
// solution may be NULL. Takes ownership of all args other than 'collection' and 'opCtx'
-
return plan_executor_factory::make(std::move(cq),
std::move(ws),
std::move(root),
@@ -2418,8 +2393,8 @@ QueryPlannerParams fillOutPlannerParamsForDistinct(OperationContext* opCtx,
// If the caller did not request a "strict" distinct scan then we may choose a plan which
// unwinds arrays and treats each element in an array as its own key.
const bool mayUnwindArrays = !(plannerOptions & QueryPlannerParams::STRICT_DISTINCT_ONLY);
- auto ii = collection->getIndexCatalog()->getIndexIterator(
- opCtx, IndexCatalog::InclusionPolicy::kReady);
+ std::unique_ptr<IndexCatalog::IndexIterator> ii =
+ collection->getIndexCatalog()->getIndexIterator(opCtx, false);
auto query = parsedDistinct.getQuery()->getFindCommandRequest().getFilter();
while (ii->more()) {
const IndexCatalogEntry* ice = ii->next();
@@ -2657,7 +2632,9 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> getExecutorDist
auto expCtx = parsedDistinct->getQuery()->getExpCtx();
OperationContext* opCtx = expCtx->opCtx;
- const auto yieldPolicy = PlanYieldPolicy::YieldPolicy::YIELD_AUTO;
+ const auto yieldPolicy = opCtx->inMultiDocumentTransaction()
+ ? PlanYieldPolicy::YieldPolicy::INTERRUPT_ONLY
+ : PlanYieldPolicy::YieldPolicy::YIELD_AUTO;
if (!collection) {
// Treat collections that do not exist as empty collections.
diff --git a/src/mongo/db/query/index_bounds.cpp b/src/mongo/db/query/index_bounds.cpp
index 16985de29a0..ecb1f21208b 100644
--- a/src/mongo/db/query/index_bounds.cpp
+++ b/src/mongo/db/query/index_bounds.cpp
@@ -244,24 +244,10 @@ bool OrderedIntervalList::isMinToMax() const {
return intervals.size() == 1 && intervals[0].isMinToMax();
}
-bool OrderedIntervalList::isMaxToMin() const {
- return intervals.size() == 1 && intervals[0].isMaxToMin();
-}
-
bool OrderedIntervalList::isPoint() const {
return intervals.size() == 1 && intervals[0].isPoint();
}
-bool OrderedIntervalList::containsOnlyPointIntervals() const {
- for (const auto& interval : intervals) {
- if (!interval.isPoint()) {
- return false;
- }
- }
-
- return true;
-}
-
// static
void OrderedIntervalList::complement() {
BSONObjBuilder minBob;
diff --git a/src/mongo/db/query/index_bounds.h b/src/mongo/db/query/index_bounds.h
index de457282224..d828f275164 100644
--- a/src/mongo/db/query/index_bounds.h
+++ b/src/mongo/db/query/index_bounds.h
@@ -95,22 +95,12 @@ struct OrderedIntervalList {
bool isMinToMax() const;
/**
- * Returns true if this OIL represents a single [MaxKey, MinKey] bound.
- */
- bool isMaxToMin() const;
-
- /**
* Returns true if this OIL represents a point predicate: [N, N].
*
* These predicates are interesting because if you have an index on {a:1, b:1},
* and a point predicate on 'a', then the index provides a sort on {b: 1}.
*/
bool isPoint() const;
-
- /**
- * Returns true if this OIL contains only point intervals (such as [N, N]).
- */
- bool containsOnlyPointIntervals() const;
};
/**
diff --git a/src/mongo/db/query/index_bounds_builder.cpp b/src/mongo/db/query/index_bounds_builder.cpp
index 3c027bde140..01e13b5f058 100644
--- a/src/mongo/db/query/index_bounds_builder.cpp
+++ b/src/mongo/db/query/index_bounds_builder.cpp
@@ -108,38 +108,13 @@ Interval makeNullPointInterval(bool isHashed) {
return isHashed ? kHashedNullInterval : IndexBoundsBuilder::kNullPointInterval;
}
-/**
- * This helper updates the query bounds tightness for the limited set of conditions where we see a
- * null query that can be covered.
- */
-void updateTightnessForNullQuery(const IndexEntry& index,
- IndexBoundsBuilder::BoundsTightness* tightnessOut) {
- if (index.sparse || index.type == IndexType::INDEX_HASHED) {
- // Sparse indexes and hashed indexes require a FETCH stage with a filter for null queries.
- *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH;
- return;
- }
-
- if (index.multikey) {
- // If we have a simple equality null query and our index is multikey, we cannot cover the
- // query. This is because null intervals are translated into the null and undefined point
- // intervals, and the undefined point interval includes entries for []. In the case of a
- // single null interval, [] should not match.
- *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH;
- return;
- }
-
- // The query may be fully covered by the index if the projection allows it, since the case above
- // about the empty array can only become an issue if there is an empty array present, which
- // would mark the index as multikey.
- *tightnessOut = IndexBoundsBuilder::EXACT_MAYBE_COVERED;
-}
-
void makeNullEqualityBounds(const IndexEntry& index,
bool isHashed,
OrderedIntervalList* oil,
IndexBoundsBuilder::BoundsTightness* tightnessOut) {
- updateTightnessForNullQuery(index, tightnessOut);
+ // An equality to null predicate cannot be covered because the index does not distinguish
+ // between the lack of a value and the literal value null.
+ *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH;
// There are two values that could possibly be equal to null in an index: undefined and null.
oil->intervals.push_back(makeUndefinedPointInterval(isHashed));
@@ -279,10 +254,7 @@ bool IndexBoundsBuilder::canUseCoveredMatching(const MatchExpression* expr,
IndexBoundsBuilder::BoundsTightness tightness;
OrderedIntervalList oil;
translate(expr, BSONElement{}, index, &oil, &tightness, /* iet::Builder */ nullptr);
- // We have additional tightness values (MAYBE_COVERED), but we cannot generally cover those
- // cases unless we have an appropriate projection.
- return tightness == IndexBoundsBuilder::INEXACT_COVERED ||
- tightness == IndexBoundsBuilder::EXACT;
+ return tightness >= IndexBoundsBuilder::INEXACT_COVERED;
}
// static
@@ -432,61 +404,6 @@ const Interval IndexBoundsBuilder::kNullPointInterval =
const Interval IndexBoundsBuilder::kEmptyArrayPointInterval =
IndexBoundsBuilder::makePointInterval(kEmptyArrayElementObj);
-bool detectIfEntireNullIntervalMatchesPredicate(const InMatchExpression* ime,
- const IndexEntry& index) {
- if (!ime->hasNull()) {
- // This isn't a null query.
- return false;
- }
-
- if (index.sparse || (IndexType::INDEX_HASHED == index.type)) {
- // Sparse indexes and hashed indexes still require a FETCH stage with a filter for null
- // queries.
- return false;
- }
-
- // Given the context of having a null $in query with eligible indexes, we may be able to cover
- // some combinations of intervals that we could not cover individually.
- if (index.multikey) {
- // If the path has multiple components and we have a multikey index, we still need a FETCH
- // in order to defend against cases where we have a multikey index on "a". These documents
- // will generate null index keys: {"a.b": null} and {a: [1,2,3]}. However, a query like
- // {"a.b": {$in: [null, []]}} should not match {a: [1, 2, 3]}.
- // TODO SERVER-71021: it may be possible to cover more cases here.
- if (ime->fieldRef()->numParts() > 1) {
- return false;
- }
-
- // We must have an equality to an empty array for this null query to be covered, otherwise,
- // because we generate both null and undefined point intervals for a null query, and because
- // a multikey index reuses the same entry for [] and undefined, we will not be able to cover
- // the query.
- if (!ime->hasEmptyArray()) {
- return false;
- }
- }
-
- return true;
-}
-
-void IndexBoundsBuilder::_mergeTightness(const BoundsTightness& tightness,
- BoundsTightness& tightnessOut) {
- // There is a special case where we may have a covered null query (EXACT_MAYBE_COVERED) and a
- // regex with inexact bounds that doesn't need a FETCH (INEXACT_COVERED). In this case, we want
- // to update the tightness to INEXACT_MAYBE_COVERED, to indicate that we need to check if the
- // projection allows us to cover the query, but ensure that we will have a filter on the index
- // if it turns out we can.
- if (((tightness == BoundsTightness::EXACT_MAYBE_COVERED) &&
- (tightnessOut == BoundsTightness::INEXACT_COVERED)) ||
- ((tightness == BoundsTightness::INEXACT_COVERED) &&
- (tightnessOut == BoundsTightness::EXACT_MAYBE_COVERED))) {
- tightnessOut = BoundsTightness::INEXACT_MAYBE_COVERED;
- } else if (tightness < tightnessOut) {
- // Otherwise, fallback to picking the new tightness if it is looser than the old tightness.
- tightnessOut = tightness;
- }
-}
-
void IndexBoundsBuilder::_translatePredicate(const MatchExpression* expr,
const BSONElement& elt,
const IndexEntry& index,
@@ -1038,45 +955,51 @@ void IndexBoundsBuilder::_translatePredicate(const MatchExpression* expr,
});
const InMatchExpression* ime = static_cast<const InMatchExpression*>(expr);
+
*tightnessOut = IndexBoundsBuilder::EXACT;
// Create our various intervals.
IndexBoundsBuilder::BoundsTightness tightness;
- // We check if the $in predicate satisfies conditions to be a covered null predicate on the
- // basis of indexes, null intervals, and array intervals.
- const bool entireNullIntervalMatchesPredicate =
- detectIfEntireNullIntervalMatchesPredicate(ime, index);
+ bool arrayOrNullPresent = false;
for (auto&& equality : ime->getEqualities()) {
- // First, we generate the bounds the same way that we would do for an individual
- // equality. This will set tightness to the value it should be if this equality is being
- // considered in isolation.
- IndexBoundsBuilder::translateEquality(equality, index, isHashed, oilOut, &tightness);
- if (entireNullIntervalMatchesPredicate &&
- (BSONType::jstNULL == equality.type() ||
- (BSONType::Array == equality.type() && equality.Obj().isEmpty()))) {
- // We may have a covered null query. In this case, we update both empty array and
- // null interval tightness to EXACT_MAYBE_COVERED, as individually they would have a
- // tightness of INEXACT_FETCH. However, we already know we will be able to cover
- // these intervals together if we have appropriate projections. Note that any other
- // intervals that cannot be covered may still require the query to use a FETCH.
- tightness = IndexBoundsBuilder::EXACT_MAYBE_COVERED;
+ translateEquality(equality, index, isHashed, oilOut, &tightness);
+ // The ordering invariant of oil has been violated by the call to translateEquality.
+ arrayOrNullPresent = arrayOrNullPresent || equality.type() == BSONType::jstNULL ||
+ equality.type() == BSONType::Array;
+ if (tightness != IndexBoundsBuilder::EXACT) {
+ *tightnessOut = tightness;
}
- IndexBoundsBuilder::_mergeTightness(tightness, *tightnessOut);
}
for (auto&& regex : ime->getRegexes()) {
translateRegex(regex.get(), index, oilOut, &tightness);
- IndexBoundsBuilder::_mergeTightness(tightness, *tightnessOut);
+ if (tightness != IndexBoundsBuilder::EXACT) {
+ *tightnessOut = tightness;
+ }
+ }
+
+ if (ime->hasNull()) {
+ // A null index key does not always match a null query value so we must fetch the
+ // doc and run a full comparison. See SERVER-4529.
+ // TODO: Do we already set the tightnessOut by calling translateEquality?
+ *tightnessOut = INEXACT_FETCH;
+ }
+
+ if (ime->hasEmptyArray()) {
+ // Empty arrays are indexed as undefined.
+ BSONObjBuilder undefinedBob;
+ undefinedBob.appendUndefined("");
+ oilOut->intervals.push_back(makePointInterval(undefinedBob.obj()));
+ *tightnessOut = IndexBoundsBuilder::INEXACT_FETCH;
}
// Equalities are already sorted and deduped so unionize is unneccesary if no regexes
// are present. Hashed indexes may also cause the bounds to be out-of-order.
- // Arrays and nulls introduce multiple elements that necessitate a sort and deduping.
- if (ime->hasNonScalarOrNonEmptyValues() || index.type == IndexType::INDEX_HASHED) {
+ // Arrays and nulls introduce multiple elements that neccesitate a sort and deduping.
+ if (!ime->getRegexes().empty() || index.type == IndexType::INDEX_HASHED ||
+ arrayOrNullPresent)
unionize(oilOut);
- }
-
} else if (MatchExpression::GEO == expr->matchType()) {
const GeoMatchExpression* gme = static_cast<const GeoMatchExpression*>(expr);
if ("2dsphere" == elt.valueStringDataSafe()) {
@@ -1392,7 +1315,6 @@ void IndexBoundsBuilder::translateEquality(const BSONElement& data,
}
std::sort(oil->intervals.begin(), oil->intervals.end(), IntervalComparison);
-
*tightnessOut = IndexBoundsBuilder::INEXACT_FETCH;
}
diff --git a/src/mongo/db/query/index_bounds_builder.h b/src/mongo/db/query/index_bounds_builder.h
index d1067caa561..b48228328f5 100644
--- a/src/mongo/db/query/index_bounds_builder.h
+++ b/src/mongo/db/query/index_bounds_builder.h
@@ -67,28 +67,16 @@ public:
* increasing tightness. These values are used when we need to do comparison between two
* BoundsTightness values. Such comparisons can answer questions such as "Does predicate
* X have tighter or looser bounds than predicate Y?".
- *
- * These enum values are ordered from loosest to tightest.
*/
enum BoundsTightness {
// Index bounds are inexact, and a fetch is required.
INEXACT_FETCH = 0,
- // Index bounds are inexact, and a fetch may be required depending on the projection.
- // For example, a count $in query on null + a regex can be covered, but a find query with
- // the same filter and no projection cannot.
- INEXACT_MAYBE_COVERED = 1,
-
- // Index bounds are exact, but a fetch may be required depending on the projection.
- // For example, a find query on null may be covered, depending on which fields we project
- // out.
- EXACT_MAYBE_COVERED = 2,
-
- // Index bounds are inexact, but no fetch is required.
- INEXACT_COVERED = 3,
+ // Index bounds are inexact, but no fetch is required
+ INEXACT_COVERED = 1,
// Index bounds are exact.
- EXACT = 4
+ EXACT = 2
};
/**
@@ -313,11 +301,6 @@ private:
OrderedIntervalList* oilOut,
BoundsTightness* tightnessOut,
interval_evaluation_tree::Builder* ietBuilder);
-
- /**
- * Helper method for merging interval tightness for $in expressions.
- */
- static void _mergeTightness(const BoundsTightness& tightness, BoundsTightness& tightnessOut);
};
} // namespace mongo
diff --git a/src/mongo/db/query/index_bounds_builder_eq_null_test.cpp b/src/mongo/db/query/index_bounds_builder_eq_null_test.cpp
index 31e17b6e2c8..af4cdf91303 100644
--- a/src/mongo/db/query/index_bounds_builder_eq_null_test.cpp
+++ b/src/mongo/db/query/index_bounds_builder_eq_null_test.cpp
@@ -48,7 +48,7 @@ void assertBoundsRepresentEqualsNull(const OrderedIntervalList& oil) {
oil.intervals[1].compare(Interval(fromjson("{'': null, '': null}"), true, true)));
}
-TEST_F(IndexBoundsBuilderTest, TranslateExprEqualToNullIsExactMaybeCovered) {
+TEST_F(IndexBoundsBuilderTest, TranslateExprEqualToNullIsInexactFetch) {
BSONObj keyPattern = BSON("a" << 1);
BSONElement elt = keyPattern.firstElement();
auto testIndex = buildSimpleIndexEntry(keyPattern);
@@ -65,11 +65,11 @@ TEST_F(IndexBoundsBuilderTest, TranslateExprEqualToNullIsExactMaybeCovered) {
oil.intervals[0].compare(Interval(fromjson("{'': undefined, '': undefined}"), true, true)));
ASSERT_EQUALS(Interval::INTERVAL_EQUALS,
oil.intervals[1].compare(Interval(fromjson("{'': null, '': null}"), true, true)));
- ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT_MAYBE_COVERED);
+ ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_FETCH);
assertIET(inputParamIdMap, ietBuilder, elt, testIndex, oil);
}
-TEST_F(IndexBoundsBuilderTest, TranslateEqualsToNullShouldBuildExactMaybeCoveredBounds) {
+TEST_F(IndexBoundsBuilderTest, TranslateEqualsToNullShouldBuildInexactBounds) {
BSONObj indexPattern = BSON("a" << 1);
auto testIndex = buildSimpleIndexEntry(indexPattern);
@@ -83,12 +83,12 @@ TEST_F(IndexBoundsBuilderTest, TranslateEqualsToNullShouldBuildExactMaybeCovered
expr.get(), indexPattern.firstElement(), testIndex, &oil, &tightness, &ietBuilder);
ASSERT_EQUALS(oil.name, "a");
- ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT_MAYBE_COVERED);
+ ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_FETCH);
assertBoundsRepresentEqualsNull(oil);
assertIET(inputParamIdMap, ietBuilder, indexPattern.firstElement(), testIndex, oil);
}
-TEST_F(IndexBoundsBuilderTest, TranslateDottedEqualsToNullShouldBuildExactMaybeCoveredBounds) {
+TEST_F(IndexBoundsBuilderTest, TranslateDottedEqualsToNullShouldBuildInexactBounds) {
BSONObj indexPattern = BSON("a.b" << 1);
auto testIndex = buildSimpleIndexEntry(indexPattern);
@@ -102,9 +102,7 @@ TEST_F(IndexBoundsBuilderTest, TranslateDottedEqualsToNullShouldBuildExactMaybeC
expr.get(), indexPattern.firstElement(), testIndex, &oil, &tightness, &ietBuilder);
ASSERT_EQUALS(oil.name, "a.b");
- // Depending on the query projection, this will either be converted to EXACT or to INEXACT_FETCH
- // before we build an IXSCAN plan.
- ASSERT_EQUALS(tightness, IndexBoundsBuilder::EXACT_MAYBE_COVERED);
+ ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_FETCH);
assertBoundsRepresentEqualsNull(oil);
assertIET(inputParamIdMap, ietBuilder, indexPattern.firstElement(), testIndex, oil);
}
diff --git a/src/mongo/db/query/index_tag.cpp b/src/mongo/db/query/index_tag.cpp
index ab5acea4de0..29c450d6ae2 100644
--- a/src/mongo/db/query/index_tag.cpp
+++ b/src/mongo/db/query/index_tag.cpp
@@ -120,33 +120,21 @@ void sortUsingTags(MatchExpression* tree) {
});
}
-/**
- * Attaches 'node' to 'target'. If 'target' is an AND, adds 'node' as a child of 'target'.
- * Otherwise, creates an AND that is a child of 'targetParent' at position 'targetPosition', and
- * adds 'target' and 'node' as its children. Tags 'node' with 'tagData'. If 'node' appears as a key
- * in 'pathsToUpdate', then we set the new path onto the clone.
- */
+// Attaches 'node' to 'target'. If 'target' is an AND, adds 'node' as a child of 'target'.
+// Otherwise, creates an AND that is a child of 'targetParent' at position 'targetPosition', and
+// adds 'target' and 'node' as its children. Tags 'node' with 'tagData'.
void attachNode(MatchExpression* node,
MatchExpression* target,
OrMatchExpression* targetParent,
size_t targetPosition,
- std::unique_ptr<MatchExpression::TagData> tagData,
- const stdx::unordered_map<MatchExpression*, FieldRef>& pathsToUpdate) {
+ std::unique_ptr<MatchExpression::TagData> tagData) {
auto clone = node->shallowClone();
if (clone->matchType() == MatchExpression::NOT) {
IndexTag* indexTag = static_cast<IndexTag*>(tagData.get());
clone->setTag(new IndexTag(indexTag->index));
clone->getChild(0)->setTag(tagData.release());
-
- if (auto it = pathsToUpdate.find(node->getChild(0)); it != pathsToUpdate.end()) {
- checked_cast<PathMatchExpression*>(clone->getChild(0))
- ->setPath(it->second.dottedField());
- }
} else {
clone->setTag(tagData.release());
- if (auto it = pathsToUpdate.find(node); it != pathsToUpdate.end()) {
- checked_cast<PathMatchExpression*>(clone.get())->setPath(it->second.dottedField());
- }
}
if (MatchExpression::AND == target->matchType()) {
@@ -176,24 +164,17 @@ stdx::unordered_map<size_t, std::vector<OrPushdownTag::Destination>> partitionCh
return childDestinations;
}
-/**
- * Finds the node within 'tree' that is an indexed OR, if one exists. It also returns the subpath in
- * which the indexed OR lives.
- */
-std::pair<MatchExpression*, FieldRef> getIndexedOr(FieldRef currentPath, MatchExpression* tree) {
+// Finds the node within 'tree' that is an indexed OR, if one exists.
+MatchExpression* getIndexedOr(MatchExpression* tree) {
if (MatchExpression::OR == tree->matchType() && tree->getTag()) {
- return {tree, std::move(currentPath)};
- }
- if (const auto* fieldRef = tree->fieldRef()) {
- currentPath = currentPath + *fieldRef;
+ return tree;
}
-
for (size_t i = 0; i < tree->numChildren(); ++i) {
- if (auto result = getIndexedOr(currentPath, tree->getChild(i)); result.first) {
- return result;
+ if (auto indexedOrChild = getIndexedOr(tree->getChild(i))) {
+ return indexedOrChild;
}
}
- return {};
+ return nullptr;
}
// Pushes down 'node' along the routes in 'target' specified in 'destinations'. Each value in the
@@ -201,8 +182,7 @@ std::pair<MatchExpression*, FieldRef> getIndexedOr(FieldRef currentPath, MatchEx
// descendant of 'target'.
bool pushdownNode(MatchExpression* node,
MatchExpression* target,
- std::vector<OrPushdownTag::Destination> destinations,
- const stdx::unordered_map<MatchExpression*, FieldRef>& pathsToUpdate) {
+ std::vector<OrPushdownTag::Destination> destinations) {
if (MatchExpression::OR == target->matchType()) {
OrMatchExpression* orNode = static_cast<OrMatchExpression*>(target);
bool moveToAllChildren = true;
@@ -226,15 +206,13 @@ bool pushdownNode(MatchExpression* node,
orNode->getChild(i),
orNode,
i,
- std::move(childDestinations->second[0].tagData),
- pathsToUpdate);
+ std::move(childDestinations->second[0].tagData));
} else {
// This child was specified by a non-trivial route in destinations, so we recur.
moveToAllChildren = pushdownNode(node,
orNode->getChild(i),
- std::move(childDestinations->second),
- pathsToUpdate) &&
+ std::move(childDestinations->second)) &&
moveToAllChildren;
}
}
@@ -243,81 +221,36 @@ bool pushdownNode(MatchExpression* node,
}
if (MatchExpression::AND == target->matchType()) {
- auto [indexedOr, fieldRef_unused] = getIndexedOr({} /*fieldRef*/, target);
+ auto indexedOr = getIndexedOr(target);
invariant(indexedOr);
- return pushdownNode(node, indexedOr, std::move(destinations), pathsToUpdate);
+ return pushdownNode(node, indexedOr, std::move(destinations));
}
MONGO_UNREACHABLE_TASSERT(4457014);
}
-/**
- * Populates 'out' with all descendants of 'node' that have OrPushdownTags, assuming the initial
- * input is an ELEM_MATCH_OBJECT. The "currentPath" argument is the combined path traversed so far.
- * Additionally, we populate a map to keep track of paths to update afterward during cloning.
- */
-void getElemMatchOrPushdownDescendants(
- const FieldRef& indexedOrPath,
- FieldRef currentPath,
- MatchExpression* node,
- std::vector<MatchExpression*>* out,
- stdx::unordered_map<MatchExpression*, FieldRef>* pathsToUpdate) {
- const bool updatePath = node->fieldRef() != nullptr;
- if (updatePath) {
- currentPath = currentPath + *node->fieldRef();
- }
-
- // Do not do extra pushdown of OR inside $elemmatch.
+// Populates 'out' with all descendants of 'node' that have OrPushdownTags, assuming the initial
+// input is an ELEM_MATCH_OBJECT.
+void getElemMatchOrPushdownDescendants(MatchExpression* node, std::vector<MatchExpression*>* out) {
if (node->getTag() && node->getTag()->getType() == TagType::OrPushdownTag) {
- if (updatePath) {
- // Make sure that we remove the common prefix between the "destination" OR and the
- // current expression, as it may be contained within the same $elemmatch.
-
- const auto prefixSize = indexedOrPath.commonPrefixSize(currentPath);
- for (auto i = 0; i < prefixSize; i++) {
- currentPath.removeFirstPart();
- }
- if (currentPath != *node->fieldRef()) {
- pathsToUpdate->emplace(node, std::move(currentPath));
- }
- }
out->push_back(node);
} else if (node->matchType() == MatchExpression::ELEM_MATCH_OBJECT ||
node->matchType() == MatchExpression::AND) {
for (size_t i = 0; i < node->numChildren(); ++i) {
- getElemMatchOrPushdownDescendants(
- indexedOrPath, currentPath, node->getChild(i), out, pathsToUpdate);
+ getElemMatchOrPushdownDescendants(node->getChild(i), out);
}
} else if (node->matchType() == MatchExpression::NOT) {
// The immediate child of NOT may be tagged, but there should be no tags deeper than this.
auto* childNode = node->getChild(0);
if (childNode->getTag() && childNode->getTag()->getType() == TagType::OrPushdownTag) {
- if (!childNode->path().empty()) {
- // Make sure that we remove the common prefix between the "destination" OR and the
- // current expression, as it may be contained within the same $elemmatch.
-
- currentPath = currentPath + *childNode->fieldRef();
- const auto prefixSize = indexedOrPath.commonPrefixSize(currentPath);
- for (auto i = 0; i < prefixSize; i++) {
- currentPath.removeFirstPart();
- }
- if (currentPath != *childNode->fieldRef()) {
- pathsToUpdate->emplace(childNode, std::move(currentPath));
- }
- }
out->push_back(node);
}
}
}
-/**
- * Attempts to push the given node down into the 'indexedOr' subtree. Returns true if the predicate
- * can subsequently be trimmed from the MatchExpression tree, false otherwise. Also supplied is a
- * map to optionally update the path of the 'node' being pushed down.
- */
-bool processOrPushdownNode(MatchExpression* node,
- MatchExpression* indexedOr,
- const stdx::unordered_map<MatchExpression*, FieldRef>& pathsToUpdate) {
+// Attempts to push the given node down into the 'indexedOr' subtree. Returns true if the predicate
+// can subsequently be trimmed from the MatchExpression tree, false otherwise.
+bool processOrPushdownNode(MatchExpression* node, MatchExpression* indexedOr) {
// If the node is a negation, then its child is the predicate node that may be tagged.
auto* predNode = node->matchType() == MatchExpression::NOT ? node->getChild(0) : node;
@@ -334,7 +267,7 @@ bool processOrPushdownNode(MatchExpression* node,
predNode->setTag(nullptr);
// Attempt to push the node into the indexedOr, then re-set its tag to the indexTag.
- const bool pushedDown = pushdownNode(node, indexedOr, std::move(destinations), pathsToUpdate);
+ const bool pushedDown = pushdownNode(node, indexedOr, std::move(destinations));
predNode->setTag(indexTag.release());
// Return true if we can trim the predicate. We could trim the node even if it had an index tag
@@ -351,26 +284,20 @@ void resolveOrPushdowns(MatchExpression* tree) {
}
if (MatchExpression::AND == tree->matchType()) {
AndMatchExpression* andNode = static_cast<AndMatchExpression*>(tree);
- auto [indexedOr, indexedOrPath] = getIndexedOr({} /*fieldRef*/, andNode);
+ MatchExpression* indexedOr = getIndexedOr(andNode);
for (size_t i = 0; i < andNode->numChildren(); ++i) {
auto child = andNode->getChild(i);
- // For ELEM_MATCH_OBJECT, we push down all tagged descendants. However, we cannot
- // trim any of these predicates, since the $elemMatch filter must be applied in its
- // entirety.
+ // For ELEM_MATCH_OBJECT, we push down all tagged descendants. However, we cannot trim
+ // any of these predicates, since the $elemMatch filter must be applied in its entirety.
if (child->matchType() == MatchExpression::ELEM_MATCH_OBJECT) {
std::vector<MatchExpression*> orPushdownDescendants;
- stdx::unordered_map<MatchExpression*, FieldRef> pathsToUpdate;
- getElemMatchOrPushdownDescendants(indexedOrPath,
- {} /*currentPath*/,
- child,
- &orPushdownDescendants,
- &pathsToUpdate);
+ getElemMatchOrPushdownDescendants(child, &orPushdownDescendants);
for (auto descendant : orPushdownDescendants) {
- static_cast<void>(processOrPushdownNode(descendant, indexedOr, pathsToUpdate));
+ static_cast<void>(processOrPushdownNode(descendant, indexedOr));
}
- } else if (processOrPushdownNode(child, indexedOr, {} /*pathsToUpdate*/)) {
+ } else if (processOrPushdownNode(child, indexedOr)) {
// The indexed $or can completely satisfy the child predicate, so we trim it.
auto ownedChild = andNode->removeChild(i);
--i;
diff --git a/src/mongo/db/query/indexability.h b/src/mongo/db/query/indexability.h
index 1736d92a89b..7e543da4a2d 100644
--- a/src/mongo/db/query/indexability.h
+++ b/src/mongo/db/query/indexability.h
@@ -55,26 +55,6 @@ public:
}
/**
- * Type bracketing does not apply to internal Expressions. This could cause the use of a sparse
- * index return incomplete results. For example, a query {$expr: {$lt: ["$missing", "r"]}} would
- * expect a document like, {a: 1}, with field "missing" missing be returned. However, a sparse
- * index, {missing: 1} does not index the document. Therefore, we should ban use of any sparse
- * index on following expression types.
- */
- static bool nodeSupportedBySparseIndex(const MatchExpression* me) {
- switch (me->matchType()) {
- case MatchExpression::INTERNAL_EXPR_EQ:
- case MatchExpression::INTERNAL_EXPR_GT:
- case MatchExpression::INTERNAL_EXPR_GTE:
- case MatchExpression::INTERNAL_EXPR_LT:
- case MatchExpression::INTERNAL_EXPR_LTE:
- return false;
- default:
- return true;
- }
- }
-
- /**
* This array operator doesn't have any children with fields and can use an index.
*
* Example: a: {$elemMatch: {$gte: 1, $lte: 1}}.
@@ -136,20 +116,6 @@ public:
}
/**
- * Returns true if 'me' is ELEM_MATCH_OBJECT and has non-empty path component.
- *
- * Note: we skip empty path components since they are not allowed in index key patterns.
- * Therefore, $elemMatch with an empty path component can never use an index.
- *
- * Example: {"": {$elemMatch: {a: "hi", b: "bye"}}.
- * In this case the predicate cannot use any indexes since the $elemMatch is with an empty path
- * component.
- */
- static bool isBoundsGeneratingElemMatchObject(const MatchExpression* me) {
- return arrayUsesIndexOnChildren(me) && !me->path().empty();
- }
-
- /**
* Returns true if 'me' is a NOT, and the child of the NOT can use
* an index on its own field.
*/
diff --git a/src/mongo/db/query/internal_plans.cpp b/src/mongo/db/query/internal_plans.cpp
index 78820eb8111..04f70b1d2cc 100644
--- a/src/mongo/db/query/internal_plans.cpp
+++ b/src/mongo/db/query/internal_plans.cpp
@@ -123,8 +123,7 @@ CollectionScanParams createCollectionScanParams(
boost::optional<RecordId> resumeAfterRecordId,
boost::optional<RecordIdBound> minRecord,
boost::optional<RecordIdBound> maxRecord,
- CollectionScanParams::ScanBoundInclusion boundInclusion,
- bool shouldReturnEofOnFilterMismatch) {
+ CollectionScanParams::ScanBoundInclusion boundInclusion) {
const auto& collection = *coll;
invariant(collection);
@@ -140,7 +139,6 @@ CollectionScanParams createCollectionScanParams(
params.direction = CollectionScanParams::BACKWARD;
}
params.boundInclusion = boundInclusion;
- params.shouldReturnEofOnFilterMismatch = shouldReturnEofOnFilterMismatch;
return params;
}
} // namespace
@@ -153,8 +151,7 @@ std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> InternalPlanner::collection
boost::optional<RecordId> resumeAfterRecordId,
boost::optional<RecordIdBound> minRecord,
boost::optional<RecordIdBound> maxRecord,
- CollectionScanParams::ScanBoundInclusion boundInclusion,
- bool shouldReturnEofOnFilterMismatch) {
+ CollectionScanParams::ScanBoundInclusion boundInclusion) {
const auto& collection = *coll;
invariant(collection);
@@ -170,8 +167,7 @@ std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> InternalPlanner::collection
resumeAfterRecordId,
minRecord,
maxRecord,
- boundInclusion,
- shouldReturnEofOnFilterMismatch);
+ boundInclusion);
auto cs = _collectionScan(expCtx, ws.get(), &collection, collScanParams);
@@ -222,19 +218,11 @@ std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> InternalPlanner::deleteWith
boost::optional<RecordIdBound> minRecord,
boost::optional<RecordIdBound> maxRecord,
CollectionScanParams::ScanBoundInclusion boundInclusion,
- std::unique_ptr<BatchedDeleteStageBatchParams> batchedDeleteParams,
- const MatchExpression* filter,
- bool shouldReturnEofOnFilterMismatch) {
+ boost::optional<std::unique_ptr<BatchedDeleteStageBatchParams>> batchParams) {
const auto& collection = *coll;
invariant(collection);
- if (shouldReturnEofOnFilterMismatch) {
- tassert(7010801,
- "MatchExpression filter must be provided when 'shouldReturnEofOnFilterMismatch' is "
- "set to true ",
- filter);
- }
-
auto ws = std::make_unique<WorkingSet>();
+
auto expCtx = make_intrusive<ExpressionContext>(
opCtx, std::unique_ptr<CollatorInterface>(nullptr), collection->ns());
@@ -249,15 +237,14 @@ std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> InternalPlanner::deleteWith
boost::none /* resumeAfterId */,
minRecord,
maxRecord,
- boundInclusion,
- shouldReturnEofOnFilterMismatch);
+ boundInclusion);
- auto root = _collectionScan(expCtx, ws.get(), &collection, collScanParams, filter);
+ auto root = _collectionScan(expCtx, ws.get(), &collection, collScanParams);
- if (batchedDeleteParams) {
+ if (batchParams) {
root = std::make_unique<BatchedDeleteStage>(expCtx.get(),
std::move(params),
- std::move(batchedDeleteParams),
+ std::move(*batchParams),
ws.get(),
collection,
root.release());
@@ -467,13 +454,12 @@ std::unique_ptr<PlanStage> InternalPlanner::_collectionScan(
const boost::intrusive_ptr<ExpressionContext>& expCtx,
WorkingSet* ws,
const CollectionPtr* coll,
- const CollectionScanParams& params,
- const MatchExpression* filter) {
+ const CollectionScanParams& params) {
const auto& collection = *coll;
invariant(collection);
- return std::make_unique<CollectionScan>(expCtx.get(), collection, params, ws, filter);
+ return std::make_unique<CollectionScan>(expCtx.get(), collection, params, ws, nullptr);
}
std::unique_ptr<PlanStage> InternalPlanner::_indexScan(
diff --git a/src/mongo/db/query/internal_plans.h b/src/mongo/db/query/internal_plans.h
index 46400c4e3fa..ea8de7c0042 100644
--- a/src/mongo/db/query/internal_plans.h
+++ b/src/mongo/db/query/internal_plans.h
@@ -83,8 +83,7 @@ public:
boost::optional<RecordIdBound> minRecord = boost::none,
boost::optional<RecordIdBound> maxRecord = boost::none,
CollectionScanParams::ScanBoundInclusion boundInclusion =
- CollectionScanParams::ScanBoundInclusion::kIncludeBothStartAndEndRecords,
- bool shouldReturnEofOnFilterMismatch = false);
+ CollectionScanParams::ScanBoundInclusion::kIncludeBothStartAndEndRecords);
static std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> collectionScan(
OperationContext* opCtx,
@@ -105,9 +104,7 @@ public:
boost::optional<RecordIdBound> maxRecord = boost::none,
CollectionScanParams::ScanBoundInclusion boundInclusion =
CollectionScanParams::ScanBoundInclusion::kIncludeBothStartAndEndRecords,
- std::unique_ptr<BatchedDeleteStageBatchParams> batchedDeleteParams = nullptr,
- const MatchExpression* filter = nullptr,
- bool shouldReturnEofOnFilterMismatch = false);
+ boost::optional<std::unique_ptr<BatchedDeleteStageBatchParams>> batchParams = boost::none);
/**
* Returns an index scan. Caller owns returned pointer.
@@ -200,8 +197,7 @@ private:
const boost::intrusive_ptr<ExpressionContext>& expCtx,
WorkingSet* ws,
const CollectionPtr* collection,
- const CollectionScanParams& params,
- const MatchExpression* filter = nullptr);
+ const CollectionScanParams& params);
/**
* Returns a plan stage that is either an index scan or an index scan with a fetch stage.
diff --git a/src/mongo/db/query/lru_key_value.h b/src/mongo/db/query/lru_key_value.h
index 2786e1c40ab..88186c70923 100644
--- a/src/mongo/db/query/lru_key_value.h
+++ b/src/mongo/db/query/lru_key_value.h
@@ -28,6 +28,7 @@
*/
#pragma once
+
#include <fmt/format.h>
#include <list>
#include <memory>
@@ -39,56 +40,30 @@
namespace mongo {
/**
- * 'InsertionEvictionListener' class to use with 'LRUBudgetTracker' that will always noop.
- */
-class NoopInsertionEvictionListener {
-public:
- // Called when a key-value pair is being inserted. Parameters are the key-value pair and its
- // estimated size.
- template <class K, class V>
- void onInsert(const K&, const V&, size_t) {}
-
- // Called when a key-value pair is being evicted. Parameters are the key-value pair and its
- // estimated size.
- template <class K, class V>
- void onEvict(const K&, const V&, size_t) {}
-
- // Called when the cache is being cleared. Parameter is the estimated size of the key-value
- // pairs in the cache before it was cleared.
- void onClear(size_t) {}
-};
-
-/**
* This class tracks a size of entries in 'LRUKeyValue'.
* The size can be understood as a number of the entries, an amount of memory they occupied,
* or any other value defined by the template parameter 'Estimator'.
* The 'Estimator' must be deterministic and always return the same value for the same entry.
- * The 'InsertionEvictionListener' will be called on every insertion and eviction as well as when
- * the cache is cleared.
*/
-template <class K, class V, typename Estimator, typename InsertionEvictionListener>
+template <typename V, typename Estimator>
class LRUBudgetTracker {
public:
LRUBudgetTracker(size_t maxBudget) : _max(maxBudget), _current(0) {}
- void onAdd(const K& k, const V& v) {
- size_t budget = _estimator(k, v);
- _current += budget;
- _listener.onInsert(k, v, budget);
+ void onAdd(const V& v) {
+ _current += _estimator(v);
}
- void onRemove(const K& k, const V& v) {
+ void onRemove(const V& v) {
using namespace fmt::literals;
- size_t budget = _estimator(k, v);
+ size_t budget = _estimator(v);
tassert(5968300,
"LRU budget underflow: current={}, budget={} "_format(_current, budget),
_current >= budget);
_current -= budget;
- _listener.onEvict(k, v, budget);
}
void onClear() {
- _listener.onClear(_current);
_current = 0;
}
@@ -109,7 +84,6 @@ private:
size_t _max;
size_t _current;
Estimator _estimator;
- InsertionEvictionListener _listener;
};
/**
@@ -117,9 +91,6 @@ private:
* policy. The size allowed in the kv-store is controlled by 'LRUBudgetTracker'
* set in the constructor.
*
- * An 'InsertionEvictionListener' may optionally be specified to track the insertion and eviction of
- * each key-value pair.
- *
* Caveat:
* This kv-store is NOT thread safe! The client to this utility is responsible
* for protecting concurrent access to the LRU store if used in a threaded
@@ -131,12 +102,7 @@ private:
* TODO: We could move this into the util/ directory and do any cleanup necessary to make it
* fully general.
*/
-template <class K,
- class V,
- class KeyValueBudgetEstimator,
- class InsertionEvictionListener = NoopInsertionEvictionListener,
- class KeyHasher = std::hash<K>,
- class Eq = std::equal_to<K>>
+template <class K, class V, class BudgetEstimator, class KeyHasher = std::hash<K>>
class LRUKeyValue {
public:
LRUKeyValue(size_t maxSize) : _budgetTracker{maxSize} {}
@@ -145,13 +111,13 @@ public:
clear();
}
- typedef std::pair<const K*, V> KVListEntry;
+ typedef std::pair<K, V> KVListEntry;
typedef std::list<KVListEntry> KVList;
typedef typename KVList::iterator KVListIt;
typedef typename KVList::const_iterator KVListConstIt;
- typedef stdx::unordered_map<K, KVListIt, KeyHasher, Eq> KVMap;
+ typedef stdx::unordered_map<K, KVListIt, KeyHasher> KVMap;
typedef typename KVMap::const_iterator KVMapConstIt;
// These type declarations are required by the 'Partitioned' utility.
@@ -170,15 +136,14 @@ public:
KVMapConstIt i = _kvMap.find(key);
if (i != _kvMap.end()) {
KVListIt found = i->second;
- _budgetTracker.onRemove(key, found->second);
+ _budgetTracker.onRemove(found->second);
_kvMap.erase(i);
_kvList.erase(found);
}
- _budgetTracker.onAdd(key, entry);
- _kvList.push_front(std::make_pair(nullptr, std::move(entry)));
+ _budgetTracker.onAdd(entry);
+ _kvList.push_front(std::make_pair(key, std::move(entry)));
_kvMap[key] = _kvList.begin();
- _kvList.begin()->first = &(_kvMap.find(key)->first);
return evict();
}
@@ -196,11 +161,10 @@ public:
KVListIt found = i->second;
// Promote the kv-store entry to the front of the list. It is now the most recently used.
- _kvList.push_front(std::make_pair(nullptr, std::move(found->second)));
+ _kvList.push_front(std::make_pair(key, std::move(found->second)));
_kvMap.erase(i);
_kvList.erase(found);
_kvMap[key] = _kvList.begin();
- _kvList.begin()->first = &(_kvMap.find(key)->first);
return _kvList.begin();
}
@@ -215,7 +179,7 @@ public:
return false;
}
KVListIt found = i->second;
- _budgetTracker.onRemove(key, found->second);
+ _budgetTracker.onRemove(found->second);
_kvMap.erase(i);
_kvList.erase(found);
return true;
@@ -229,9 +193,9 @@ public:
size_t removeIf(KeyValuePredicate predicate) {
size_t removed = 0;
for (auto it = _kvList.begin(); it != _kvList.end();) {
- if (predicate(*it->first, *it->second)) {
- _budgetTracker.onRemove(*it->first, it->second);
- _kvMap.erase(*it->first);
+ if (predicate(it->first, *it->second)) {
+ _budgetTracker.onRemove(it->second);
+ _kvMap.erase(it->first);
it = _kvList.erase(it);
++removed;
} else {
@@ -245,9 +209,9 @@ public:
* Deletes all entries in the kv-store.
*/
void clear() {
+ _budgetTracker.onClear();
_kvList.clear();
_kvMap.clear();
- _budgetTracker.onClear();
}
/**
@@ -294,8 +258,8 @@ private:
while (_budgetTracker.isOverBudget()) {
invariant(!_kvList.empty());
- _budgetTracker.onRemove(*_kvList.back().first, _kvList.back().second);
- _kvMap.erase(*_kvList.back().first);
+ _budgetTracker.onRemove(_kvList.back().second);
+ _kvMap.erase(_kvList.back().first);
_kvList.pop_back();
++nEvicted;
@@ -304,14 +268,13 @@ private:
return nEvicted;
}
- LRUBudgetTracker<K, V, KeyValueBudgetEstimator, InsertionEvictionListener> _budgetTracker;
+ LRUBudgetTracker<V, BudgetEstimator> _budgetTracker;
// (K, V) pairs are stored in this std::list. They are sorted in order of use, where the front
// is the most recently used and the back is the least recently used.
mutable KVList _kvList;
// Maps from a key to the corresponding std::list entry.
- // TODO: SERVER-73659 LRUKeyValue should track and include the size of _kvMap in overall budget.
mutable KVMap _kvMap;
};
diff --git a/src/mongo/db/query/lru_key_value_test.cpp b/src/mongo/db/query/lru_key_value_test.cpp
index 6dcdfc5a4ea..3ebf47267fb 100644
--- a/src/mongo/db/query/lru_key_value_test.cpp
+++ b/src/mongo/db/query/lru_key_value_test.cpp
@@ -64,15 +64,15 @@ struct ValueType {
struct TrivialBudgetEstimator {
static constexpr size_t kSize = 1;
- size_t operator()(const int&, const ValueType&) {
+ size_t operator()(const ValueType&) {
return kSize;
}
- size_t operator()(const int&, const std::unique_ptr<int>&) {
+ size_t operator()(const std::shared_ptr<int>&) {
return kSize;
}
- size_t operator()(const int&, const std::shared_ptr<int>) {
+ size_t operator()(const std::unique_ptr<int>&) {
return kSize;
}
};
@@ -87,7 +87,7 @@ struct NonTrivialEntry {
};
struct NonTrivialBudgetEstimator {
- size_t operator()(const int& key, const std::shared_ptr<NonTrivialEntry> value) {
+ size_t operator()(const std::shared_ptr<NonTrivialEntry>& value) {
return value->budgetSize;
}
};
@@ -95,40 +95,8 @@ struct NonTrivialBudgetEstimator {
using NonTrivialTestSharedPtrValue =
LRUKeyValue<size_t, std::shared_ptr<NonTrivialEntry>, NonTrivialBudgetEstimator>;
-class NonTrivialInsertionEvictionListener {
-public:
- NonTrivialInsertionEvictionListener() {
- keyTotal = 0;
- valueTotal = 0;
- budgetTotal = 0;
- }
-
- void onInsert(const int& k, const ValueType& v, size_t budget) {
- keyTotal += k;
- valueTotal += v.val;
- budgetTotal += budget;
- }
-
- void onEvict(const int& k, const ValueType& v, size_t budget) {
- keyTotal -= k;
- valueTotal -= v.val;
- budgetTotal -= budget;
- }
-
- void onClear(size_t budget) {
- budgetTotal -= budget;
- }
-
- static size_t keyTotal;
- static size_t valueTotal;
- static size_t budgetTotal;
-};
-size_t NonTrivialInsertionEvictionListener::keyTotal;
-size_t NonTrivialInsertionEvictionListener::valueTotal;
-size_t NonTrivialInsertionEvictionListener::budgetTotal;
-
-template <typename Key, typename Value, typename Estimator, typename Listener>
-void assertInKVStore(LRUKeyValue<Key, Value, Estimator, Listener>& cache, Key key, Value value) {
+template <typename Key, typename Value, typename Estimator>
+void assertInKVStore(LRUKeyValue<Key, Value, Estimator>& cache, Key key, Value value) {
ASSERT_TRUE(cache.hasKey(key));
auto s = cache.get(key);
ASSERT(s.isOK());
@@ -137,8 +105,8 @@ void assertInKVStore(LRUKeyValue<Key, Value, Estimator, Listener>& cache, Key ke
ASSERT_EQUALS(*(kvItr->second), *value);
}
-template <typename Key, typename Value, typename Estimator, typename Listener>
-void assertNotInKVStore(LRUKeyValue<Key, Value, Estimator, Listener>& cache, Key key) {
+template <typename Key, typename Value, typename Estimator>
+void assertNotInKVStore(LRUKeyValue<Key, Value, Estimator>& cache, Key key) {
ASSERT_FALSE(cache.hasKey(key));
auto s = cache.get(key);
ASSERT(!s.isOK());
@@ -327,10 +295,10 @@ TEST(LRUKeyValueTest, IterationTest) {
cache.add(2, std::make_shared<int>(2));
auto i = cache.begin();
- ASSERT_EQUALS(*i->first, 2);
+ ASSERT_EQUALS(i->first, 2);
ASSERT_EQUALS(*i->second, 2);
++i;
- ASSERT_EQUALS(*i->first, 1);
+ ASSERT_EQUALS(i->first, 1);
ASSERT_EQUALS(*i->second, 1);
++i;
ASSERT(i == cache.end());
@@ -388,8 +356,7 @@ TEST(LRUKeyValueTest, UniquePtrKeyValue) {
assertNotInKVStore(cacheForEviction, 1); // The entry with key '1' has been Evicted.
}
-using TestScalarValue =
- LRUKeyValue<int, ValueType, TrivialBudgetEstimator, NonTrivialInsertionEvictionListener>;
+using TestScalarValue = LRUKeyValue<int, ValueType, TrivialBudgetEstimator>;
void assertValueInKVStore(TestScalarValue& cache, int key, ValueType value) {
ASSERT_TRUE(cache.hasKey(key));
@@ -406,17 +373,9 @@ TEST(LRUKeyValueTest, ScalarKeyValue) {
assertValueInKVStore(cache, 1, ValueType{2});
assertNotInKVStore(cache, 3);
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::keyTotal, 1);
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::valueTotal, 2);
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 1);
-
cache.add(1, ValueType{3});
assertValueInKVStore(cache, 1, ValueType{3});
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::keyTotal, 1);
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::valueTotal, 3);
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 1);
-
// Test eviction.
TestScalarValue cacheForEviction{2};
cacheForEviction.add(1, ValueType{1});
@@ -425,18 +384,6 @@ TEST(LRUKeyValueTest, ScalarKeyValue) {
ASSERT_EQUALS(cacheForEviction.size(), static_cast<size_t>(2));
assertNotInKVStore(cacheForEviction, 1); // The entry with key '1' has been Evicted.
-
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::keyTotal, 5);
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::valueTotal, 5);
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 2);
-
- // Clear the remaining values.
- cacheForEviction.clear();
-
- assertNotInKVStore(cacheForEviction, 2); // The entry with key '2' has been Evicted.
- assertNotInKVStore(cacheForEviction, 3); // The entry with key '3' has been Evicted.
-
- ASSERT_EQUALS(NonTrivialInsertionEvictionListener::budgetTotal, 0);
}
} // namespace
diff --git a/src/mongo/db/query/mock_yield_policies.h b/src/mongo/db/query/mock_yield_policies.h
index 5774c24fd6e..fa2cee9fbba 100644
--- a/src/mongo/db/query/mock_yield_policies.h
+++ b/src/mongo/db/query/mock_yield_policies.h
@@ -39,10 +39,8 @@ namespace mongo {
*/
class MockYieldPolicy : public PlanYieldPolicy {
public:
- MockYieldPolicy(OperationContext* opCtx,
- ClockSource* clockSource,
- PlanYieldPolicy::YieldPolicy policy)
- : PlanYieldPolicy(opCtx, policy, clockSource, 0, Milliseconds{0}, nullptr, nullptr) {}
+ MockYieldPolicy(ClockSource* clockSource, PlanYieldPolicy::YieldPolicy policy)
+ : PlanYieldPolicy(policy, clockSource, 0, Milliseconds{0}, nullptr, nullptr) {}
private:
void saveState(OperationContext* opCtx) override final {
@@ -60,8 +58,8 @@ private:
*/
class AlwaysTimeOutYieldPolicy final : public MockYieldPolicy {
public:
- AlwaysTimeOutYieldPolicy(OperationContext* opCtx, ClockSource* cs)
- : MockYieldPolicy(opCtx, cs, PlanYieldPolicy::YieldPolicy::ALWAYS_TIME_OUT) {}
+ AlwaysTimeOutYieldPolicy(ClockSource* cs)
+ : MockYieldPolicy(cs, PlanYieldPolicy::YieldPolicy::ALWAYS_TIME_OUT) {}
bool shouldYieldOrInterrupt(OperationContext*) override {
return true;
@@ -78,8 +76,8 @@ public:
*/
class AlwaysPlanKilledYieldPolicy final : public MockYieldPolicy {
public:
- AlwaysPlanKilledYieldPolicy(OperationContext* opCtx, ClockSource* cs)
- : MockYieldPolicy(opCtx, cs, PlanYieldPolicy::YieldPolicy::ALWAYS_MARK_KILLED) {}
+ AlwaysPlanKilledYieldPolicy(ClockSource* cs)
+ : MockYieldPolicy(cs, PlanYieldPolicy::YieldPolicy::ALWAYS_MARK_KILLED) {}
bool shouldYieldOrInterrupt(OperationContext*) override {
return true;
@@ -96,8 +94,8 @@ public:
*/
class NoopYieldPolicy final : public MockYieldPolicy {
public:
- NoopYieldPolicy(OperationContext* opCtx, ClockSource* clockSource)
- : MockYieldPolicy(opCtx, clockSource, PlanYieldPolicy::YieldPolicy::NO_YIELD) {}
+ NoopYieldPolicy(ClockSource* clockSource)
+ : MockYieldPolicy(clockSource, PlanYieldPolicy::YieldPolicy::NO_YIELD) {}
bool shouldYieldOrInterrupt(OperationContext*) override {
return false;
diff --git a/src/mongo/db/query/multiple_collection_accessor.h b/src/mongo/db/query/multiple_collection_accessor.h
index 9d302ce2b28..26fc081000e 100644
--- a/src/mongo/db/query/multiple_collection_accessor.h
+++ b/src/mongo/db/query/multiple_collection_accessor.h
@@ -101,17 +101,6 @@ public:
_secondaryColls.clear();
}
- void forEach(std::function<void(const CollectionPtr&)> func) const {
- if (hasMainCollection()) {
- func(getMainCollection());
- }
- for (const auto& [name, coll] : getSecondaryCollections()) {
- if (coll) {
- func(coll);
- }
- }
- }
-
private:
const CollectionPtr* _mainColl{&CollectionPtr::null};
diff --git a/src/mongo/db/query/parsed_find_command.cpp b/src/mongo/db/query/parsed_find_command.cpp
deleted file mode 100644
index 2ef2e955c06..00000000000
--- a/src/mongo/db/query/parsed_find_command.cpp
+++ /dev/null
@@ -1,381 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/parsed_find_command.h"
-
-#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
-
-#include "mongo/db/cst/cst_parser.h"
-#include "mongo/db/query/collation/collator_factory_interface.h"
-#include "mongo/db/query/projection_ast_util.h"
-#include "mongo/db/query/projection_parser.h"
-#include "mongo/db/query/query_planner_common.h"
-#include "mongo/db/query/query_request_helper.h"
-#include "mongo/logv2/log.h"
-
-namespace mongo {
-
-namespace {
-/**
- * Does 'root' have a subtree of type 'subtreeType' with a node of type 'childType' inside?
- */
-bool hasNodeInSubtree(const MatchExpression* root,
- MatchExpression::MatchType childType,
- MatchExpression::MatchType subtreeType) {
- if (subtreeType == root->matchType()) {
- return QueryPlannerCommon::hasNode(root, childType);
- }
- for (size_t i = 0; i < root->numChildren(); ++i) {
- if (hasNodeInSubtree(root->getChild(i), childType, subtreeType)) {
- return true;
- }
- }
- return false;
-}
-
-bool parsingCanProduceNoopMatchNodes(const ExtensionsCallback& extensionsCallback,
- MatchExpressionParser::AllowedFeatureSet allowedFeatures) {
- return extensionsCallback.hasNoopExtensions() &&
- (allowedFeatures & MatchExpressionParser::AllowedFeatures::kText ||
- allowedFeatures & MatchExpressionParser::AllowedFeatures::kJavascript);
-}
-
-} // namespace
-
-std::unique_ptr<CollatorInterface> resolveCollator(
- OperationContext* opCtx, const std::unique_ptr<FindCommandRequest>& findCommand) {
- if (!findCommand->getCollation().isEmpty()) {
- return uassertStatusOKWithContext(CollatorFactoryInterface::get(opCtx->getServiceContext())
- ->makeFromBSON(findCommand->getCollation()),
- "unable to parse collation");
- }
- return nullptr;
-}
-
-/**
- * Helper for building 'out.' If there is a projection, parse it and add any metadata dependencies
- * it induces.
- *
- * Throws exceptions if there is an error parsing the projection.
- */
-void setProjection(ParsedFindCommand* out,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const std::unique_ptr<FindCommandRequest>& findCommand,
- const ProjectionPolicies& policies) {
- if (!findCommand->getProjection().isEmpty()) {
- out->savedProjectionPolicies.emplace(policies);
- out->proj.emplace(projection_ast::parseAndAnalyze(expCtx,
- findCommand->getProjection(),
- out->filter.get(),
- findCommand->getFilter(),
- policies));
-
- // This will throw if any of the projection's dependencies are unavailable.
- DepsTracker{out->unavailableMetadata}.requestMetadata(out->proj->metadataDeps());
- }
-}
-
-/**
- * Helper for building 'out.' If there is a sort, parse it and add any metadata dependencies it
- * induces.
- *
- * Throws exceptions if there is an error parsing the sort pattern.
- */
-void setSort(ParsedFindCommand* out,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const std::unique_ptr<FindCommandRequest>& findCommand) {
- if (!findCommand->getSort().isEmpty()) {
- // A $natural sort is really a hint, and should be handled as such. Furthermore, the
- // downstream sort handling code may not expect a $natural sort.
- //
- // We have already validated that if there is a $natural sort and a hint, that the hint
- // also specifies $natural with the same direction. Therefore, it is safe to clear the
- // $natural sort and rewrite it as a $natural hint.
- if (findCommand->getSort()[query_request_helper::kNaturalSortField]) {
- findCommand->setHint(findCommand->getSort().getOwned());
- findCommand->setSort(BSONObj{});
- }
- if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) {
- out->sort = cst::parseToSortPattern(findCommand->getSort(), expCtx);
- } else {
- out->sort.emplace(findCommand->getSort(), expCtx);
- }
- }
-}
-
-/**
- * Helper for building 'out.' If there is a sort, parse it and add any metadata dependencies it
- * induces.
- */
-Status setSortAndProjection(ParsedFindCommand* out,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const std::unique_ptr<FindCommandRequest>& findCommand,
- const ProjectionPolicies& policies) {
- try {
- setProjection(out, expCtx, findCommand, policies);
- setSort(out, expCtx, findCommand);
- } catch (const DBException& ex) {
- return ex.toStatus();
- }
-
- return Status::OK();
-}
-
-/**
- * Helper for building 'out.' Sets 'out->filter' and validates that it is well formed. In the
- * process, also populates 'out->unavailableMetadata.'
- */
-Status setFilter(ParsedFindCommand* out,
- std::unique_ptr<MatchExpression> filter,
- const std::unique_ptr<FindCommandRequest>& findCommand) {
- // Verify the filter follows certain rules like there must be at most one text clause.
- auto swMeta = parsed_find_command::isValid(filter.get(), *findCommand);
- if (!swMeta.isOK()) {
- return swMeta.getStatus();
- }
- out->unavailableMetadata = swMeta.getValue();
- out->filter = std::move(filter);
- return Status::OK();
-}
-
-
-StatusWith<std::unique_ptr<ParsedFindCommand>> parseWithValidatedCollator(
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- std::unique_ptr<FindCommandRequest> findCommand,
- const ExtensionsCallback& extensionsCallback,
- MatchExpressionParser::AllowedFeatureSet allowedFeatures,
- const ProjectionPolicies& projectionPolicies) {
- auto out = std::make_unique<ParsedFindCommand>();
-
- tassert(5746107,
- "ntoreturn should not be set on the findCommand",
- findCommand->getNtoreturn() == boost::none);
-
- if (auto status = query_request_helper::validateFindCommandRequest(*findCommand);
- !status.isOK()) {
- return status;
- }
-
- // Parse the MatchExpression.
- StatusWithMatchExpression statusWithMatcher = [&]() -> StatusWithMatchExpression {
- if (getTestCommandsEnabled() && internalQueryEnableCSTParser.load()) {
- try {
- return cst::parseToMatchExpression(
- findCommand->getFilter(), expCtx, extensionsCallback);
- } catch (const DBException& ex) {
- return ex.toStatus();
- }
- } else {
- return MatchExpressionParser::parse(
- findCommand->getFilter(), expCtx, extensionsCallback, allowedFeatures);
- }
- }();
- if (!statusWithMatcher.isOK()) {
- return statusWithMatcher.getStatus();
- }
-
- // Stop counting expressions after they have been parsed to exclude expressions created
- // during optimization and other processing steps.
- expCtx->stopExpressionCounters();
- out->canHaveNoopMatchNodes =
- parsingCanProduceNoopMatchNodes(extensionsCallback, allowedFeatures);
-
- if (auto status = setFilter(out.get(), std::move(statusWithMatcher.getValue()), findCommand);
- !status.isOK()) {
- return status;
- }
-
- if (auto status = setSortAndProjection(out.get(), expCtx, findCommand, projectionPolicies);
- !status.isOK()) {
- return status;
- }
-
- out->findCommandRequest = std::move(findCommand);
- return {std::move(out)};
-}
-
-StatusWith<std::unique_ptr<ParsedFindCommand>> ParsedFindCommand::withExistingFilter(
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- std::unique_ptr<CollatorInterface> collator,
- std::unique_ptr<MatchExpression> filter,
- std::unique_ptr<FindCommandRequest> findCommandRequest) {
- auto out = std::make_unique<ParsedFindCommand>();
- out->collator = std::move(collator);
- if (auto status = setFilter(out.get(), std::move(filter), findCommandRequest); !status.isOK()) {
- return status;
- }
- if (auto status = setSortAndProjection(
- out.get(), expCtx, findCommandRequest, ProjectionPolicies::findProjectionPolicies());
- !status.isOK()) {
- return status;
- }
- out->findCommandRequest = std::move(findCommandRequest);
- return std::move(out);
-}
-
-namespace parsed_find_command {
-StatusWith<QueryMetadataBitSet> isValid(const MatchExpression* root,
- const FindCommandRequest& findCommand) {
- QueryMetadataBitSet unavailableMetadata{};
-
- // There can only be one TEXT. If there is a TEXT, it cannot appear inside a NOR.
- //
- // Note that the query grammar (as enforced by the MatchExpression parser) forbids TEXT
- // inside of value-expression clauses like NOT, so we don't check those here.
- size_t numText = QueryPlannerCommon::countNodes(root, MatchExpression::TEXT);
- if (numText > 1) {
- return Status(ErrorCodes::BadValue, "Too many text expressions");
- } else if (1 == numText) {
- if (hasNodeInSubtree(root, MatchExpression::TEXT, MatchExpression::NOR)) {
- return Status(ErrorCodes::BadValue, "text expression not allowed in nor");
- }
- } else {
- // Text metadata is not available.
- unavailableMetadata.set(DocumentMetadataFields::kTextScore);
- }
-
- // There can only be one NEAR. If there is a NEAR, it must be either the root or the root
- // must be an AND and its child must be a NEAR.
- size_t numGeoNear = QueryPlannerCommon::countNodes(root, MatchExpression::GEO_NEAR);
- if (numGeoNear > 1) {
- return Status(ErrorCodes::BadValue, "Too many geoNear expressions");
- } else if (1 == numGeoNear) {
- // Do nothing, we will perform extra checks in CanonicalQuery::isValidNormalized.
- } else {
- // Geo distance and geo point metadata are unavailable.
- unavailableMetadata |= DepsTracker::kAllGeoNearData;
- }
-
- const BSONObj& sortObj = findCommand.getSort();
- BSONElement sortNaturalElt = sortObj["$natural"];
- const BSONObj& hintObj = findCommand.getHint();
- BSONElement hintNaturalElt = hintObj["$natural"];
-
- if (sortNaturalElt && sortObj.nFields() != 1) {
- return Status(ErrorCodes::BadValue,
- str::stream() << "Cannot include '$natural' in compound sort: " << sortObj);
- }
-
- if (hintNaturalElt && hintObj.nFields() != 1) {
- return Status(ErrorCodes::BadValue,
- str::stream() << "Cannot include '$natural' in compound hint: " << hintObj);
- }
-
- // NEAR cannot have a $natural sort or $natural hint.
- if (numGeoNear > 0) {
- if (sortNaturalElt) {
- return Status(ErrorCodes::BadValue,
- "geoNear expression not allowed with $natural sort order");
- }
-
- if (hintNaturalElt) {
- return Status(ErrorCodes::BadValue,
- "geoNear expression not allowed with $natural hint");
- }
- }
-
- // TEXT and NEAR cannot both be in the query.
- if (numText > 0 && numGeoNear > 0) {
- return Status(ErrorCodes::BadValue, "text and geoNear not allowed in same query");
- }
-
- // TEXT and {$natural: ...} sort order cannot both be in the query.
- if (numText > 0 && sortNaturalElt) {
- return Status(ErrorCodes::BadValue, "text expression not allowed with $natural sort order");
- }
-
- // TEXT and hint cannot both be in the query.
- if (numText > 0 && !hintObj.isEmpty()) {
- return Status(ErrorCodes::BadValue, "text and hint not allowed in same query");
- }
-
- // TEXT and tailable are incompatible.
- if (numText > 0 && findCommand.getTailable()) {
- return Status(ErrorCodes::BadValue, "text and tailable cursor not allowed in same query");
- }
-
- // NEAR and tailable are incompatible.
- if (numGeoNear > 0 && findCommand.getTailable()) {
- return Status(ErrorCodes::BadValue,
- "Tailable cursors and geo $near cannot be used together");
- }
-
- // $natural sort order must agree with hint.
- if (sortNaturalElt) {
- if (!hintObj.isEmpty() && !hintNaturalElt) {
- return Status(ErrorCodes::BadValue, "index hint not allowed with $natural sort order");
- }
- if (hintNaturalElt) {
- if (hintNaturalElt.numberInt() != sortNaturalElt.numberInt()) {
- return Status(ErrorCodes::BadValue,
- "$natural hint must be in the same direction as $natural sort order");
- }
- }
- }
-
- return unavailableMetadata;
-}
-
-StatusWith<std::pair<boost::intrusive_ptr<ExpressionContext>, std::unique_ptr<ParsedFindCommand>>>
-parse(OperationContext* opCtx,
- std::unique_ptr<FindCommandRequest> findCommand,
- const ExtensionsCallback& extensionsCallback,
- MatchExpressionParser::AllowedFeatureSet allowedFeatures,
- const ProjectionPolicies& projectionPolicies) {
- // Make the expCtx.
- invariant(findCommand->getNamespaceOrUUID().nss().has_value());
- auto expCtx = make_intrusive<ExpressionContext>(
- opCtx, *findCommand, resolveCollator(opCtx, findCommand), true /* mayDbProfile */);
- auto swResult = parseWithValidatedCollator(
- expCtx, std::move(findCommand), extensionsCallback, allowedFeatures, projectionPolicies);
- if (!swResult.isOK()) {
- return swResult.getStatus();
- }
-
- return std::pair{std::move(expCtx), std::move(swResult.getValue())};
-}
-
-StatusWith<std::unique_ptr<ParsedFindCommand>> parse(
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- std::unique_ptr<FindCommandRequest> findCommand,
- const ExtensionsCallback& extensionsCallback,
- MatchExpressionParser::AllowedFeatureSet allowedFeatures,
- const ProjectionPolicies& projectionPolicies) {
- // A collator can enter through both the FindCommandRequest and ExpressionContext arguments.
- // This invariant ensures that both collators are the same because downstream we
- // pull the collator from only one of the ExpressionContext carrier.
- auto collator = resolveCollator(expCtx->opCtx, findCommand);
- if (collator.get() && expCtx->getCollator()) {
- invariant(CollatorInterface::collatorsMatch(collator.get(), expCtx->getCollator()));
- }
- return parseWithValidatedCollator(
- expCtx, std::move(findCommand), extensionsCallback, allowedFeatures, projectionPolicies);
-}
-} // namespace parsed_find_command
-} // namespace mongo
diff --git a/src/mongo/db/query/parsed_find_command.h b/src/mongo/db/query/parsed_find_command.h
deleted file mode 100644
index f3ec78b9204..00000000000
--- a/src/mongo/db/query/parsed_find_command.h
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/db/matcher/expression.h"
-#include "mongo/db/query/find_command_gen.h"
-#include "mongo/db/query/projection.h"
-#include "mongo/db/query/projection_policies.h"
-#include "mongo/db/query/sort_pattern.h"
-
-namespace mongo {
-
-/**
- * Represents a find command request, but with more fully parsed ASTs for some fields which are
- * still raw BSONObj on the FindCommandRequest type.
- */
-struct ParsedFindCommand {
- ParsedFindCommand() = default;
-
- /**
- * This API adds the ability to construct from a pre-parsed filter. The other arguments will be
- * re-parsed again from BSON on the 'findCommandRequest' argument, since we don't have a good
- * way of cloning them.
- */
- static StatusWith<std::unique_ptr<ParsedFindCommand>> withExistingFilter(
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- std::unique_ptr<CollatorInterface> collator,
- std::unique_ptr<MatchExpression> filter,
- std::unique_ptr<FindCommandRequest> findCommandRequest);
-
- std::unique_ptr<CollatorInterface> collator;
- std::unique_ptr<MatchExpression> filter;
- boost::optional<projection_ast::Projection> proj;
- boost::optional<SortPattern> sort;
-
- // Based on parsing the query, which metadata will *not* be available. For example, if there is
- // no $text clause, then a text score will not be available.
- QueryMetadataBitSet unavailableMetadata;
-
- // This is saved for an edge case where we need to re-parse a projection later. Only populated
- // if there is a non-empty projection.
- boost::optional<ProjectionPolicies> savedProjectionPolicies;
-
- // True if this canonical query may have converted extensions such as $where and $text into
- // no-ops during parsing. This will be the case if it allowed $where and $text in parsing, but
- // parsed using an ExtensionsCallbackNoop. This does not guarantee that a $where or $text
- // existed in the query. Queries with a no-op extension context are special because they can be
- // parsed and planned, but they cannot be executed.
- bool canHaveNoopMatchNodes;
-
- // All other parameters to the find command which do not have AST-like types and can be
- // appropriately tracked as raw value types like ints. The fields above like 'filter' are all
- // still present in their raw form on this FidnCommandRequest, but it is not expected that they
- // will be useful other than to keep the original BSON values around in-memory to avoid copying
- // large strings and such.
- std::unique_ptr<FindCommandRequest> findCommandRequest;
-};
-
-namespace parsed_find_command {
-/**
- * Validates the match expression 'root' as well as the query specified by 'request', checking for
- * illegal combinations of operators. Returns a non-OK status if any such illegal combination is
- * found.
- *
- * This method can be called both on normalized and non-normalized 'root'. However, some checks can
- * only be performed once the match expressions is normalized. To perform these checks one can call
- * 'CanonicalQuery::isValidNormalized()'.
- *
- * On success, returns a bitset indicating which types of metadata are *unavailable*. For example,
- * if 'root' does not contain a $text predicate, then the returned metadata bitset will indicate
- * that text score metadata is unavailable. This means that if subsequent $meta:"textScore"
- * expressions are found during analysis of the query, we should raise in an error.
- */
-StatusWith<QueryMetadataBitSet> isValid(const MatchExpression* root,
- const FindCommandRequest& findCommand);
-
-/**
- * Parses each big component of the input 'findCommand.' Throws exceptions if failing to parse.
- * Comes in one overload which will create an ExpressionContext for the caller, and one overload to
- * be used when the caller already has an ExpressionContext.
- */
-StatusWith<std::pair<boost::intrusive_ptr<ExpressionContext>, std::unique_ptr<ParsedFindCommand>>>
-parse(OperationContext* opCtx,
- std::unique_ptr<FindCommandRequest> findCommand,
- const ExtensionsCallback& extensionsCallback = ExtensionsCallbackNoop(),
- MatchExpressionParser::AllowedFeatureSet allowedFeatures =
- MatchExpressionParser::kDefaultSpecialFeatures,
- const ProjectionPolicies& projectionPolicies = ProjectionPolicies::findProjectionPolicies());
-
-// Overload of the above for when the caller has an available ExpressionContext.
-StatusWith<std::unique_ptr<ParsedFindCommand>> parse(
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- std::unique_ptr<FindCommandRequest> findCommand,
- const ExtensionsCallback& extensionsCallback = ExtensionsCallbackNoop(),
- MatchExpressionParser::AllowedFeatureSet allowedFeatures =
- MatchExpressionParser::kDefaultSpecialFeatures,
- const ProjectionPolicies& projectionPolicies = ProjectionPolicies::findProjectionPolicies());
-
-} // namespace parsed_find_command
-} // namespace mongo
diff --git a/src/mongo/db/query/partitioned_cache.h b/src/mongo/db/query/partitioned_cache.h
deleted file mode 100644
index ca10f731b71..00000000000
--- a/src/mongo/db/query/partitioned_cache.h
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/db/catalog/util/partitioned.h"
-#include "mongo/db/commands/server_status_metric.h"
-#include "mongo/db/query/lru_key_value.h"
-#include "mongo/platform/mutex.h"
-#include "mongo/util/container_size_helper.h"
-
-namespace mongo {
-
-/**
- * A partitioned cache combines a size-bounded map (LRU-based entry eviction) with a partition
- * function which allows reducing contention.
- */
-template <class KeyType,
- class ValueType,
- class KeyBudgetEstimator,
- class Partitioner,
- class InsertionEvictionListener,
- class KeyHasher = std::hash<KeyType>,
- class Eq = std::equal_to<KeyType>>
-class PartitionedCache {
-private:
- PartitionedCache(const PartitionedCache&) = delete;
- PartitionedCache& operator=(const PartitionedCache&) = delete;
-
-public:
- using Lru = LRUKeyValue<KeyType,
- ValueType,
- KeyBudgetEstimator,
- InsertionEvictionListener,
- KeyHasher,
- Eq>;
- using Partition = typename Partitioned<Lru, Partitioner>::OnePartition;
- using PartitionId = typename Partitioned<Lru, Partitioner>::PartitionId;
-
- /**
- * Initialize plan cache with the total cache size in bytes and number of partitions.
- *
- * Important edge cases to consider include:
- *
- * 1. Adding an entry that is larger than the max partition size to a non-empty partition.
- *
- * This will evict both entries. This is because entries are evicted from the partition in
- * order of least recently used. Thus, the oldest, small entry will be evicted first but the
- * partition will still be over budget with the new, too-large entry so it will be evicted as
- * well.
- *
- * 2. Adding a queryStats store entry that is smaller than the overall cache size but larger
- * than single partition max size.
- *
- * It is not possible to write entries to the cache that are larger than a single
- * partition's max size, even if it is smaller than the entire cache max size. This is because
- * the cache's budget is configured/regulated on the partition level (cacheSize /
- * numPartitions). This makes sense as each entry is written to a specific partition, but might
- * not be immediately obvious so worthy to highlight.
- *
- * 3. Too few partitions can cause unnecessary evictions
- *
- * Every class that implements the PartitionedCache template provides a partitioner() that
- * returns the id of the partition to which to write the entry. In existing implementations,
- * partitioner() returns the remainder after dividing the entry's key hash by numPartitions. In
- * the case where we have only two partitions, every odd key hash will be written to the first
- * partition (and vice versa). In this way, it can quickly be the case that one partition
- * fills up completely but the partitioner() call keeps returning the already full partition and
- * the cache evict old entries from it to put the new one in. At the end of all the write
- * operations, the cache is below it's budget (as the second partition is only partially full)
- * but we don't have all the entries we expect. It is therefore important to have sufficient
- * enough number of partitions so the entries can be more equally dispersed to avoid unnecessary
- * evictions.
- */
- explicit PartitionedCache(size_t cacheSize, size_t numPartitions)
- : _numPartitions(numPartitions) {
- invariant(numPartitions > 0);
- Lru lru{cacheSize / numPartitions};
- _partitionedCache =
- std::make_unique<Partitioned<Lru, Partitioner>>(numPartitions, std::move(lru));
- }
-
- ~PartitionedCache() = default;
- /**
- * Inserts the provided <key, value> into the partition associated with that key. Returns the
- * number of older entries evicted to fit this new one.
- */
- size_t put(const KeyType& key, ValueType value) {
- auto partition = _partitionedCache->lockOnePartition(key);
- return partition->add(key, std::move(value));
- }
- /**
- * Inserts the provided <key, value> into the specified partition. Returns the number of older
- * entries evicted to fit this new one.
- */
- size_t put(const KeyType& key, ValueType value, Partition& partition) {
- return partition->add(key, std::move(value));
- }
-
- StatusWith<ValueType*> lookup(const KeyType& key) const {
- auto partition = _partitionedCache->lockOnePartition(key);
- auto entry = partition->get(key);
- if (!entry.isOK()) {
- return {entry.getStatus()};
- }
-
- return {&entry.getValue()->second};
- }
-
- /**
- * Lookup an entry and also return a lock over the partition. The lock is returned whether
- * or not the entry is found.
- */
- std::pair<StatusWith<ValueType*>, Partition> getWithPartitionLock(const KeyType& key) const {
- auto partition = _partitionedCache->lockOnePartition(key);
- auto entry = partition->get(key);
- if (!entry.isOK()) {
- return std::make_pair(entry.getStatus(), std::move(partition));
- }
-
- return std::make_pair(StatusWith{&entry.getValue()->second}, std::move(partition));
- }
-
- /**
- * Remove the entry with the 'key' from the cache. If there is no entry for the given key in
- * the cache, this call is a no-op.
- */
- void remove(const KeyType& key) {
- _partitionedCache->erase(key);
- }
-
- /**
- * Remove all the entries for keys for which the predicate returns true. Return the number of
- * removed entries.
- */
- template <typename UnaryPredicate>
- size_t removeIf(UnaryPredicate predicate) {
- size_t nRemoved = 0;
- for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) {
- auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId);
- nRemoved += lockedPartition->removeIf(predicate);
- }
- return nRemoved;
- }
-
- /**
- * Remove *all* cache entries.
- */
- void clear() {
- _partitionedCache->clear();
- }
-
- /**
- * Reset total cache size. If the size is set to a smaller value than before, enough entries are
- * evicted in order to ensure that the cache fits within the new budget. Returns the number of
- * entries evicted.
- */
- size_t reset(size_t cacheSize) {
- size_t numEvicted = 0;
- for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) {
- auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId);
- numEvicted += lockedPartition->reset(cacheSize / _numPartitions);
- }
-
- return numEvicted;
- }
-
- /**
- * Returns the size of the cache.
- * Used for testing.
- */
- size_t size() const {
- return _partitionedCache->size();
- }
-
- /**
- * Returns the number of partitions.
- */
- size_t numPartitions() const {
- return _numPartitions;
- }
-
- /**
- * Invoke `op` for each entry in the cache. Consistency across partitions is not guaranteed.
- */
- void forEach(const std::function<void(const KeyType&, const ValueType&)>& op) const {
- for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) {
- auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId);
-
- for (auto&& [key, entry] : *lockedPartition) {
- op(*key, entry);
- }
- }
- }
-
- /**
- * Allow iterating over partitions. The provided function is called for each partition. The
- * argument to the function is another function which can delay acquiring the implicitly locked
- * partition until it's needed.
- */
- void forEachPartition(const std::function<void(const std::function<Partition()>&)>& op) const {
- for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) {
- op([&]() { return _partitionedCache->lockOnePartitionById(partitionId); });
- }
- }
-
- Partition getPartition(PartitionId partitionId) {
- return _partitionedCache->lockOnePartitionById(partitionId);
- }
-
-private:
- std::size_t _numPartitions;
- std::unique_ptr<Partitioned<Lru, Partitioner>> _partitionedCache;
-};
-
-} // namespace mongo
diff --git a/src/mongo/db/query/plan_cache.h b/src/mongo/db/query/plan_cache.h
index e193309230e..dc52b10c4f4 100644
--- a/src/mongo/db/query/plan_cache.h
+++ b/src/mongo/db/query/plan_cache.h
@@ -31,7 +31,6 @@
#include "mongo/db/catalog/util/partitioned.h"
#include "mongo/db/query/lru_key_value.h"
-#include "mongo/db/query/partitioned_cache.h"
#include "mongo/db/query/plan_cache_callbacks.h"
#include "mongo/db/query/plan_cache_debug_info.h"
#include "mongo/platform/mutex.h"
@@ -282,34 +281,21 @@ private:
*/
template <class KeyType,
class CachedPlanType,
- class KeyBudgetEstimator,
+ class BudgetEstimator,
class DebugInfoType,
class Partitioner,
class KeyHasher = std::hash<KeyType>>
-class PlanCacheBase
- : public PartitionedCache<
- KeyType,
- // The 'Value' being "std::shared_ptr<const Entry>" is because we allow readers to clone
- // cache entries out of the lock, therefore it is illegal to mutate the pieces of a cache
- // entry that can be cloned whether you are holding a lock or not.
- std::shared_ptr<const PlanCacheEntryBase<CachedPlanType, DebugInfoType>>,
- KeyBudgetEstimator,
- Partitioner,
- NoopInsertionEvictionListener,
- KeyHasher> {
+class PlanCacheBase {
private:
PlanCacheBase(const PlanCacheBase&) = delete;
PlanCacheBase& operator=(const PlanCacheBase&) = delete;
public:
- using Base =
- PartitionedCache<KeyType,
- std::shared_ptr<const PlanCacheEntryBase<CachedPlanType, DebugInfoType>>,
- KeyBudgetEstimator,
- Partitioner,
- NoopInsertionEvictionListener,
- KeyHasher>;
using Entry = PlanCacheEntryBase<CachedPlanType, DebugInfoType>;
+ // The 'Value' being "std::shared_ptr<const Entry>" is because we allow readers to clone cache
+ // entries out of the lock, therefore it is illegal to mutate the pieces of a cache entry that
+ // can be cloned whether you are holding a lock or not.
+ using Lru = LRUKeyValue<KeyType, std::shared_ptr<const Entry>, BudgetEstimator, KeyHasher>;
// We have three states for a cache entry to be in. Rather than just 'present' or 'not
// present', we use a notion of 'inactive entries' as a way of remembering how performant our
@@ -342,7 +328,11 @@ public:
* Initialize plan cache with the total cache size in bytes and number of partitions.
*/
explicit PlanCacheBase(size_t cacheSize, size_t numPartitions = 1)
- : Base(cacheSize, numPartitions) {}
+ : _numPartitions(numPartitions) {
+ invariant(numPartitions > 0);
+ Lru lru{cacheSize / numPartitions};
+ _partitionedCache = std::make_unique<Partitioned<Lru, Partitioner>>(numPartitions, lru);
+ }
~PlanCacheBase() = default;
@@ -385,11 +375,7 @@ public:
}},
why.stats);
- auto oldEntryWithPartitionLock = this->getWithPartitionLock(key);
- // Can't use reference to structured bindings in a lambda until C++20 so manually
- // destructure it here.
- auto partitionLock = std::move(oldEntryWithPartitionLock.second);
- auto oldEntryWithStatus = std::move(oldEntryWithPartitionLock.first);
+ auto partition = _partitionedCache->lockOnePartition(key);
auto [queryHash, planCacheKey, isNewEntryActive, shouldBeCreated, increasedWorks] = [&]() {
if (internalQueryCacheDisableInactiveEntries.load()) {
// All entries are always active.
@@ -399,34 +385,32 @@ public:
true /* shouldBeCreated */,
boost::optional<size_t>(boost::none));
} else {
+ auto oldEntryWithStatus = partition->get(key);
tassert(6007020,
"LRU store must get value or NoSuchKey error code",
oldEntryWithStatus.isOK() ||
oldEntryWithStatus.getStatus() == ErrorCodes::NoSuchKey);
- bool hasOldEntry = oldEntryWithStatus.isOK();
+ auto oldEntry =
+ oldEntryWithStatus.isOK() ? oldEntryWithStatus.getValue()->second : nullptr;
const auto newState = getNewEntryState(
key,
- // Deference the pointer, then the shared_ptr, and then back to a raw pointer.
- hasOldEntry ? &**oldEntryWithStatus.getValue() : nullptr,
+ oldEntry.get(),
newWorks,
worksGrowthCoefficient.get_value_or(internalQueryCacheWorksGrowthCoefficient),
callbacks);
// Avoid recomputing the hashes if we've got an old entry to grab them from.
- auto [queryHash, planCacheKey] = [&]() {
- if (hasOldEntry) {
- auto&& oldEntry = &**oldEntryWithStatus.getValue();
- return std::make_pair(oldEntry->queryHash, oldEntry->planCacheKey);
- } else {
- return std::make_pair(key.queryHash(), key.planCacheKeyHash());
- }
- }();
- return std::make_tuple(queryHash,
- planCacheKey,
- newState.shouldBeActive,
- newState.shouldBeCreated,
- newState.increasedWorks);
+ return oldEntry ? std::make_tuple(oldEntry->queryHash,
+ oldEntry->planCacheKey,
+ newState.shouldBeActive,
+ newState.shouldBeCreated,
+ newState.increasedWorks)
+ : std::make_tuple(key.queryHash(),
+ key.planCacheKeyHash(),
+ newState.shouldBeActive,
+ newState.shouldBeCreated,
+ newState.increasedWorks);
}
}();
@@ -450,7 +434,7 @@ public:
increasedWorks ? *increasedWorks : newWorks,
callbacks->buildDebugInfo());
- this->put(key, std::move(newEntry), partitionLock);
+ partition->add(key, std::move(newEntry));
return Status::OK();
}
@@ -470,7 +454,10 @@ public:
indexFilterKey,
now,
std::move(debugInfo));
- this->put(key, std::move(entry));
+ auto partition = _partitionedCache->lockOnePartition(key);
+ // We're not interested in the number of evicted entries if the cache store exceeds the
+ // budget after add(), so we just ignore the return value.
+ partition->add(key, std::move(entry));
}
/**
@@ -484,8 +471,8 @@ public:
return;
}
- auto [entry, partitionLock] = this->getWithPartitionLock(key);
-
+ auto partition = _partitionedCache->lockOnePartition(key);
+ auto entry = partition->get(key);
if (!entry.isOK()) {
tassert(6007021,
"Unexpected error code from LRU store",
@@ -493,11 +480,11 @@ public:
return;
}
- auto entryPtr = *entry.getValue();
+ auto entryPtr = entry.getValue()->second;
if (entryPtr->isActive == true) {
std::shared_ptr<Entry> newEntry = entryPtr->clone();
newEntry->isActive = false;
- this->put(key, std::move(newEntry), partitionLock);
+ partition->add(key, std::move(newEntry));
}
}
@@ -509,26 +496,29 @@ public:
* for the query (if there is one).
*/
GetResult get(const KeyType& key) const {
- std::shared_ptr<const Entry> entrySharedPtr;
+ std::shared_ptr<const Entry> entryPtr;
CacheEntryState state;
{
- auto [entry, partitionLock] = this->getWithPartitionLock(key);
+ auto partition = _partitionedCache->lockOnePartition(key);
+ auto entry = partition->get(key);
if (!entry.isOK()) {
tassert(6007023,
"Unexpected error code from LRU store",
entry.getStatus() == ErrorCodes::NoSuchKey);
return {CacheEntryState::kNotPresent, nullptr};
}
- entrySharedPtr = *entry.getValue();
- state = entrySharedPtr->isActive ? CacheEntryState::kPresentActive
- : CacheEntryState::kPresentInactive;
+ entryPtr = entry.getValue()->second;
+ state = entryPtr->isActive ? CacheEntryState::kPresentActive
+ : CacheEntryState::kPresentInactive;
}
- // The purpose of cloning 'entry' (in CachedPlanHolder ctor) after we release the lock
- // is to allow multiple threads to clone the same plan cache entry at once. 'entry'
- // cannot be deleted by another thread even if the plan cache is being concurrently
- // modified by other threads because we are holding a std::shared_ptr to this entry.
+ // The purpose of cloning 'entry' after we release the lock is to allow multiple threads to
+ // clone the same plan cache entry at once. 'entry' cannot be deleted by another thread even
+ // if the plan cache is being concurrently modified by other threads because we are holding
+ // a std::shared_ptr to this entry.
+ tassert(6007024, "LRU store must get a value or an error code", entryPtr);
+
return {state,
- std::make_unique<CachedPlanHolder<CachedPlanType, DebugInfoType>>(*entrySharedPtr)};
+ std::make_unique<CachedPlanHolder<CachedPlanType, DebugInfoType>>(*entryPtr)};
}
/**
@@ -547,16 +537,59 @@ public:
}
/**
+ * Remove the entry with the 'key' from the cache. If there is no entry for the given key in
+ * the cache, this call is a no-op.
+ */
+ void remove(const KeyType& key) {
+ _partitionedCache->erase(key);
+ }
+
+ /**
+ * Remove all the entries for keys for which the predicate returns true. Return the number of
+ * removed entries.
+ */
+ template <typename UnaryPredicate>
+ size_t removeIf(UnaryPredicate predicate) {
+ size_t nRemoved = 0;
+ for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) {
+ auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId);
+ nRemoved += lockedPartition->removeIf(predicate);
+ }
+ return nRemoved;
+ }
+
+ /**
+ * Remove *all* cached plans. Does not clear index information.
+ */
+ void clear() {
+ _partitionedCache->clear();
+ }
+
+ /**
+ * Reset total cache size. If the size is set to a smaller value than before, enough entries are
+ * evicted in order to ensure that the cache fits within the new budget.
+ */
+ void reset(size_t cacheSize) {
+ for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) {
+ auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId);
+ lockedPartition->reset(cacheSize / _numPartitions);
+ }
+ }
+
+ /**
* Returns a copy of a cache entry, looked up by the plan cache key.
*
* If there is no entry in the cache for the 'query', returns an error Status.
*/
StatusWith<std::unique_ptr<Entry>> getEntry(const KeyType& key) const {
- auto result = this->lookup(key);
- if (!result.isOK()) {
- return {result.getStatus()};
+ auto partition = _partitionedCache->lockOnePartition(key);
+ auto entry = partition->get(key);
+ if (!entry.isOK()) {
+ return entry.getStatus();
}
- return {result.getValue()->get()->clone()};
+ invariant(entry.getValue()->second);
+
+ return std::unique_ptr<Entry>(entry.getValue()->second->clone());
}
/**
@@ -565,14 +598,26 @@ public:
std::vector<std::unique_ptr<Entry>> getAllEntries() const {
std::vector<std::unique_ptr<Entry>> entries;
- this->forEach([&](const KeyType& key, const std::shared_ptr<Entry>& entry) {
- entries.emplace_back(entry);
- });
+ for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) {
+ auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId);
+
+ for (auto&& [key, entry] : *lockedPartition) {
+ entries.emplace_back(entry->clone());
+ }
+ }
return entries;
}
/**
+ * Returns the size of the cache.
+ * Used for testing.
+ */
+ size_t size() const {
+ return _partitionedCache->size();
+ }
+
+ /**
* Iterates over the plan cache. For each entry, first filters according to the predicate
* function 'cacheKeyFilterFunc', (Note that 'cacheKeyFilterFunc' could be empty, if so, we
* don't filter by plan cache key.), then serializes the PlanCacheEntryBase according to
@@ -589,15 +634,20 @@ public:
std::vector<BSONObj> results;
- this->forEach([&](const KeyType& key, const std::shared_ptr<const Entry>& entry) {
- if (cacheKeyFilterFunc && !cacheKeyFilterFunc(key)) {
- return;
- }
- auto serializedEntry = serializationFunc(*entry);
- if (filterFunc(serializedEntry)) {
- results.push_back(serializedEntry);
+ for (size_t partitionId = 0; partitionId < _numPartitions; ++partitionId) {
+ auto lockedPartition = _partitionedCache->lockOnePartitionById(partitionId);
+
+ for (auto&& cacheEntry : *lockedPartition) {
+ if (cacheKeyFilterFunc && !cacheKeyFilterFunc(cacheEntry.first)) {
+ continue;
+ }
+ const auto& entry = cacheEntry.second;
+ auto serializedEntry = serializationFunc(*entry);
+ if (filterFunc(serializedEntry)) {
+ results.push_back(serializedEntry);
+ }
}
- });
+ }
return results;
}
@@ -685,6 +735,9 @@ private:
return res;
}
+
+ std::size_t _numPartitions;
+ std::unique_ptr<Partitioned<Lru, Partitioner>> _partitionedCache;
};
} // namespace mongo
diff --git a/src/mongo/db/query/plan_cache_indexability.cpp b/src/mongo/db/query/plan_cache_indexability.cpp
index 0498d5edc26..4f1b0c176f8 100644
--- a/src/mongo/db/query/plan_cache_indexability.cpp
+++ b/src/mongo/db/query/plan_cache_indexability.cpp
@@ -79,6 +79,7 @@ IndexabilityDiscriminator getCollatedIndexDiscriminator(const CollatorInterface*
}
return true;
}
+
// The predicate never compares strings so it is not affected by collation.
return true;
};
@@ -103,7 +104,14 @@ void PlanCacheIndexabilityState::processSparseIndex(const std::string& indexName
void PlanCacheIndexabilityState::processPartialIndex(const std::string& indexName,
const MatchExpression* filterExpr) {
- _globalDiscriminatorMap[indexName].addDiscriminator(getPartialIndexDiscriminator(filterExpr));
+ invariant(filterExpr);
+ for (size_t i = 0; i < filterExpr->numChildren(); ++i) {
+ processPartialIndex(indexName, filterExpr->getChild(i));
+ }
+ if (filterExpr->getCategory() != MatchExpression::MatchCategory::kLogical) {
+ _pathDiscriminatorsMap[filterExpr->path()][indexName].addDiscriminator(
+ getPartialIndexDiscriminator(filterExpr));
+ }
}
void PlanCacheIndexabilityState::processWildcardIndex(const CoreIndexInfo& cii) {
@@ -126,7 +134,7 @@ namespace {
const IndexToDiscriminatorMap emptyDiscriminators{};
} // namespace
-const IndexToDiscriminatorMap& PlanCacheIndexabilityState::getPathDiscriminators(
+const IndexToDiscriminatorMap& PlanCacheIndexabilityState::getDiscriminators(
StringData path) const {
PathDiscriminatorsMap::const_iterator it = _pathDiscriminatorsMap.find(path);
if (it == _pathDiscriminatorsMap.end()) {
@@ -158,7 +166,6 @@ IndexToDiscriminatorMap PlanCacheIndexabilityState::buildWildcardDiscriminators(
void PlanCacheIndexabilityState::updateDiscriminators(
const std::vector<CoreIndexInfo>& indexCores) {
_pathDiscriminatorsMap = PathDiscriminatorsMap();
- _globalDiscriminatorMap = IndexToDiscriminatorMap();
_wildcardIndexDiscriminators.clear();
for (const auto& idx : indexCores) {
diff --git a/src/mongo/db/query/plan_cache_indexability.h b/src/mongo/db/query/plan_cache_indexability.h
index 0aa08359c27..9bc03494865 100644
--- a/src/mongo/db/query/plan_cache_indexability.h
+++ b/src/mongo/db/query/plan_cache_indexability.h
@@ -47,7 +47,6 @@ class ProjectionExecutor;
using IndexabilityDiscriminator = std::function<bool(const MatchExpression* me)>;
using IndexabilityDiscriminators = std::vector<IndexabilityDiscriminator>;
using IndexToDiscriminatorMap = StringMap<CompositeIndexabilityDiscriminator>;
-using PathDiscriminatorsMap = StringMap<IndexToDiscriminatorMap>;
/**
* CompositeIndexabilityDiscriminator holds all indexability discriminators for a particular path,
@@ -78,14 +77,9 @@ private:
};
/**
- * PlanCacheIndexabilityState holds a set of "indexability discriminators. An indexability
- * discriminator is a binary predicate function, used to classify match expressions based on the
- * data values in the expression.
- *
- * These discriminators are used to distinguish between queries of a similar shape but not the same
- * candidate indexes. So each discriminator typically represents a decision like "is this index
- * valid?" or "does this piece of the query disqualify it from using this index?". The output of
- * these decisions is included in the plan cache key.
+ * PlanCacheIndexabilityState holds a set of "indexability discriminators" for certain paths.
+ * An indexability discriminator is a binary predicate function, used to classify match
+ * expressions based on the data values in the expression.
*/
class PlanCacheIndexabilityState {
PlanCacheIndexabilityState(const PlanCacheIndexabilityState&) = delete;
@@ -101,15 +95,7 @@ public:
* The object returned by reference is valid until the next call to updateDiscriminators() or
* until destruction of 'this', whichever is first.
*/
- const IndexToDiscriminatorMap& getPathDiscriminators(StringData path) const;
-
- /**
- * Returns a map of index name to discriminator set. These discriminators are not
- * associated with a particular path of a query and apply to the entire MatchExpression.
- */
- const IndexToDiscriminatorMap& getGlobalDiscriminators() const {
- return _globalDiscriminatorMap;
- }
+ const IndexToDiscriminatorMap& getDiscriminators(StringData path) const;
/**
* Construct an IndexToDiscriminator map for the given path, only for the wildcard indexes
@@ -123,6 +109,8 @@ public:
void updateDiscriminators(const std::vector<CoreIndexInfo>& indexCores);
private:
+ using PathDiscriminatorsMap = StringMap<IndexToDiscriminatorMap>;
+
/**
* A $** index may index an infinite number of fields. We cannot just store a discriminator for
* every possible field that it indexes, so we have to maintain some special context about the
@@ -154,8 +142,8 @@ private:
void processSparseIndex(const std::string& indexName, const BSONObj& keyPattern);
/**
- * Adds a global discriminator for the partial index with the given filter expression
- * to the discriminators for that index in '_globalDiscriminatorMap'.
+ * Adds partial index discriminators for the partial index with the given filter expression
+ * to the discriminators for that index in '_pathDiscriminatorsMap'.
*
* A partial index discriminator distinguishes expressions that match a given partial index
* predicate from expressions that don't match the partial index predicate. For example,
@@ -186,10 +174,6 @@ private:
// PathDiscriminatorsMap is a map from field path to index name to IndexabilityDiscriminator.
PathDiscriminatorsMap _pathDiscriminatorsMap;
- // Map from index name to global discriminators. These are discriminators which do not apply to
- // a single path but the entire MatchExpression.
- IndexToDiscriminatorMap _globalDiscriminatorMap;
-
std::vector<WildcardIndexDiscriminatorContext> _wildcardIndexDiscriminators;
};
diff --git a/src/mongo/db/query/plan_cache_indexability_test.cpp b/src/mongo/db/query/plan_cache_indexability_test.cpp
index 49f9fb79285..af7677a8cd5 100644
--- a/src/mongo/db/query/plan_cache_indexability_test.cpp
+++ b/src/mongo/db/query/plan_cache_indexability_test.cpp
@@ -104,7 +104,7 @@ TEST(PlanCacheIndexabilityTest, SparseIndexSimple) {
nullptr,
nullptr)});
- auto discriminators = state.getPathDiscriminators("a");
+ auto discriminators = state.getDiscriminators("a");
ASSERT_EQ(1U, discriminators.size());
ASSERT(discriminators.find("a_1") != discriminators.end());
@@ -146,7 +146,7 @@ TEST(PlanCacheIndexabilityTest, SparseIndexCompound) {
nullptr)});
{
- auto discriminators = state.getPathDiscriminators("a");
+ auto discriminators = state.getDiscriminators("a");
ASSERT_EQ(1U, discriminators.size());
ASSERT(discriminators.find("a_1_b_1") != discriminators.end());
@@ -159,7 +159,7 @@ TEST(PlanCacheIndexabilityTest, SparseIndexCompound) {
}
{
- auto discriminators = state.getPathDiscriminators("b");
+ auto discriminators = state.getDiscriminators("b");
ASSERT_EQ(1U, discriminators.size());
ASSERT(discriminators.find("a_1_b_1") != discriminators.end());
@@ -193,17 +193,12 @@ TEST(PlanCacheIndexabilityTest, PartialIndexSimple) {
nullptr,
nullptr)});
- // The partial index is represented as a global discriminator that applies to the entire
- // incoming MatchExpression.
{
- auto discriminators = state.getPathDiscriminators("f");
- ASSERT_EQ(0U, discriminators.size());
-
- auto globalDiscriminators = state.getGlobalDiscriminators();
- ASSERT_EQ(1U, globalDiscriminators.size());
- ASSERT(globalDiscriminators.find("a_1") != globalDiscriminators.end());
+ auto discriminators = state.getDiscriminators("f");
+ ASSERT_EQ(1U, discriminators.size());
+ ASSERT(discriminators.find("a_1") != discriminators.end());
- auto disc = globalDiscriminators["a_1"];
+ auto disc = discriminators["a_1"];
ASSERT_EQ(false,
disc.isMatchCompatibleWithIndex(
parseMatchExpression(BSON("f" << BSON("$gt" << -5))).get()));
@@ -213,7 +208,7 @@ TEST(PlanCacheIndexabilityTest, PartialIndexSimple) {
}
{
- auto discriminators = state.getPathDiscriminators("a");
+ auto discriminators = state.getDiscriminators("a");
ASSERT_EQ(1U, discriminators.size());
ASSERT(discriminators.find("a_1") != discriminators.end());
@@ -248,52 +243,32 @@ TEST(PlanCacheIndexabilityTest, PartialIndexAnd) {
nullptr,
nullptr)});
- // partial index discriminators are global to the entire query, so an individual path should not
- // have any discriminators. Also the entire query must be a subset of the partial filter
- // expression, not just the leaves.
- auto globalDiscriminators = state.getGlobalDiscriminators();
- ASSERT(globalDiscriminators.find("a_1") != globalDiscriminators.end());
- auto globalDisc = globalDiscriminators["a_1"];
-
{
- auto discriminators = state.getPathDiscriminators("f");
- ASSERT_EQ(0U, discriminators.size());
+ auto discriminators = state.getDiscriminators("f");
+ ASSERT_EQ(1U, discriminators.size());
+ ASSERT(discriminators.find("a_1") != discriminators.end());
- ASSERT_EQ(
- false,
- globalDisc.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 0)).get()));
- ASSERT_EQ(
- false,
- globalDisc.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 1)).get()));
+ auto disc = discriminators["a_1"];
+ ASSERT_EQ(false,
+ disc.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 0)).get()));
+ ASSERT_EQ(true,
+ disc.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 1)).get()));
}
{
- auto discriminators = state.getPathDiscriminators("g");
- ASSERT_EQ(0U, discriminators.size());
-
- ASSERT_EQ(
- false,
- globalDisc.isMatchCompatibleWithIndex(parseMatchExpression(BSON("g" << 0)).get()));
- ASSERT_EQ(
- false,
- globalDisc.isMatchCompatibleWithIndex(parseMatchExpression(BSON("g" << 1)).get()));
- }
+ auto discriminators = state.getDiscriminators("g");
+ ASSERT_EQ(1U, discriminators.size());
+ ASSERT(discriminators.find("a_1") != discriminators.end());
- {
- // A match expression which is covered entirely by the partial filter should pass the global
- // discriminator.
+ auto disc = discriminators["a_1"];
ASSERT_EQ(false,
- globalDisc.isMatchCompatibleWithIndex(
- parseMatchExpression(BSON("g" << 1 << "f" << 0)).get()));
+ disc.isMatchCompatibleWithIndex(parseMatchExpression(BSON("g" << 0)).get()));
ASSERT_EQ(true,
- globalDisc.isMatchCompatibleWithIndex(
- parseMatchExpression(BSON("g" << 1 << "f" << 1)).get()));
+ disc.isMatchCompatibleWithIndex(parseMatchExpression(BSON("g" << 1)).get()));
}
{
- // The path 'a' will still have a discriminator for the collation (even though it's
- // defaulted).
- auto discriminators = state.getPathDiscriminators("a");
+ auto discriminators = state.getDiscriminators("a");
ASSERT_EQ(1U, discriminators.size());
ASSERT(discriminators.find("a_1") != discriminators.end());
@@ -344,44 +319,33 @@ TEST(PlanCacheIndexabilityTest, MultiplePartialIndexes) {
nullptr,
nullptr)});
- // partial index discriminators are global to the entire query, so an individual path within the
- // partial filter should not have any discriminators. Also the entire query must be a subset of
- // the partial filter expression, not just the leaves.
- auto globalDiscriminators = state.getGlobalDiscriminators();
- ASSERT(globalDiscriminators.find("a_1") != globalDiscriminators.end());
- ASSERT(globalDiscriminators.find("b_1") != globalDiscriminators.end());
- auto globalDiscA = globalDiscriminators["a_1"];
- auto globalDiscB = globalDiscriminators["b_1"];
-
{
- auto discriminators = state.getPathDiscriminators("f");
- ASSERT_EQ(0U, discriminators.size());
+ auto discriminators = state.getDiscriminators("f");
+ ASSERT_EQ(2U, discriminators.size());
+ ASSERT(discriminators.find("a_1") != discriminators.end());
+ ASSERT(discriminators.find("b_1") != discriminators.end());
- ASSERT_EQ(
- false,
- globalDiscA.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 0)).get()));
- ASSERT_EQ(
- false,
- globalDiscB.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 0)).get()));
+ auto discA = discriminators["a_1"];
+ auto discB = discriminators["b_1"];
- ASSERT_EQ(
- true,
- globalDiscA.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 1)).get()));
- ASSERT_EQ(
- false,
- globalDiscB.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 1)).get()));
+ ASSERT_EQ(false,
+ discA.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 0)).get()));
+ ASSERT_EQ(false,
+ discB.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 0)).get()));
- ASSERT_EQ(
- false,
- globalDiscA.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 2)).get()));
- ASSERT_EQ(
- true,
- globalDiscB.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 2)).get()));
+ ASSERT_EQ(true,
+ discA.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 1)).get()));
+ ASSERT_EQ(false,
+ discB.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 1)).get()));
+
+ ASSERT_EQ(false,
+ discA.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 2)).get()));
+ ASSERT_EQ(true,
+ discB.isMatchCompatibleWithIndex(parseMatchExpression(BSON("f" << 2)).get()));
}
- // The paths 'a' and 'b' will have one discriminator each to capture the collation of the index.
{
- auto discriminators = state.getPathDiscriminators("a");
+ auto discriminators = state.getDiscriminators("a");
ASSERT_EQ(1U, discriminators.size());
ASSERT(discriminators.find("a_1") != discriminators.end());
@@ -395,7 +359,7 @@ TEST(PlanCacheIndexabilityTest, MultiplePartialIndexes) {
}
{
- auto discriminators = state.getPathDiscriminators("b");
+ auto discriminators = state.getDiscriminators("b");
ASSERT_EQ(1U, discriminators.size());
ASSERT(discriminators.find("b_1") != discriminators.end());
@@ -428,7 +392,7 @@ TEST(PlanCacheIndexabilityTest, IndexNeitherSparseNorPartial) {
BSONObj(),
nullptr,
nullptr)});
- auto discriminators = state.getPathDiscriminators("a");
+ auto discriminators = state.getDiscriminators("a");
ASSERT_EQ(1U, discriminators.size());
ASSERT(discriminators.find("a_1") != discriminators.end());
}
@@ -457,7 +421,7 @@ TEST(PlanCacheIndexabilityTest, DiscriminatorForCollationIndicatesWhenCollations
boost::intrusive_ptr<ExpressionContextForTest> expCtx(new ExpressionContextForTest());
expCtx->setCollator(collator.clone());
- auto discriminators = state.getPathDiscriminators("a");
+ auto discriminators = state.getDiscriminators("a");
ASSERT_EQ(1U, discriminators.size());
ASSERT(discriminators.find("a_1") != discriminators.end());
@@ -542,11 +506,11 @@ TEST(PlanCacheIndexabilityTest, CompoundIndexCollationDiscriminator) {
nullptr,
nullptr)});
- auto discriminatorsA = state.getPathDiscriminators("a");
+ auto discriminatorsA = state.getDiscriminators("a");
ASSERT_EQ(1U, discriminatorsA.size());
ASSERT(discriminatorsA.find("a_1_b_1") != discriminatorsA.end());
- auto discriminatorsB = state.getPathDiscriminators("b");
+ auto discriminatorsB = state.getDiscriminators("b");
ASSERT_EQ(1U, discriminatorsB.size());
ASSERT(discriminatorsB.find("a_1_b_1") != discriminatorsB.end());
}
@@ -655,15 +619,13 @@ TEST(PlanCacheIndexabilityTest, WildcardPartialIndexDiscriminator) {
ASSERT_TRUE(wildcardDiscriminators.isMatchCompatibleWithIndex(
parseMatchExpression(fromjson("{b: 6}")).get()));
- // The global discriminator for the index "indexName" should reflect whether a MatchExpression
- // is compatible with the partial filter expression.
+ // The regular (non-wildcard) set of discriminators for the path "a" should reflect whether a
+ // predicate on "a" is compatible with the partial filter expression.
{
- discriminatorsA = state.getPathDiscriminators("a");
- ASSERT(discriminatorsA.find("indexName") == discriminatorsA.end());
-
- auto globalDisc = state.getGlobalDiscriminators();
- ASSERT(globalDisc.find("indexName") != globalDisc.end());
- auto disc = globalDisc["indexName"];
+ discriminatorsA = state.getDiscriminators("a");
+ auto discriminatorsIt = discriminatorsA.find("indexName");
+ ASSERT(discriminatorsIt != discriminatorsA.end());
+ auto disc = discriminatorsIt->second;
ASSERT_FALSE(
disc.isMatchCompatibleWithIndex(parseMatchExpression(fromjson("{a: 0}")).get()));
@@ -678,7 +640,7 @@ TEST(PlanCacheIndexabilityTest, WildcardPartialIndexDiscriminator) {
// There shouldn't be any regular discriminators associated with path "b".
{
- auto&& discriminatorsB = state.getPathDiscriminators("b");
+ auto&& discriminatorsB = state.getDiscriminators("b");
ASSERT_FALSE(discriminatorsB.count("indexName"));
}
}
diff --git a/src/mongo/db/query/plan_cache_key_factory.cpp b/src/mongo/db/query/plan_cache_key_factory.cpp
index c5dbfbd7ea4..d47f1768858 100644
--- a/src/mongo/db/query/plan_cache_key_factory.cpp
+++ b/src/mongo/db/query/plan_cache_key_factory.cpp
@@ -29,29 +29,30 @@
#include "mongo/db/query/plan_cache_key_factory.h"
-#include "mongo/db/query/canonical_query_encoder.h"
#include "mongo/db/query/collection_query_info.h"
#include "mongo/db/query/planner_ixselect.h"
#include "mongo/db/s/operation_sharding_state.h"
namespace mongo {
namespace plan_cache_detail {
+// Delimiters for cache key encoding.
+const char kEncodeDiscriminatorsBegin = '<';
+const char kEncodeDiscriminatorsEnd = '>';
void encodeIndexabilityForDiscriminators(const MatchExpression* tree,
const IndexToDiscriminatorMap& discriminators,
StringBuilder* keyBuilder) {
-
for (auto&& indexAndDiscriminatorPair : discriminators) {
*keyBuilder << indexAndDiscriminatorPair.second.isMatchCompatibleWithIndex(tree);
}
}
-void encodeIndexabilityRecursive(const MatchExpression* tree,
- const PlanCacheIndexabilityState& indexabilityState,
- StringBuilder* keyBuilder) {
+void encodeIndexability(const MatchExpression* tree,
+ const PlanCacheIndexabilityState& indexabilityState,
+ StringBuilder* keyBuilder) {
if (!tree->path().empty()) {
const IndexToDiscriminatorMap& discriminators =
- indexabilityState.getPathDiscriminators(tree->path());
+ indexabilityState.getDiscriminators(tree->path());
IndexToDiscriminatorMap wildcardDiscriminators =
indexabilityState.buildWildcardDiscriminators(tree->path());
if (!discriminators.empty() || !wildcardDiscriminators.empty()) {
@@ -71,26 +72,8 @@ void encodeIndexabilityRecursive(const MatchExpression* tree,
}
for (size_t i = 0; i < tree->numChildren(); ++i) {
- encodeIndexabilityRecursive(tree->getChild(i), indexabilityState, keyBuilder);
- }
-}
-
-void encodeIndexability(const MatchExpression* tree,
- const PlanCacheIndexabilityState& indexabilityState,
- StringBuilder* keyBuilder) {
- // Before encoding the indexability of the leaf MatchExpressions, apply the global
- // discriminators to the expression as a whole. This is for cases such as partial indexes which
- // must discriminate based on the entire query.
- const auto& globalDiscriminators = indexabilityState.getGlobalDiscriminators();
- if (!globalDiscriminators.empty()) {
- *keyBuilder << kEncodeGlobalDiscriminatorsBegin;
- for (auto&& indexAndDiscriminatorPair : globalDiscriminators) {
- *keyBuilder << indexAndDiscriminatorPair.second.isMatchCompatibleWithIndex(tree);
- }
- *keyBuilder << kEncodeGlobalDiscriminatorsEnd;
+ encodeIndexability(tree->getChild(i), indexabilityState, keyBuilder);
}
-
- encodeIndexabilityRecursive(tree, indexabilityState, keyBuilder);
}
PlanCacheKeyInfo makePlanCacheKeyInfo(const CanonicalQuery& query,
@@ -128,8 +111,8 @@ boost::optional<Timestamp> computeNewestVisibleIndexTimestamp(OperationContext*
Timestamp currentNewestVisible = Timestamp::min();
- auto ii = collection->getIndexCatalog()->getIndexIterator(
- opCtx, IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished);
+ std::unique_ptr<IndexCatalog::IndexIterator> ii =
+ collection->getIndexCatalog()->getIndexIterator(opCtx, /*includeUnfinishedIndexes*/ true);
while (ii->more()) {
const IndexCatalogEntry* ice = ii->next();
auto minVisibleSnapshot = ice->getMinimumVisibleSnapshot();
diff --git a/src/mongo/db/query/plan_cache_key_info_test.cpp b/src/mongo/db/query/plan_cache_key_info_test.cpp
index a13616e12c4..7235386e7f4 100644
--- a/src/mongo/db/query/plan_cache_key_info_test.cpp
+++ b/src/mongo/db/query/plan_cache_key_info_test.cpp
@@ -194,107 +194,6 @@ TEST(PlanCacheKeyInfoTest, ComputeKeyPartialIndex) {
makeKey(*cqGtZero, indexCores));
}
-TEST(PlanCacheKeyInfoTest, ComputeKeyPartialIndexConjunction) {
- BSONObj filterObj = fromjson("{f: {$gt: 0, $lt: 10}}");
- unique_ptr<MatchExpression> filterExpr(parseMatchExpression(filterObj));
-
- const auto keyPattern = BSON("a" << 1);
- const std::vector<CoreIndexInfo> indexCores = {
- CoreIndexInfo(keyPattern,
- IndexNames::nameToType(IndexNames::findPluginName(keyPattern)),
- false, // sparse
- IndexEntry::Identifier{""}, // name
- filterExpr.get())}; // filterExpr
-
- unique_ptr<CanonicalQuery> satisfySinglePredicate(canonicalize("{f: {$gt: 0}}"));
- ASSERT_EQ(makeKey(*satisfySinglePredicate, indexCores).getIndexabilityDiscriminators(), "(0)");
-
- unique_ptr<CanonicalQuery> satisfyBothPredicates(canonicalize("{f: {$eq: 5}}"));
- ASSERT_EQ(makeKey(*satisfyBothPredicates, indexCores).getIndexabilityDiscriminators(), "(1)");
-
- unique_ptr<CanonicalQuery> conjSingleField(canonicalize("{f: {$gt: 2, $lt: 9}}"));
- ASSERT_EQ(makeKey(*conjSingleField, indexCores).getIndexabilityDiscriminators(), "(1)");
-
- unique_ptr<CanonicalQuery> conjSingleFieldNoMatch(canonicalize("{f: {$gt: 2, $lt: 11}}"));
- ASSERT_EQ(makeKey(*conjSingleFieldNoMatch, indexCores).getIndexabilityDiscriminators(), "(0)");
-
- // Note that these queries get optimized to a single $in over 'f'.
- unique_ptr<CanonicalQuery> disjSingleFieldBothSatisfy(
- canonicalize("{$or: [{f: {$eq: 2}}, {f: {$eq: 3}}]}"));
- ASSERT_EQ(makeKey(*disjSingleFieldBothSatisfy, indexCores).getIndexabilityDiscriminators(),
- "(1)");
-
- unique_ptr<CanonicalQuery> disjSingleFieldNotSubset(
- canonicalize("{$or: [{f: {$eq: 2}}, {f: {$eq: 11}}]}"));
- ASSERT_EQ(makeKey(*disjSingleFieldNotSubset, indexCores).getIndexabilityDiscriminators(),
- "(0)");
-}
-
-TEST(PlanCacheKeyInfoTest, ComputeKeyPartialIndexDisjunction) {
- BSONObj filterObj = fromjson("{$or: [{f: {$gt: 10}}, {f: {$lt: 0}}]}");
- unique_ptr<MatchExpression> filterExpr(parseMatchExpression(filterObj));
-
- const auto keyPattern = BSON("a" << 1);
- const std::vector<CoreIndexInfo> indexCores = {
- CoreIndexInfo(keyPattern,
- IndexNames::nameToType(IndexNames::findPluginName(keyPattern)),
- false, // sparse
- IndexEntry::Identifier{""}, // name
- filterExpr.get())}; // filterExpr
-
- unique_ptr<CanonicalQuery> satisfySinglePredicate(canonicalize("{f: {$eq: 11}}"));
- ASSERT_EQ(makeKey(*satisfySinglePredicate, indexCores).getIndexabilityDiscriminators(), "(1)");
-
- unique_ptr<CanonicalQuery> satisfyNeither(canonicalize("{f: {$eq: 5}}"));
- ASSERT_EQ(makeKey(*satisfyNeither, indexCores).getIndexabilityDiscriminators(), "(0)");
-
- unique_ptr<CanonicalQuery> conjSingleFieldMatch(canonicalize("{f: {$lt: 20, $gt: 10}}"));
- ASSERT_EQ(makeKey(*conjSingleFieldMatch, indexCores).getIndexabilityDiscriminators(), "(1)");
-
- unique_ptr<CanonicalQuery> conjSingleFieldNoMatch(canonicalize("{f: {$gt: 2, $lt: 10}}"));
- ASSERT_EQ(makeKey(*conjSingleFieldNoMatch, indexCores).getIndexabilityDiscriminators(), "(0)");
-
- unique_ptr<CanonicalQuery> conjSingleFieldOverlap(canonicalize("{f: {$gt: 2, $lt: 12}}"));
- ASSERT_EQ(makeKey(*conjSingleFieldOverlap, indexCores).getIndexabilityDiscriminators(), "(0)");
-
- // Although this query is technically a subset of the partial filter, the logic to determine
- // such ('isSubsetOf' in the code) is conservative in how it compares certain shapes of
- // expression trees.
- unique_ptr<CanonicalQuery> disjSingleFieldBothSatisfy(
- canonicalize("{$or: [{f: {$eq: -1}}, {f: {$gt: 10}}]}"));
- ASSERT_EQ(makeKey(*disjSingleFieldBothSatisfy, indexCores).getIndexabilityDiscriminators(),
- "(0)");
-
- unique_ptr<CanonicalQuery> disjSingleFieldNotSubset(
- canonicalize("{$or: [{f: {$eq: 2}}, {f: {$eq: 11}}]}"));
- ASSERT_EQ(makeKey(*disjSingleFieldNotSubset, indexCores).getIndexabilityDiscriminators(),
- "(0)");
-}
-
-TEST(PlanCacheKeyInfoTest, ComputeKeyPartialIndexNestedDisjunction) {
- BSONObj filterObj = fromjson(R"(
- {$and: [
- {$or: [{f: {$gt: 10}}, {f: {$lt: 0}}]},
- {$or: [{f: {$gt: 11}}, {f: {$lt: 1}}]}
- ]})");
- unique_ptr<MatchExpression> filterExpr(parseMatchExpression(filterObj));
-
- const auto keyPattern = BSON("a" << 1);
- const std::vector<CoreIndexInfo> indexCores = {
- CoreIndexInfo(keyPattern,
- IndexNames::nameToType(IndexNames::findPluginName(keyPattern)),
- false, // sparse
- IndexEntry::Identifier{""}, // name
- filterExpr.get())}; // filterExpr
-
-
- unique_ptr<CanonicalQuery> satisfySinglePredicate(canonicalize("{f: {$eq: 11}}"));
- ASSERT_EQ(makeKey(*satisfySinglePredicate, indexCores).getIndexabilityDiscriminators(), "(0)");
-
- unique_ptr<CanonicalQuery> notCompat(canonicalize("{f: {$eq: 12}}"));
- ASSERT_EQ(makeKey(*notCompat, indexCores).getIndexabilityDiscriminators(), "(1)");
-}
-
// Query shapes should get the same plan cache key if they have the same collation indexability.
TEST(PlanCacheKeyInfoTest, ComputeKeyCollationIndex) {
CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kReverseString);
@@ -467,8 +366,8 @@ TEST(PlanCacheKeyInfoTest, ComputeKeyWildcardDiscriminatesCorrectlyBasedOnPartia
// The discriminator strings have the format "<xx>". That is, there are two discriminator
// bits for the "x" predicate, the first pertaining to the partialFilterExpression and the
// second around applicability to the wildcard index.
- ASSERT_EQ(compatibleKey.getIndexabilityDiscriminators(), "(1)<1>");
- ASSERT_EQ(incompatibleKey.getIndexabilityDiscriminators(), "(0)<1>");
+ ASSERT_EQ(compatibleKey.getIndexabilityDiscriminators(), "<11>");
+ ASSERT_EQ(incompatibleKey.getIndexabilityDiscriminators(), "<01>");
}
// The partialFilterExpression should lead to a discriminator over field 'x', but not over 'y'.
@@ -483,8 +382,8 @@ TEST(PlanCacheKeyInfoTest, ComputeKeyWildcardDiscriminatesCorrectlyBasedOnPartia
// The discriminator strings have the format "<xx><y>". That is, there are two discriminator
// bits for the "x" predicate (the first pertaining to the partialFilterExpression, the
// second around applicability to the wildcard index) and one discriminator bit for "y".
- ASSERT_EQ(compatibleKey.getIndexabilityDiscriminators(), "(1)<1><1>");
- ASSERT_EQ(incompatibleKey.getIndexabilityDiscriminators(), "(0)<1><1>");
+ ASSERT_EQ(compatibleKey.getIndexabilityDiscriminators(), "<11><1>");
+ ASSERT_EQ(incompatibleKey.getIndexabilityDiscriminators(), "<01><1>");
}
// $eq:null predicates cannot be assigned to a wildcard index. Make sure that this is
@@ -499,8 +398,8 @@ TEST(PlanCacheKeyInfoTest, ComputeKeyWildcardDiscriminatesCorrectlyBasedOnPartia
// The discriminator strings have the format "<xx><y>". That is, there are two discriminator
// bits for the "x" predicate (the first pertaining to the partialFilterExpression, the
// second around applicability to the wildcard index) and one discriminator bit for "y".
- ASSERT_EQ(compatibleKey.getIndexabilityDiscriminators(), "(1)<1><1>");
- ASSERT_EQ(incompatibleKey.getIndexabilityDiscriminators(), "(1)<1><0>");
+ ASSERT_EQ(compatibleKey.getIndexabilityDiscriminators(), "<11><1>");
+ ASSERT_EQ(incompatibleKey.getIndexabilityDiscriminators(), "<11><0>");
}
// Test that the discriminators are correct for an $eq:null predicate on 'x'. This predicate is
@@ -509,7 +408,7 @@ TEST(PlanCacheKeyInfoTest, ComputeKeyWildcardDiscriminatesCorrectlyBasedOnPartia
// result in two "0" bits inside the discriminator string.
{
auto key = makeKey(*canonicalize("{x: {$eq: null}}"), indexCores);
- ASSERT_EQ(key.getIndexabilityDiscriminators(), "(0)<0>");
+ ASSERT_EQ(key.getIndexabilityDiscriminators(), "<00>");
}
}
@@ -550,11 +449,11 @@ TEST(PlanCacheKeyInfoTest, ComputeKeyWildcardDiscriminatesCorrectlyWithPartialFi
const std::vector<CoreIndexInfo> indexCores = {indexInfo};
{
- // TODO update The discriminators should have the format <xx><yy><z>. The 'z' predicate has
- // just one discriminator because it is not referenced in the partial filter expression. All
+ // The discriminators should have the format <xx><yy><z>. The 'z' predicate has just one
+ // discriminator because it is not referenced in the partial filter expression. All
// predicates are compatible.
auto key = makeKey(*canonicalize("{x: {$eq: 1}, y: {$eq: 2}, z: {$eq: 3}}"), indexCores);
- ASSERT_EQ(key.getIndexabilityDiscriminators(), "(1)<1><1><1>");
+ ASSERT_EQ(key.getIndexabilityDiscriminators(), "<11><11><1>");
}
{
@@ -562,7 +461,7 @@ TEST(PlanCacheKeyInfoTest, ComputeKeyWildcardDiscriminatesCorrectlyWithPartialFi
// compatible with the partial filter expression, leading to one of the 'y' bits being set
// to zero.
auto key = makeKey(*canonicalize("{x: {$eq: 1}, y: {$eq: -2}, z: {$eq: 3}}"), indexCores);
- ASSERT_EQ(key.getIndexabilityDiscriminators(), "(0)<1><1><1>");
+ ASSERT_EQ(key.getIndexabilityDiscriminators(), "<11><01><1>");
}
}
@@ -581,20 +480,20 @@ TEST(PlanCacheKeyInfoTest, ComputeKeyDiscriminatesCorrectlyWithPartialFilterAndW
// the predicate is compatible with the partial filter expression, whereas the disciminator
// for 'y' is about compatibility with the wildcard index.
auto key = makeKey(*canonicalize("{x: {$eq: 1}, y: {$eq: 2}, z: {$eq: 3}}"), indexCores);
- ASSERT_EQ(key.getIndexabilityDiscriminators(), "(1)<1>");
+ ASSERT_EQ(key.getIndexabilityDiscriminators(), "<1><1>");
}
{
// Similar to the previous case, except with an 'x' predicate that is incompatible with the
// partial filter expression.
auto key = makeKey(*canonicalize("{x: {$eq: -1}, y: {$eq: 2}, z: {$eq: 3}}"), indexCores);
- ASSERT_EQ(key.getIndexabilityDiscriminators(), "(0)<1>");
+ ASSERT_EQ(key.getIndexabilityDiscriminators(), "<0><1>");
}
{
// Case where the 'y' predicate is not compatible with the wildcard index.
auto key = makeKey(*canonicalize("{x: {$eq: 1}, y: {$eq: null}, z: {$eq: 3}}"), indexCores);
- ASSERT_EQ(key.getIndexabilityDiscriminators(), "(1)<0>");
+ ASSERT_EQ(key.getIndexabilityDiscriminators(), "<1><0>");
}
}
@@ -612,14 +511,14 @@ TEST(PlanCacheKeyInfoTest, ComputeKeyWildcardDiscriminatesCorrectlyWithPartialFi
// The discriminators have the format <x><(x.y)(x.y)<y>. All predicates are compatible
auto key =
makeKey(*canonicalize("{x: {$eq: 1}, y: {$eq: 2}, 'x.y': {$eq: 3}}"), indexCores);
- ASSERT_EQ(key.getIndexabilityDiscriminators(), "(1)<1><1><1>");
+ ASSERT_EQ(key.getIndexabilityDiscriminators(), "<1><11><1>");
}
{
// Here, the predicate on "x.y" is not compatible with the partial filter expression.
auto key =
makeKey(*canonicalize("{x: {$eq: 1}, y: {$eq: 2}, 'x.y': {$eq: -3}}"), indexCores);
- ASSERT_EQ(key.getIndexabilityDiscriminators(), "(0)<1><1><1>");
+ ASSERT_EQ(key.getIndexabilityDiscriminators(), "<1><01><1>");
}
}
diff --git a/src/mongo/db/query/query_stats/aggregated_metric.h b/src/mongo/db/query/plan_cache_size_parameter.cpp
index fca94c3183e..46e42efafbf 100644
--- a/src/mongo/db/query/query_stats/aggregated_metric.h
+++ b/src/mongo/db/query/plan_cache_size_parameter.cpp
@@ -1,5 +1,5 @@
/**
- * Copyright (C) 2023-present MongoDB, Inc.
+ * Copyright (C) 2021-present MongoDB, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
@@ -27,52 +27,48 @@
* it in the license file.
*/
-#pragma once
+#include "mongo/db/query/plan_cache_size_parameter.h"
-#include <algorithm>
-#include <cstdint>
+#include <pcrecpp.h>
-#include "mongo/base/string_data.h"
-#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/query/query_knobs_gen.h"
-namespace mongo::query_stats {
+namespace mongo::plan_cache_util {
-/**
- * An aggregated metric stores a compressed view of data. It balances the loss of information
- * with the reduction in required storage.
- */
-struct AggregatedMetric {
+StatusWith<PlanCacheSizeUnits> parseUnitString(const std::string& strUnit) {
+ if (strUnit.empty()) {
+ return Status(ErrorCodes::Error{6007010}, "Unit value cannot be empty");
+ }
- /**
- * Aggregate an observed value into the metric.
- */
- void aggregate(uint64_t val) {
- sum += val;
- max = std::max(val, max);
- min = std::min(val, min);
- sumOfSquares += val * val;
+ if (strUnit[0] == '%') {
+ return PlanCacheSizeUnits::kPercent;
+ } else if (strUnit[0] == 'M' || strUnit[0] == 'm') {
+ return PlanCacheSizeUnits::kMB;
+ } else if (strUnit[0] == 'G' || strUnit[0] == 'g') {
+ return PlanCacheSizeUnits::kGB;
}
- void appendTo(BSONObjBuilder& builder, const StringData& fieldName) const {
- BSONObjBuilder metricsBuilder = builder.subobjStart(fieldName);
- metricsBuilder.append("sum", (long long)sum);
- metricsBuilder.append("max", (long long)max);
- metricsBuilder.append("min", (long long)min);
- metricsBuilder.append("sumOfSquares", (long long)sumOfSquares);
- metricsBuilder.done();
+ return Status(ErrorCodes::Error{6007011}, "Incorrect unit value");
+}
+
+StatusWith<PlanCacheSizeParameter> PlanCacheSizeParameter::parse(const std::string& str) {
+ pcrecpp::RE_Options opt;
+ opt.set_caseless(true);
+ // Looks for a floating point number with followed by a unit suffix (MB, GB, %).
+ pcrecpp::RE re("\\s*(\\d+\\.?\\d*)\\s*(MB|GB|%)\\s*", opt);
+
+ double size{};
+ std::string strUnit{};
+ if (!re.FullMatch(str, &size, &strUnit)) {
+ return {ErrorCodes::Error{6007012}, "Unable to parse plan cache size string"};
}
- uint64_t sum = 0;
- // Default to the _signed_ maximum (which fits in unsigned range) because we cast to
- // BSONNumeric when serializing.
- uint64_t min = (uint64_t)std::numeric_limits<int64_t>::max;
- uint64_t max = 0;
+ auto statusWithUnit = parseUnitString(strUnit);
+ if (!statusWithUnit.isOK()) {
+ return statusWithUnit.getStatus();
+ }
- /**
- * The sum of squares along with (an externally stored) count will allow us to compute the
- * variance/stddev.
- */
- uint64_t sumOfSquares = 0;
-};
+ return PlanCacheSizeParameter{size, statusWithUnit.getValue()};
+}
-} // namespace mongo::query_stats
+} // namespace mongo::plan_cache_util
diff --git a/src/mongo/db/query/util/memory_util.h b/src/mongo/db/query/plan_cache_size_parameter.h
index 345780b4c84..322a1fff564 100644
--- a/src/mongo/db/query/util/memory_util.h
+++ b/src/mongo/db/query/plan_cache_size_parameter.h
@@ -31,36 +31,29 @@
#include <string>
-#include "mongo/base/error_codes.h"
#include "mongo/base/status_with.h"
-namespace mongo::memory_util {
+namespace mongo::plan_cache_util {
/**
- * Defines units of memory.
+ * Defines units of planCacheSize parameter.
*/
-enum class MemoryUnits {
+enum class PlanCacheSizeUnits {
kPercent,
kMB,
kGB,
};
+StatusWith<PlanCacheSizeUnits> parseUnitString(const std::string& strUnit);
+
/**
- * Represents parsed memory size parameter.
+ * Represents parsed planCacheSize parameter.
*/
-struct MemorySize {
- static StatusWith<MemorySize> parse(const std::string& str);
+struct PlanCacheSizeParameter {
+ static StatusWith<PlanCacheSizeParameter> parse(const std::string& str);
const double size;
- const MemoryUnits units;
+ const PlanCacheSizeUnits units;
};
-StatusWith<MemoryUnits> parseUnitString(const std::string& strUnit);
-size_t convertToSizeInBytes(const MemorySize& memSize);
-size_t capMemorySize(size_t requestedSizeBytes,
- size_t maximumSizeGB,
- double percentTotalSystemMemory);
-size_t getRequestedMemSizeInBytes(const MemorySize& memSize);
-
-
-} // namespace mongo::memory_util
+} // namespace mongo::plan_cache_util
diff --git a/src/mongo/db/query/plan_cache_size_parameter_test.cpp b/src/mongo/db/query/plan_cache_size_parameter_test.cpp
new file mode 100644
index 00000000000..1f0fbf76a27
--- /dev/null
+++ b/src/mongo/db/query/plan_cache_size_parameter_test.cpp
@@ -0,0 +1,79 @@
+/**
+ * Copyright (C) 2021-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/db/query/plan_cache_size_parameter.h"
+
+#include "mongo/unittest/unittest.h"
+
+namespace mongo::plan_cache_util {
+
+bool operator==(const PlanCacheSizeParameter& lhs, const PlanCacheSizeParameter& rhs) {
+ constexpr double kEpsilon = 1e-10;
+ return std::abs(lhs.size - rhs.size) < kEpsilon && lhs.units == rhs.units;
+}
+
+TEST(PlanCacheParameterTest, ParseUnitStringPercent) {
+ ASSERT_TRUE(PlanCacheSizeUnits::kPercent == parseUnitString("%"));
+}
+
+TEST(PlanCacheParameterTest, ParseUnitStringMB) {
+ ASSERT_TRUE(PlanCacheSizeUnits::kMB == parseUnitString("MB"));
+ ASSERT_TRUE(PlanCacheSizeUnits::kMB == parseUnitString("mb"));
+ ASSERT_TRUE(PlanCacheSizeUnits::kMB == parseUnitString("mB"));
+ ASSERT_TRUE(PlanCacheSizeUnits::kMB == parseUnitString("Mb"));
+}
+
+TEST(PlanCacheParameterTest, ParseUnitStringGB) {
+ ASSERT_TRUE(PlanCacheSizeUnits::kGB == parseUnitString("GB"));
+ ASSERT_TRUE(PlanCacheSizeUnits::kGB == parseUnitString("gb"));
+ ASSERT_TRUE(PlanCacheSizeUnits::kGB == parseUnitString("gB"));
+ ASSERT_TRUE(PlanCacheSizeUnits::kGB == parseUnitString("Gb"));
+}
+
+TEST(PlanCacheParameterTest, ParseUnitStringIncorrectValue) {
+ ASSERT_NOT_OK(parseUnitString("").getStatus());
+ ASSERT_NOT_OK(parseUnitString(" ").getStatus());
+ ASSERT_NOT_OK(parseUnitString("KB").getStatus());
+}
+
+TEST(PlanCacheParameterTest, ParsePlanCacheSizeParameter) {
+ ASSERT_TRUE((PlanCacheSizeParameter{10.0, PlanCacheSizeUnits::kPercent}) ==
+ PlanCacheSizeParameter::parse("10%"));
+ ASSERT_TRUE((PlanCacheSizeParameter{300.0, PlanCacheSizeUnits::kMB}) ==
+ PlanCacheSizeParameter::parse("300MB"));
+ ASSERT_TRUE((PlanCacheSizeParameter{4.0, PlanCacheSizeUnits::kGB}) ==
+ PlanCacheSizeParameter::parse("4GB"));
+ ASSERT_TRUE((PlanCacheSizeParameter{5.1, PlanCacheSizeUnits::kPercent}) ==
+ PlanCacheSizeParameter::parse(" 5.1%"));
+ ASSERT_TRUE((PlanCacheSizeParameter{11.1, PlanCacheSizeUnits::kMB}) ==
+ PlanCacheSizeParameter::parse("11.1 mb"));
+ ASSERT_TRUE((PlanCacheSizeParameter{12.1, PlanCacheSizeUnits::kGB}) ==
+ PlanCacheSizeParameter::parse(" 12.1 Gb "));
+}
+} // namespace mongo::plan_cache_util
diff --git a/src/mongo/db/query/plan_enumerator.cpp b/src/mongo/db/query/plan_enumerator.cpp
index 9e30afa0049..a11bc9b415a 100644
--- a/src/mongo/db/query/plan_enumerator.cpp
+++ b/src/mongo/db/query/plan_enumerator.cpp
@@ -35,7 +35,6 @@
#include "mongo/db/query/index_tag.h"
#include "mongo/db/query/indexability.h"
-#include "mongo/db/query/query_planner_common.h"
#include "mongo/logv2/log.h"
#include "mongo/util/string_map.h"
@@ -59,8 +58,8 @@ std::string getPathPrefix(std::string path) {
* is a predicate that is required to use an index.
*/
bool expressionRequiresIndex(const MatchExpression* node) {
- return QueryPlannerCommon::countNodes(node, MatchExpression::GEO_NEAR) > 0 ||
- QueryPlannerCommon::countNodes(node, MatchExpression::TEXT) > 0;
+ return CanonicalQuery::countNodes(node, MatchExpression::GEO_NEAR) > 0 ||
+ CanonicalQuery::countNodes(node, MatchExpression::TEXT) > 0;
}
size_t getPathLength(const MatchExpression* expr) {
@@ -338,8 +337,6 @@ string PlanEnumerator::NodeAssignment::toString() const {
str::stream ss;
ss << "ALL OF (lockstep): {";
ss << "\n\ttotalEnumerated: " << lockstepOrAssignment->totalEnumerated;
- ss << "\n\texhaustedLockstepIteration: "
- << lockstepOrAssignment->exhaustedLockstepIteration;
ss << "\n\tsubnodes: [ ";
for (auto&& node : lockstepOrAssignment->subnodes) {
ss << "\n\t\t{";
@@ -1290,8 +1287,6 @@ void PlanEnumerator::getIndexedPreds(MatchExpression* node,
std::vector<MatchExpression*>* indexedPreds) {
if (Indexability::nodeCanUseIndexOnOwnField(node)) {
RelevantTag* rt = static_cast<RelevantTag*>(node->getTag());
- tassert(9074700, "RelevantTag is not assigned to the match expression node", rt != nullptr);
-
if (context.elemMatchExpr) {
// If we're in an $elemMatch context, store the
// innermost parent $elemMatch, as well as the
@@ -1308,7 +1303,7 @@ void PlanEnumerator::getIndexedPreds(MatchExpression* node,
indexedPreds->push_back(node);
} else if (Indexability::isBoundsGeneratingNot(node)) {
getIndexedPreds(node->getChild(0), context, indexedPreds);
- } else if (Indexability::isBoundsGeneratingElemMatchObject(node)) {
+ } else if (MatchExpression::ELEM_MATCH_OBJECT == node->matchType()) {
PrepMemoContext childContext;
childContext.elemMatchExpr = node;
for (size_t i = 0; i < node->numChildren(); ++i) {
@@ -1694,8 +1689,8 @@ bool PlanEnumerator::LockstepOrAssignment::allIdentical() const {
return true;
}
-bool PlanEnumerator::LockstepOrAssignment::shouldResetBeforeProceeding(size_t totalEnumerated,
- size_t orLimit) const {
+bool PlanEnumerator::LockstepOrAssignment::shouldResetBeforeProceeding(
+ size_t totalEnumerated) const {
if (totalEnumerated == 0 || !exhaustedLockstepIteration) {
return false;
}
@@ -1705,12 +1700,7 @@ bool PlanEnumerator::LockstepOrAssignment::shouldResetBeforeProceeding(size_t to
if (!subnode.maxIterCount) {
return false; // Haven't yet looped over this child entirely, not ready yet.
}
- totalPossibleEnumerations *= subnode.maxIterCount.value();
- // If 'totalPossibleEnumerations' reaches the limit, we can just shortcut it. Otherwise,
- // 'totalPossibleEnumerations' could overflow if we have a large $or.
- if (totalPossibleEnumerations >= orLimit) {
- return false;
- }
+ totalPossibleEnumerations *= subnode.maxIterCount.get();
}
// If we're able to compute a total number expected enumerations, we must have already cycled
@@ -1747,7 +1737,7 @@ bool PlanEnumerator::_nextMemoForLockstepOrAssignment(
}
// Edge case: if every child has only one option available, we are already finished
// enumerating.
- if (assignment->shouldResetBeforeProceeding(assignment->totalEnumerated, _orLimit)) {
+ if (assignment->shouldResetBeforeProceeding(assignment->totalEnumerated)) {
assignment->exhaustedLockstepIteration = false;
return true; // We're back at the beginning, no need to reset.
}
@@ -1774,35 +1764,27 @@ bool PlanEnumerator::_nextMemoForLockstepOrAssignment(
}
};
advanceOnce();
- if (assignment->allIdentical()) {
- // All sub-nodes have the same enumeration state, skip this one since we already did it
- // above. This is expected to happen pretty often. For example, if we have two subnodes each
- // enumerating two states, we'd expect the order to be: 00, 11 (these two iterated above),
- // then 00 (skipped here when we fall through after finishing lockstep iteration), then 10,
- // 01, then finally 11 (skipped here).
- //
- // In this example, when we finally roll back to 00, enumeration is complete. We will fall
- // through the code below which is responsible for resetting all enumeration state to the
- // starting point (which need not reset the child nodes in this case because they already
- // all rolled back to the starting point of 00). Finally, we return true to indicate that
- // all possibilities have been enumerated.
+ while (assignment->allIdentical()) {
+ // All sub-nodes have the same enumeration state, skip this one since we already did
+ // it above. This is expected to happen pretty often. For example, if we have two subnodes
+ // each enumerating two states, we'd expect the order to be: 00, 11 (these two iterated
+ // above), then 00 (skipped by falling through above after finishing lockstep iteration),
+ // then 10, 11 (skipped here), 00 (skipped here), then finally 01.
advanceOnce();
}
// This special ordering is tricky to reset. Because it iterates the sub nodes in such a
// unique order, it can be difficult to know when it has actually finished iterating. Our
// strategy is just to compute a total and go back to the beginning once we hit that total.
- if (!assignment->shouldResetBeforeProceeding(assignment->totalEnumerated, _orLimit)) {
+ if (!assignment->shouldResetBeforeProceeding(assignment->totalEnumerated)) {
return false;
}
// Reset!
for (auto&& subnode : assignment->subnodes) {
- if (subnode.iterationCount != 0) {
- while (!nextMemo(subnode.memoId)) {
- // Keep advancing till it rolls over.
- }
- subnode.iterationCount = 0;
+ while (!nextMemo(subnode.memoId)) {
+ // Keep advancing till it rolls over.
}
+ subnode.iterationCount = 0;
}
assignment->exhaustedLockstepIteration = false;
return true;
diff --git a/src/mongo/db/query/plan_enumerator.h b/src/mongo/db/query/plan_enumerator.h
index b82b738c57b..60344f0c9ee 100644
--- a/src/mongo/db/query/plan_enumerator.h
+++ b/src/mongo/db/query/plan_enumerator.h
@@ -244,7 +244,7 @@ private:
* Returns true if 'totalEnumerated' matches the total number of expected plans for this
* assignment.
*/
- bool shouldResetBeforeProceeding(size_t totalEnumerated, size_t orLimit) const;
+ bool shouldResetBeforeProceeding(size_t totalEnumerated) const;
/**
* Returns true if each sub node is at the same iterationCount.
diff --git a/src/mongo/db/query/plan_executor.cpp b/src/mongo/db/query/plan_executor.cpp
index 99b2fd8fefa..ee41d15d84c 100644
--- a/src/mongo/db/query/plan_executor.cpp
+++ b/src/mongo/db/query/plan_executor.cpp
@@ -38,10 +38,6 @@ namespace {
MONGO_FAIL_POINT_DEFINE(planExecutorAlwaysFails);
} // namespace
-const OperationContext::Decoration<boost::optional<SharedSemiFuture<void>>>
- planExecutorShardingCriticalSectionFuture =
- OperationContext::declareDecoration<boost::optional<SharedSemiFuture<void>>>();
-
std::string PlanExecutor::stateToStr(ExecState execState) {
switch (execState) {
case PlanExecutor::ADVANCED:
diff --git a/src/mongo/db/query/plan_executor.h b/src/mongo/db/query/plan_executor.h
index 30ba3d69c63..33fbd075b93 100644
--- a/src/mongo/db/query/plan_executor.h
+++ b/src/mongo/db/query/plan_executor.h
@@ -52,21 +52,8 @@ class RecordId;
* 'clientsLastKnownCommittedOpTime' represents the time passed to the getMore command.
* If the replication coordinator ever reports a higher committed op time, we should stop waiting
* for inserts and return immediately to speed up the propagation of commit level changes.
- *
- * A boost::none value opts out of the commit point propagation. A null optime compares less than
- * any non-null optimes and thus will always trigger an empty batch for commit point propagation.
- */
-extern const OperationContext::Decoration<boost::optional<repl::OpTime>>
- clientsLastKnownCommittedOpTime;
-
-/**
- * If a plan yielded because it encountered a sharding critical section,
- * 'planExecutorShardingCriticalSectionFuture' will be set to a future that becomes ready when the
- * critical section ends. This future can be waited on to hold off resuming the plan execution while
- * the critical section is still active.
*/
-extern const OperationContext::Decoration<boost::optional<SharedSemiFuture<void>>>
- planExecutorShardingCriticalSectionFuture;
+extern const OperationContext::Decoration<repl::OpTime> clientsLastKnownCommittedOpTime;
/**
* A PlanExecutor is the abstraction that knows how to crank a tree of stages into execution.
diff --git a/src/mongo/db/query/plan_executor_factory.cpp b/src/mongo/db/query/plan_executor_factory.cpp
index 386bf38a538..0b66c3dafce 100644
--- a/src/mongo/db/query/plan_executor_factory.cpp
+++ b/src/mongo/db/query/plan_executor_factory.cpp
@@ -31,8 +31,6 @@
#include "mongo/platform/basic.h"
-#include <iostream>
-
#include "mongo/db/query/plan_executor_factory.h"
#include "mongo/db/exec/plan_stage.h"
@@ -42,7 +40,6 @@
#include "mongo/db/query/query_planner_params.h"
#include "mongo/db/query/util/make_data_structure.h"
#include "mongo/logv2/log.h"
-#include "mongo/util/duration.h"
namespace mongo::plan_executor_factory {
@@ -68,7 +65,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make(
yieldPolicy);
}
-
StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make(
const boost::intrusive_ptr<ExpressionContext>& expCtx,
std::unique_ptr<WorkingSet> ws,
@@ -78,7 +74,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make(
size_t plannerOptions,
NamespaceString nss,
std::unique_ptr<QuerySolution> qs) {
-
return make(expCtx->opCtx,
std::move(ws),
std::move(rt),
@@ -103,7 +98,6 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make(
NamespaceString nss,
PlanYieldPolicy::YieldPolicy yieldPolicy) {
dassert(collection);
-
try {
auto execImpl = new PlanExecutorImpl(opCtx,
std::move(ws),
@@ -134,6 +128,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make(
NamespaceString nss,
std::unique_ptr<PlanYieldPolicySBE> yieldPolicy) {
auto&& [rootStage, data] = root;
+
LOGV2_DEBUG(4822860,
5,
"SBE plan",
@@ -162,6 +157,7 @@ StatusWith<std::unique_ptr<PlanExecutor, PlanExecutor::Deleter>> make(
size_t plannerOptions,
NamespaceString nss,
std::unique_ptr<PlanYieldPolicySBE> yieldPolicy) {
+
LOGV2_DEBUG(4822861,
5,
"SBE plan",
diff --git a/src/mongo/db/query/plan_executor_factory.h b/src/mongo/db/query/plan_executor_factory.h
index df7184583ec..bf41f169af9 100644
--- a/src/mongo/db/query/plan_executor_factory.h
+++ b/src/mongo/db/query/plan_executor_factory.h
@@ -29,7 +29,6 @@
#pragma once
-#include "mongo/util/duration.h"
#include <queue>
#include "mongo/db/exec/sbe/stages/stages.h"
diff --git a/src/mongo/db/query/plan_executor_impl.cpp b/src/mongo/db/query/plan_executor_impl.cpp
index ae40ecd070e..6691a52fb8a 100644
--- a/src/mongo/db/query/plan_executor_impl.cpp
+++ b/src/mongo/db/query/plan_executor_impl.cpp
@@ -33,12 +33,10 @@
#include "mongo/db/query/plan_executor_impl.h"
-#include "mongo/util/duration.h"
#include <memory>
#include "mongo/bson/simple_bsonobj_comparator.h"
#include "mongo/db/catalog/collection.h"
-#include "mongo/db/concurrency/exception_util.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/exec/cached_plan.h"
@@ -63,7 +61,6 @@
#include "mongo/db/query/plan_yield_policy_impl.h"
#include "mongo/db/query/yield_policy_callbacks_impl.h"
#include "mongo/db/repl/replication_coordinator.h"
-#include "mongo/db/s/operation_sharding_state.h"
#include "mongo/db/service_context.h"
#include "mongo/logv2/log.h"
#include "mongo/util/fail_point.h"
@@ -77,8 +74,8 @@ using std::string;
using std::unique_ptr;
using std::vector;
-const OperationContext::Decoration<boost::optional<repl::OpTime>> clientsLastKnownCommittedOpTime =
- OperationContext::declareDecoration<boost::optional<repl::OpTime>>();
+const OperationContext::Decoration<repl::OpTime> clientsLastKnownCommittedOpTime =
+ OperationContext::declareDecoration<repl::OpTime>();
// This failpoint is also accessed by the SBE executor so we define it outside of an anonymous
// namespace.
@@ -103,11 +100,11 @@ std::unique_ptr<PlanYieldPolicy> makeYieldPolicy(PlanExecutorImpl* exec,
}
case PlanYieldPolicy::YieldPolicy::ALWAYS_TIME_OUT: {
return std::make_unique<AlwaysTimeOutYieldPolicy>(
- exec->getOpCtx(), exec->getOpCtx()->getServiceContext()->getFastClockSource());
+ exec->getOpCtx()->getServiceContext()->getFastClockSource());
}
case PlanYieldPolicy::YieldPolicy::ALWAYS_MARK_KILLED: {
return std::make_unique<AlwaysPlanKilledYieldPolicy>(
- exec->getOpCtx(), exec->getOpCtx()->getServiceContext()->getFastClockSource());
+ exec->getOpCtx()->getServiceContext()->getFastClockSource());
}
default:
MONGO_UNREACHABLE;
@@ -137,6 +134,13 @@ PlanExecutorImpl::PlanExecutorImpl(OperationContext* opCtx,
invariant(!_expCtx || _expCtx->opCtx == _opCtx);
invariant(!_cq || !_expCtx || _cq->getExpCtx() == _expCtx);
+ // If this PlanExecutor is executing a COLLSCAN, keep a pointer directly to the COLLSCAN
+ // stage. This is used for change streams in order to keep the the latest oplog timestamp
+ // and post batch resume token up to date as the oplog scan progresses.
+ if (auto collectionScan = getStageByType(_root.get(), STAGE_COLLSCAN)) {
+ _collScanStage = static_cast<CollectionScan*>(collectionScan);
+ }
+
// If we don't yet have a namespace string, then initialize it from either 'collection' or
// '_cq'.
if (_nss.isEmpty()) {
@@ -168,13 +172,6 @@ PlanExecutorImpl::PlanExecutorImpl(OperationContext* opCtx,
_planExplainer->updateEnumeratorExplainInfo(
subplanStage->compositeSolution()->_enumeratorExplainInfo);
}
-
- // If this PlanExecutor is executing a COLLSCAN, keep a pointer directly to the COLLSCAN
- // stage. This is used for change streams in order to keep the the latest oplog timestamp
- // and post batch resume token up to date as the oplog scan progresses.
- if (auto collectionScan = getStageByType(_root.get(), STAGE_COLLSCAN)) {
- _collScanStage = static_cast<CollectionScan*>(collectionScan);
- }
}
Status PlanExecutorImpl::_pickBestPlan() {
@@ -362,25 +359,8 @@ PlanExecutor::ExecState PlanExecutorImpl::_getNextImpl(Snapshotted<Document>* ob
// 2) some stage requested a yield, or
// 3) we need to yield and retry due to a WriteConflictException.
// In all cases, the actual yielding happens here.
-
- const auto whileYieldingFn = [&]() {
- // If we yielded because we encountered a sharding critical section, wait for the
- // critical section to end before continuing. By waiting for the critical section to be
- // exited we avoid busy spinning immediately and encountering the same critical section
- // again. It is important that this wait happens after having released the lock
- // hierarchy -- otherwise deadlocks could happen, or the very least, locks would be
- // unnecessarily held while waiting.
- const auto& shardingCriticalSection = planExecutorShardingCriticalSectionFuture(_opCtx);
- if (shardingCriticalSection) {
- OperationShardingState::waitForCriticalSectionToComplete(_opCtx,
- *shardingCriticalSection)
- .ignore();
- planExecutorShardingCriticalSectionFuture(_opCtx).reset();
- }
- };
-
if (_yieldPolicy->shouldYieldOrInterrupt(_opCtx)) {
- uassertStatusOK(_yieldPolicy->yieldOrInterrupt(_opCtx, whileYieldingFn));
+ uassertStatusOK(_yieldPolicy->yieldOrInterrupt(_opCtx));
}
WorkingSetID id = WorkingSet::INVALID_ID;
@@ -447,7 +427,8 @@ PlanExecutor::ExecState PlanExecutorImpl::_getNextImpl(Snapshotted<Document>* ob
CurOp::get(_opCtx)->debug().additiveMetrics.incrementWriteConflicts(1);
writeConflictsInARow++;
- logWriteConflictAndBackoff(writeConflictsInARow, "plan execution", _nss.ns());
+ WriteConflictException::logAndBackoff(
+ writeConflictsInARow, "plan execution", _nss.ns());
// If we're allowed to, we will yield next time through the loop.
if (_yieldPolicy->canAutoYield()) {
diff --git a/src/mongo/db/query/plan_executor_impl.h b/src/mongo/db/query/plan_executor_impl.h
index ac593cd710c..672cd75f243 100644
--- a/src/mongo/db/query/plan_executor_impl.h
+++ b/src/mongo/db/query/plan_executor_impl.h
@@ -29,7 +29,6 @@
#pragma once
-#include "mongo/util/duration.h"
#include <boost/optional.hpp>
#include <queue>
diff --git a/src/mongo/db/query/plan_executor_sbe.cpp b/src/mongo/db/query/plan_executor_sbe.cpp
index e1909acf95c..c8d516cb718 100644
--- a/src/mongo/db/query/plan_executor_sbe.cpp
+++ b/src/mongo/db/query/plan_executor_sbe.cpp
@@ -41,7 +41,6 @@
#include "mongo/db/query/sbe_stage_builder.h"
#include "mongo/logv2/log.h"
#include "mongo/s/resharding/resume_token_gen.h"
-#include "mongo/util/duration.h"
namespace mongo {
// This failpoint is defined by the classic executor but is also accessed here.
diff --git a/src/mongo/db/query/plan_executor_sbe.h b/src/mongo/db/query/plan_executor_sbe.h
index c11d97392ee..547f0fb0a07 100644
--- a/src/mongo/db/query/plan_executor_sbe.h
+++ b/src/mongo/db/query/plan_executor_sbe.h
@@ -29,7 +29,6 @@
#pragma once
-#include "mongo/util/duration.h"
#include <queue>
#include "mongo/db/exec/sbe/stages/stages.h"
diff --git a/src/mongo/db/query/plan_explainer.h b/src/mongo/db/query/plan_explainer.h
index ffaabecc43a..b53a10e5655 100644
--- a/src/mongo/db/query/plan_explainer.h
+++ b/src/mongo/db/query/plan_explainer.h
@@ -29,14 +29,12 @@
#pragma once
-#include "mongo/bson/bsonobj.h"
#include "mongo/db/exec/plan_stats.h"
#include "mongo/db/query/classic_plan_cache.h"
#include "mongo/db/query/explain_options.h"
#include "mongo/db/query/plan_enumerator_explain_info.h"
#include "mongo/db/query/plan_summary_stats.h"
#include "mongo/db/query/query_solution.h"
-#include "mongo/util/duration.h"
namespace mongo {
/**
diff --git a/src/mongo/db/query/plan_explainer_factory.cpp b/src/mongo/db/query/plan_explainer_factory.cpp
index 9307ec349f6..544ab33fdd2 100644
--- a/src/mongo/db/query/plan_explainer_factory.cpp
+++ b/src/mongo/db/query/plan_explainer_factory.cpp
@@ -34,8 +34,6 @@
#include "mongo/db/exec/plan_cache_util.h"
#include "mongo/db/query/plan_explainer_impl.h"
#include "mongo/db/query/plan_explainer_sbe.h"
-#include "mongo/util/duration.h"
-#include <ratio>
namespace mongo::plan_explainer_factory {
std::unique_ptr<PlanExplainer> make(PlanStage* root) {
diff --git a/src/mongo/db/query/plan_explainer_factory.h b/src/mongo/db/query/plan_explainer_factory.h
index 0064df7cb39..5e24a755747 100644
--- a/src/mongo/db/query/plan_explainer_factory.h
+++ b/src/mongo/db/query/plan_explainer_factory.h
@@ -36,7 +36,6 @@
#include "mongo/db/query/plan_explainer.h"
#include "mongo/db/query/query_solution.h"
#include "mongo/db/query/sbe_plan_ranker.h"
-#include "mongo/util/duration.h"
namespace mongo::plan_explainer_factory {
std::unique_ptr<PlanExplainer> make(PlanStage* root);
diff --git a/src/mongo/db/query/plan_explainer_impl.cpp b/src/mongo/db/query/plan_explainer_impl.cpp
index a67c9e98a60..1e34cf73ebe 100644
--- a/src/mongo/db/query/plan_explainer_impl.cpp
+++ b/src/mongo/db/query/plan_explainer_impl.cpp
@@ -655,6 +655,7 @@ boost::optional<double> getWinningPlanScore(PlanStage* root) {
void PlanExplainerImpl::getSummaryStats(PlanSummaryStats* statsOut) const {
invariant(statsOut);
+
// We can get some of the fields we need from the common stats stored in the
// root stage of the plan tree.
const CommonStats* common = _root->getCommonStats();
@@ -667,9 +668,6 @@ void PlanExplainerImpl::getSummaryStats(PlanSummaryStats* statsOut) const {
statsOut->totalKeysExamined = 0;
statsOut->totalDocsExamined = 0;
- statsOut->indexesUsed.clear();
- statsOut->collectionScans = 0;
- statsOut->collectionScansNonTailable = 0;
for (size_t i = 0; i < stages.size(); i++) {
statsOut->totalKeysExamined +=
diff --git a/src/mongo/db/query/plan_explainer_impl.h b/src/mongo/db/query/plan_explainer_impl.h
index f73a2ec6250..73ef81ae825 100644
--- a/src/mongo/db/query/plan_explainer_impl.h
+++ b/src/mongo/db/query/plan_explainer_impl.h
@@ -29,12 +29,10 @@
#pragma once
-#include "mongo/bson/bsonobj.h"
#include "mongo/db/exec/plan_stage.h"
#include "mongo/db/query/plan_enumerator_explain_info.h"
#include "mongo/db/query/plan_explainer.h"
#include "mongo/db/query/query_solution.h"
-#include "mongo/util/duration.h"
namespace mongo {
/**
@@ -49,6 +47,7 @@ public:
PlanExplainerImpl(PlanStage* root, const PlanEnumeratorExplainInfo& explainInfo)
: PlanExplainer{explainInfo}, _root{root} {}
PlanExplainerImpl(PlanStage* root) : _root{root} {}
+
const ExplainVersion& getVersion() const final;
bool isMultiPlan() const final;
std::string getPlanSummary() const final;
diff --git a/src/mongo/db/query/plan_explainer_sbe.cpp b/src/mongo/db/query/plan_explainer_sbe.cpp
index 25ef5435572..2f8f5b43b08 100644
--- a/src/mongo/db/query/plan_explainer_sbe.cpp
+++ b/src/mongo/db/query/plan_explainer_sbe.cpp
@@ -371,10 +371,9 @@ void PlanExplainerSBE::getSummaryStats(PlanSummaryStats* statsOut) const {
// Use the pre-computed summary stats instead of traversing the QuerySolution tree.
const auto& indexesUsed = _debugInfo->mainStats.indexesUsed;
- statsOut->indexesUsed.clear();
statsOut->indexesUsed.insert(indexesUsed.begin(), indexesUsed.end());
- statsOut->collectionScans = _debugInfo->mainStats.collectionScans;
- statsOut->collectionScansNonTailable = _debugInfo->mainStats.collectionScansNonTailable;
+ statsOut->collectionScans += _debugInfo->mainStats.collectionScans;
+ statsOut->collectionScansNonTailable += _debugInfo->mainStats.collectionScansNonTailable;
}
void PlanExplainerSBE::getSecondarySummaryStats(std::string secondaryColl,
diff --git a/src/mongo/db/query/plan_explainer_sbe.h b/src/mongo/db/query/plan_explainer_sbe.h
index a256e1371c6..5dc97f90641 100644
--- a/src/mongo/db/query/plan_explainer_sbe.h
+++ b/src/mongo/db/query/plan_explainer_sbe.h
@@ -35,7 +35,6 @@
#include "mongo/db/query/plan_explainer.h"
#include "mongo/db/query/query_solution.h"
#include "mongo/db/query/sbe_plan_ranker.h"
-#include "mongo/util/duration.h"
namespace mongo {
/**
diff --git a/src/mongo/db/query/plan_insert_listener.cpp b/src/mongo/db/query/plan_insert_listener.cpp
index 1b7270eef76..0d86c76c9e7 100644
--- a/src/mongo/db/query/plan_insert_listener.cpp
+++ b/src/mongo/db/query/plan_insert_listener.cpp
@@ -65,10 +65,9 @@ bool shouldWaitForInserts(OperationContext* opCtx,
// coordinator's lastCommittedOpTime has progressed past the client's lastCommittedOpTime.
// In that case, we will return early so that we can inform the client of the new
// lastCommittedOpTime immediately.
- if (clientsLastKnownCommittedOpTime(opCtx)) {
+ if (!clientsLastKnownCommittedOpTime(opCtx).isNull()) {
auto replCoord = repl::ReplicationCoordinator::get(opCtx);
- return clientsLastKnownCommittedOpTime(opCtx).value() >=
- replCoord->getLastCommittedOpTime();
+ return clientsLastKnownCommittedOpTime(opCtx) >= replCoord->getLastCommittedOpTime();
}
return true;
}
diff --git a/src/mongo/db/query/plan_ranker.cpp b/src/mongo/db/query/plan_ranker.cpp
index 42fb5645ff8..87f189e89f7 100644
--- a/src/mongo/db/query/plan_ranker.cpp
+++ b/src/mongo/db/query/plan_ranker.cpp
@@ -93,21 +93,6 @@ void logFailedPlan(std::function<std::string()> planSummary) {
LOGV2_DEBUG(
20960, 2, "Not scoring a plan because the plan failed", "planSummary"_attr = planSummary());
}
-
-void logTieBreaking(double score,
- double docsExaminedBonus,
- double indexPrefixBonus,
- bool isPlanTied) {
- LOGV2_DEBUG(8027500, 2, "Tie breaking heuristics", "formula"_attr = [&]() {
- StringBuilder sb;
- sb << "isPlanTied: " << isPlanTied << ". finalScore("
- << str::convertDoubleToString(score + docsExaminedBonus + indexPrefixBonus)
- << ") = score(" << str::convertDoubleToString(score) << ") + docsExaminedBonus("
- << str::convertDoubleToString(docsExaminedBonus) << ") + indexPrefixBonus("
- << str::convertDoubleToString(indexPrefixBonus) << ")";
- return sb.str();
- }());
-}
} // namespace log_detail
namespace {
@@ -153,296 +138,9 @@ protected:
return false;
}
};
-
-/**
- * Return true if the nodes have the same type and the same number of children.
- */
-bool areNodesCompatible(const std::vector<const QuerySolutionNode*>& nodes) {
- for (size_t i = 1; i < nodes.size(); ++i) {
- if (nodes[i - 1]->getType() != nodes[i]->getType()) {
- return false;
- }
-
- if (nodes[i - 1]->children.size() != nodes[i]->children.size()) {
- return false;
- }
- }
-
- return true;
-}
-
-/**
- * Returns true if the value can serve as a type lower bound for the purposes of type bracketing.
- * The function is designed to work with the 'interesting' for index prefix heuristic types only:
- * Number, String, Date, Timestamp, Boolean, Object, Array, ObjectId. For other types it may return
- * false positive results. The code of the function is based on index bounds build logic from
- * 'index_bounds_builder.cpp'.
- */
-bool isLowerBound(const BSONElement& value, bool isInclusive) {
- switch (value.type()) {
- case NumberInt:
- case NumberDouble:
- case NumberLong:
- case NumberDecimal:
- // Lower bound value for numbers.
- return (std::isinf(value.numberDouble()) || std::isnan(value.numberDouble())) &&
- isInclusive == true;
- case String:
- // Lower bound value for strings.
- return value.str().empty() && isInclusive == true;
- case Date:
- // Lower bound value for dates.
- return value.date() == Date_t::min() && isInclusive == true;
- case bsonTimestamp:
- // Lower bound value for timestamps.
- return value.timestamp() == Timestamp::min() && isInclusive == true;
- case jstOID:
- // Lower bound value for ObjectID.
- return value.OID() == OID() && isInclusive == true;
- case Object:
- case Array:
- // Lower bound value for Object and Array.
- return value.Obj().isEmpty() && isInclusive == true;
- case BinData:
- case EOO:
- case MinKey:
- case MaxKey:
- case Bool: // Boolean bounds are considered always open since they are non-selective.
- case jstNULL:
- case Undefined:
- case Symbol:
- case RegEx:
- case DBRef:
- case Code:
- case CodeWScope:
- return true;
- }
-
- MONGO_UNREACHABLE_TASSERT(8102100);
-}
-
-/**
- * Returns true if the value can serve as a type upper bound for the purposes of type bracketing.
- * The function is designed to work with the 'interesting' for index prefix heuristic types only:
- * Number, String, Date, Timestamp, Boolean, Object, Array, ObjectId. For other types it may return
- * false positive results. The code of the function is based on index bounds build logic from
- * 'index_bounds_builder.cpp'.
- */
-bool isUpperBound(const BSONElement& value, bool isInclusive) {
- switch (value.type()) {
- case NumberInt:
- case NumberDouble:
- case NumberLong:
- case NumberDecimal:
- // Upper bound value for numbers.
- return std::isinf(value.numberDouble()) && isInclusive == true;
- case String:
- // A string value cannot be an upper bound value.
- return false;
- case Date:
- // Upper bound value for Date.
- return value.date() == Date_t::max() && isInclusive == true;
- case bsonTimestamp:
- // Upper bound value for Timestamp.
- return value.timestamp() == Timestamp::max() && isInclusive == true;
- case jstOID:
- // Upper bound value for ObjectID.
- return value.OID() == OID::max() && isInclusive == true;
- case Object:
- // Upper bound value for String.
- return value.Obj().isEmpty() && isInclusive == false;
- case Array:
- // Upper bound value for Object.
- return value.Obj().isEmpty() && isInclusive == false;
- case BinData:
- // Upper bound value for Array.
- return value.valuesize() == 0 && isInclusive == false;
- case EOO:
- case MinKey:
- case MaxKey:
- case Bool: // Boolean bounds are considered always open since they are non-selective.
- case jstNULL:
- case Undefined:
- case Symbol:
- case RegEx:
- case DBRef:
- case Code:
- case CodeWScope:
- return true;
- }
-
- MONGO_UNREACHABLE_TASSERT(8102101);
-}
-
-/**
- * The function tries to detect if the interval is closed on both ends. Can return false
- * positive results for the types not mentioned in the comment to 'isMinMaxValue' function.
- */
-bool isClosedInterval(const Interval& interval) {
- // If the bound types are different the interval is considered to be open.
- if (interval.start.type() != interval.end.type()) {
- return false;
- }
-
- switch (interval.getDirection()) {
- // Point intervals, empty intervals, and null intervals have no direction.
- case Interval::Direction::kDirectionNone:
- return true;
- case Interval::Direction::kDirectionAscending:
- return !isLowerBound(interval.start, interval.startInclusive) &&
- !isUpperBound(interval.end, interval.endInclusive);
- case Interval::Direction::kDirectionDescending:
- return !isUpperBound(interval.start, interval.startInclusive) &&
- !isLowerBound(interval.end, interval.endInclusive);
- }
-
- MONGO_UNREACHABLE_TASSERT(8102102);
-}
-
-/**
- * Returns true if this OIL contains only closed intervals.
- */
-bool containsOnlyClosedIntervals(const OrderedIntervalList& oil) {
- for (const auto& interval : oil.intervals) {
- if (!isClosedInterval(interval)) {
- return false;
- }
- }
-
- return true;
-}
-
-/**
- * Calculates score for the given index bounds. The score reflects the following rules:
- * - IndexBounds that has longest single point interval prefix wins,
- * - if winner is not defined on the previous step then IndexBounds with the longest point
- * interval prefix wins,
- * - if winner is not defined on the previous step then IndexBounds with the longest closed
- * interval prefix wins,
- * - if winner is not defined, then IndexBounds with longest interval prefix wins
- * - if winner is not defined, them IndexBounds with shortest index key pattern wins.
- */
-uint64_t getIndexBoundsScore(const IndexBounds& bounds) {
- const uint64_t indexKeyLength = static_cast<uint64_t>(bounds.fields.size());
- uint64_t singlePointIntervalPrefix = 0;
- uint64_t pointsIntervalPrefix = 0;
- uint64_t closedIntervalPrefix = 0;
- uint64_t intervalLength = 0;
-
- for (const auto& field : bounds.fields) {
- // Skip the $** index virtual field, as it's not part of the actual index key.
- if (field.name == "$_path") {
- continue;
- }
-
- // Stop scoring index bounds as soon as we see an all-values interval.
- if (field.isMinToMax() || field.isMaxToMin()) {
- break;
- }
-
- if (intervalLength == singlePointIntervalPrefix && field.isPoint()) {
- ++singlePointIntervalPrefix;
- }
-
- if (intervalLength == pointsIntervalPrefix && field.containsOnlyPointIntervals()) {
- ++pointsIntervalPrefix;
- }
-
- if (intervalLength == closedIntervalPrefix && containsOnlyClosedIntervals(field)) {
- ++closedIntervalPrefix;
- }
-
- ++intervalLength;
- }
-
- // We pack calculated stats into one value to make their comparison simplier. For every
- // prefix length we allocate 12 bits (4096 values) which is more then enough since an index
- // can have no more than 32 fields (see "MongoDB Limits and Thresholds" reference).
- // 'indexKeyLength' is treated differently because, unlike others, we prefer shorter index
- // key prefix length (see the comment to the function for details).
- uint64_t result = (singlePointIntervalPrefix << 52) | (pointsIntervalPrefix << 40) |
- (closedIntervalPrefix << 28) | (intervalLength << 16) |
- (std::numeric_limits<uint16_t>::max() - indexKeyLength);
-
- return result;
-}
-
-/**
- * Calculates scores for the given IndexBounds and add 1 to every winner's resultScores. i-th
- * position in resultScores corresponds to i-th field in IndexBound.
- */
-void scoreIndexBounds(const std::vector<const IndexBounds*>& bounds,
- std::vector<size_t>& resultScores) {
- const size_t nfields = bounds.size();
-
- std::vector<uint64_t> scores{};
- scores.reserve(nfields);
- for (size_t i = 0; i < bounds.size(); ++i) {
- scores.emplace_back(getIndexBoundsScore(*bounds[i]));
- }
-
- auto topScore = max_element(scores.begin(), scores.end());
- for (size_t i = 0; i < nfields; ++i) {
- if (*topScore == scores[i]) {
- resultScores[i] += 1;
- }
- }
-}
} // namespace
std::unique_ptr<PlanScorer<PlanStageStats>> makePlanScorer() {
return std::make_unique<DefaultPlanScorer>();
}
-
-std::vector<size_t> applyIndexPrefixHeuristic(const std::vector<const QuerySolution*>& solutions) {
- std::vector<size_t> solutionScores(solutions.size(), 0);
-
- std::vector<std::vector<const QuerySolutionNode*>> stack{};
- stack.emplace_back();
- stack.back().reserve(solutions.size());
- for (auto solution : solutions) {
- stack.back().emplace_back(solution->root());
- }
-
- while (!stack.empty()) {
- auto top = std::move(stack.back());
- stack.pop_back();
-
- if (!areNodesCompatible(top)) {
- return {};
- }
-
- // Compatible nodes have the same number of children, see comment to 'areNodesCompatible'
- // function.
- for (size_t childIndex = 0; childIndex < top.front()->children.size(); ++childIndex) {
- stack.emplace_back();
- stack.back().reserve(solutions.size());
- for (auto node : top) {
- stack.back().emplace_back(node->children[childIndex]);
- }
- }
-
- if (top.front()->getType() == STAGE_IXSCAN) {
- std::vector<const IndexBounds*> bounds{};
- bounds.reserve(solutions.size());
-
- for (auto node : top) {
- bounds.emplace_back(&static_cast<const IndexScanNode*>(node)->bounds);
- }
-
- scoreIndexBounds(bounds, solutionScores);
- }
- }
-
- std::vector<size_t> winningSolutionIndices{};
- winningSolutionIndices.reserve(solutions.size());
- const auto topScore = max_element(solutionScores.begin(), solutionScores.end());
- for (size_t index = 0; index < solutionScores.size(); ++index) {
- if (solutionScores[index] == *topScore) {
- winningSolutionIndices.emplace_back(index);
- }
- }
-
- return winningSolutionIndices;
-}
} // namespace mongo::plan_ranker
diff --git a/src/mongo/db/query/plan_ranker.h b/src/mongo/db/query/plan_ranker.h
index c9c7baf46b1..8a6a2c18ad9 100644
--- a/src/mongo/db/query/plan_ranker.h
+++ b/src/mongo/db/query/plan_ranker.h
@@ -65,15 +65,8 @@ void logScoringPlan(std::function<std::string()> solution,
void logScore(double score);
void logEOFBonus(double eofBonus);
void logFailedPlan(std::function<std::string()> planSummary);
-void logTieBreaking(double score,
- double docsFetchedBonus,
- double indexPrefixBonus,
- bool isPlanTied);
} // namespace log_detail
-// Constant used for tie breakers.
-const double kBonusEpsilon = 1e-4;
-
/**
* Assigns the stats tree a 'goodness' score. The higher the score, the better the plan. The exact
* value isn't meaningful except for imposing a ranking.
@@ -96,7 +89,7 @@ public:
const auto productivity = calculateProductivity(stats);
const auto advances = getNumberOfAdvances(stats);
const double epsilon =
- std::min(1.0 / static_cast<double>(10 * (advances > 0 ? advances : 1)), kBonusEpsilon);
+ std::min(1.0 / static_cast<double>(10 * (advances > 0 ? advances : 1)), 1e-4);
// We prefer queries that don't require a fetch stage.
@@ -206,11 +199,4 @@ struct BaseCandidatePlan {
};
using CandidatePlan = BaseCandidatePlan<PlanStage*, WorkingSetID, WorkingSet*>;
-
-/**
- * Apply index prefix heuristic (see comment to 'getIndexBoundsScore' function in the cpp file) for
- * the given list of solutions, if the solutions are compatible (have the same plan shape), the
- * vector of winner indexes are returned, otherwise an empty vector is returned.
- */
-std::vector<size_t> applyIndexPrefixHeuristic(const std::vector<const QuerySolution*>& solutions);
} // namespace mongo::plan_ranker
diff --git a/src/mongo/db/query/plan_ranker_index_prefix_test.cpp b/src/mongo/db/query/plan_ranker_index_prefix_test.cpp
deleted file mode 100644
index ae3a5455863..00000000000
--- a/src/mongo/db/query/plan_ranker_index_prefix_test.cpp
+++ /dev/null
@@ -1,422 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-/**
- * This file contains tests for mongo/db/query/plan_ranker.h
- */
-
-#include "mongo/db/query/plan_ranker.h"
-
-#include "mongo/unittest/unittest.h"
-
-namespace mongo::plan_ranker {
-namespace {
-IndexEntry buildSimpleIndexEntry(const BSONObj& kp) {
- return {kp,
- IndexNames::nameToType(IndexNames::findPluginName(kp)),
- IndexDescriptor::kLatestIndexVersion,
- false,
- {},
- {},
- false,
- false,
- CoreIndexInfo::Identifier("test_foo"),
- nullptr,
- {},
- nullptr,
- nullptr};
-}
-
-OrderedIntervalList makeOIL(const std::string& fieldName,
- std::initializer_list<BSONObj> intervals) {
- OrderedIntervalList oil(fieldName);
- for (auto&& interval : intervals) {
- oil.intervals.emplace_back(std::move(interval), true, true);
- }
- return oil;
-}
-
-std::unique_ptr<QuerySolution> makeSolution(std::unique_ptr<QuerySolutionNode> child) {
- auto solution = std::make_unique<QuerySolution>();
- solution->setRoot(std::make_unique<FetchNode>(std::move(child)));
- return solution;
-}
-
-std::unique_ptr<IndexScanNode> makeIndexScan(
- BSONObj indexKey, std::initializer_list<OrderedIntervalList> indexBoundsFields) {
- IndexBounds bounds{};
- for (auto&& field : indexBoundsFields) {
- bounds.fields.emplace_back(std::move(field));
- }
-
- auto indexScan = std::make_unique<IndexScanNode>(buildSimpleIndexEntry(indexKey));
- indexScan->bounds = std::move(bounds);
- indexScan->computeProperties();
-
- return indexScan;
-}
-} // namespace
-
-TEST(IndexPrefixHeuristicTest, SingleIndexScan_LongerSinglePointPrefixWins) {
- // a: [[1, 1]]; b: [[1, 3]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 1 << "" << 1)}),
- makeOIL("b", {BSON("" << 1 << "" << 3)}),
- });
- auto solution1 = makeSolution(std::move(indexScan1));
-
- // c: [[1, 1]]; d: [[1, 1]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1 << "d" << 1),
- {
- makeOIL("c", {BSON("" << 1 << "" << 1)}),
- makeOIL("d", {BSON("" << 1 << "" << 1)}),
- });
- auto solution2 = makeSolution(std::move(indexScan2));
-
- // e: [[1, 1]]; f: [[2, 2], [3, 3]]; g: [[4, 4]]
- auto indexScan3 =
- makeIndexScan(BSON("e" << 1 << "f" << 1 << "g" << 1),
- {
- makeOIL("e", {BSON("" << 1 << "" << 1)}),
- makeOIL("f", {BSON("" << 2 << "" << 2), BSON("" << 3 << "" << 3)}),
- makeOIL("g", {BSON("" << 4 << "" << 4)}),
- });
- auto solution3 = makeSolution(std::move(indexScan3));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get(), solution3.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(1, winners.size());
- ASSERT_EQ(1, winners[0]);
-}
-
-TEST(IndexPrefixHeuristicTest, SingleIndexScan_LongerPointPrefixWins) {
- // a: [[1, 1], [2, 2]]; b: [[1, 3]]
- auto indexScan1 =
- makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 1 << "" << 1), BSON("" << 2 << "" << 2)}),
- makeOIL("b", {BSON("" << 1 << "" << 3)}),
- });
- auto solution1 = makeSolution(std::move(indexScan1));
-
- // c: [[1, 1], [2, 2]]; d: [[1, 1], [3, 3]], e: [[1, 1]]
- auto indexScan2 =
- makeIndexScan(BSON("c" << 1 << "d" << 1 << "e" << 1),
- {
- makeOIL("c", {BSON("" << 1 << "" << 1), BSON("" << 2 << "" << 2)}),
- makeOIL("d", {BSON("" << 1 << "" << 1), BSON("" << 3 << "" << 3)}),
- makeOIL("e", {BSON("" << 1 << "" << 1)}),
- });
- auto solution2 = makeSolution(std::move(indexScan2));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(1, winners.size());
- ASSERT_EQ(1, winners[0]);
-}
-
-TEST(IndexPrefixHeuristicTest, SingleIndexScan_ClosedIntervalPrefixWins) {
- // a: [[1, 1], [2, 2]]; b: [[1, MaxKey]]
- auto indexScan1 =
- makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 1 << "" << 1), BSON("" << 2 << "" << 2)}),
- makeOIL("b", {BSON("" << 1 << "" << MAXKEY)}),
- });
- auto solution1 = makeSolution(std::move(indexScan1));
-
- // c: [[1, 1], [2, 2]]; d: [[1, 3]]
- auto indexScan2 =
- makeIndexScan(BSON("c" << 1 << "d" << 1 << "e" << 1),
- {
- makeOIL("c", {BSON("" << 1 << "" << 1), BSON("" << 2 << "" << 2)}),
- makeOIL("b", {BSON("" << 1 << "" << 3)}),
- });
- auto solution2 = makeSolution(std::move(indexScan2));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(1, winners.size());
- ASSERT_EQ(1, winners[0]);
-}
-
-TEST(IndexPrefixHeuristicTest, SingleIndexScan_LongerPrefixWins) {
- // a: [[2, 3]]; b: [[1, 1]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 3)}),
- makeOIL("b", {BSON("" << 1 << "" << 1)}),
- });
- auto solution1 = makeSolution(std::move(indexScan1));
-
- // c: [[5, 10]]; d: [[10, 20]], e: [[1, 101]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1 << "d" << 1 << "e" << 1),
- {
- makeOIL("c", {BSON("" << 5 << "" << 10)}),
- makeOIL("d", {BSON("" << 10 << "" << 20)}),
- makeOIL("e", {BSON("" << 1 << "" << 101)}),
- });
- auto solution2 = makeSolution(std::move(indexScan2));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(1, winners.size());
- ASSERT_EQ(1, winners[0]);
-}
-
-TEST(IndexPrefixHeuristicTest, SingleIndexScan_MultipleWinners) {
- // a: [[2, 2]]; b: [[1, 1]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 2)}),
- makeOIL("b", {BSON("" << 1 << "" << 1)}),
- });
- auto solution1 = makeSolution(std::move(indexScan1));
-
- // c: [[5, 5]]; d: [[10, 20]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1 << "d" << 1),
- {
- makeOIL("c", {BSON("" << 5 << "" << 5)}),
- makeOIL("d", {BSON("" << 10 << "" << 10)}),
- });
- auto solution2 = makeSolution(std::move(indexScan2));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(2, winners.size());
-}
-
-TEST(IndexPrefixHeuristicTest, SingleIndexScan_ShortestIndexKeyWins) {
- // a: [[2, 2]]; b: [[MinKey, MaxKey]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 2)}),
- makeOIL("b", {BSON("" << MINKEY << "" << MAXKEY)}),
- });
- auto solution1 = makeSolution(std::move(indexScan1));
-
- // c: [[5, 5]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1),
- {
- makeOIL("c", {BSON("" << 5 << "" << 5)}),
- });
- auto solution2 = makeSolution(std::move(indexScan2));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(1, winners.size());
- ASSERT_EQ(1, winners[0]);
-}
-
-TEST(IndexPrefixHeuristicTest, SingleIndexScan_DifferentQueryShapesNoWin) {
- // FETCH <- INDEX_SCAN
- auto indexScan =
- makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 1 << "" << 1), BSON("" << 2 << "" << 2)}),
- makeOIL("b", {BSON("" << 1 << "" << 3)}),
- });
- auto solution1 = makeSolution(std::move(indexScan));
-
- // FETCH <- COLLECTION_SCAN
- auto collScan = std::make_unique<CollectionScanNode>();
- auto solution2 = makeSolution(std::move(collScan));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(0, winners.size());
-}
-
-
-TEST(IndexPrefixHeuristicTest, UnionIndexScan_ShortestIndexKeyWins) {
- auto orNode1 = std::make_unique<OrNode>();
- {
- // a: [[2, 2]]; b: [[1, 1]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 3)}),
- makeOIL("b", {BSON("" << 1 << "" << 1)}),
- });
-
- // c: [[2, 2]], d: [[MaxKey, MinKey]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1 << "d" << 1),
- {
- makeOIL("c", {BSON("" << 2 << "" << 2)}),
- makeOIL("d", {BSON("" << MAXKEY << "" << MINKEY)}),
- });
- orNode1->children.emplace_back(indexScan1.release());
- orNode1->children.emplace_back(indexScan2.release());
- }
- auto solution1 = makeSolution(std::move(orNode1));
-
- auto orNode2 = std::make_unique<OrNode>();
- {
- // a: [[2, 2]]; b: [[1, 1]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 3)}),
- makeOIL("b", {BSON("" << 1 << "" << 1)}),
- });
-
- // c: [[2, 2]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1),
- {
- makeOIL("c", {BSON("" << 2 << "" << 2)}),
- });
- orNode2->children.emplace_back(indexScan1.release());
- orNode2->children.emplace_back(indexScan2.release());
- }
- auto solution2 = makeSolution(std::move(orNode2));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(1, winners.size());
- ASSERT_EQ(1, winners[0]);
-}
-
-TEST(IndexPrefixHeuristicTest, UnionIndexScan_MultipleWinners) {
- auto orNode1 = std::make_unique<OrNode>();
- {
- // a: [[2, 2]]; b: [[1, 1]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 3)}),
- makeOIL("b", {BSON("" << 1 << "" << 1)}),
- });
-
- // c: [[2, 2]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1),
- {
- makeOIL("c", {BSON("" << 2 << "" << 2)}),
- });
- orNode1->children.emplace_back(indexScan1.release());
- orNode1->children.emplace_back(indexScan2.release());
- }
- auto solution1 = makeSolution(std::move(orNode1));
-
- auto orNode2 = std::make_unique<OrNode>();
- {
- // a: [[2, 2]]; b: [[1, 1]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 3)}),
- makeOIL("b", {BSON("" << 1 << "" << 1)}),
- });
-
- // c: [[2, 2]], d: [[MaxKey, MinKey]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1 << "d" << 1),
- {
- makeOIL("c", {BSON("" << 2 << "" << 2)}),
- makeOIL("d", {BSON("" << MAXKEY << "" << MINKEY)}),
- });
- orNode2->children.emplace_back(indexScan1.release());
- orNode2->children.emplace_back(indexScan2.release());
- }
- auto solution2 = makeSolution(std::move(orNode2));
-
- auto orNode3 = std::make_unique<OrNode>();
- {
- // a: [[2, 2]]; b: [[1, 1]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 3)}),
- makeOIL("b", {BSON("" << 1 << "" << 1)}),
- });
-
- // c: [[2, 2]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1),
- {
- makeOIL("c", {BSON("" << 2 << "" << 2)}),
- });
- orNode3->children.emplace_back(indexScan1.release());
- orNode3->children.emplace_back(indexScan2.release());
- }
- auto solution3 = makeSolution(std::move(orNode3));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get(), solution3.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(2, winners.size());
- ASSERT_EQ(0, winners[0]);
- ASSERT_EQ(2, winners[1]);
-}
-
-TEST(IndexPrefixHeuristicTest, UnionIndexScan_DifferentQueryShapeNoWin) {
- auto orNode1 = std::make_unique<OrNode>();
- {
- // a: [[2, 2]]; b: [[1, 1]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 3)}),
- makeOIL("b", {BSON("" << 1 << "" << 1)}),
- });
-
- // c: [[2, 2]], d: [[MaxKey, MinKey]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1 << "d" << 1),
- {
- makeOIL("c", {BSON("" << 2 << "" << 2)}),
- makeOIL("d", {BSON("" << MAXKEY << "" << MINKEY)}),
- });
- orNode1->children.emplace_back(indexScan1.release());
- orNode1->children.emplace_back(indexScan2.release());
- }
- auto solution1 = makeSolution(std::move(orNode1));
-
- auto orNode2 = std::make_unique<OrNode>();
- {
- // a: [[2, 2]]; b: [[1, 1]]
- auto indexScan1 = makeIndexScan(BSON("a" << 1 << "b" << 1),
- {
- makeOIL("a", {BSON("" << 2 << "" << 3)}),
- makeOIL("b", {BSON("" << 1 << "" << 1)}),
- });
-
- // c: [[2, 2]], d: [[MaxKey, MinKey]]
- auto indexScan2 = makeIndexScan(BSON("c" << 1 << "d" << 1),
- {
- makeOIL("c", {BSON("" << 2 << "" << 2)}),
- makeOIL("d", {BSON("" << MAXKEY << "" << MINKEY)}),
- });
-
- // e: [[2, 2]]
- auto indexScan3 = makeIndexScan(BSON("e" << 1),
- {
- makeOIL("e", {BSON("" << 2 << "" << 2)}),
- });
- orNode2->children.emplace_back(indexScan1.release());
- orNode2->children.emplace_back(indexScan2.release());
- orNode2->children.emplace_back(indexScan3.release());
- }
- auto solution2 = makeSolution(std::move(orNode2));
-
- std::vector<const QuerySolution*> solutions{solution1.get(), solution2.get()};
- auto winners = applyIndexPrefixHeuristic(solutions);
- ASSERT_EQ(0, winners.size());
-}
-} // namespace mongo::plan_ranker
diff --git a/src/mongo/db/query/plan_ranker_util.h b/src/mongo/db/query/plan_ranker_util.h
index 60a86205699..4874969db0d 100644
--- a/src/mongo/db/query/plan_ranker_util.h
+++ b/src/mongo/db/query/plan_ranker_util.h
@@ -29,8 +29,6 @@
#pragma once
-#include <algorithm>
-
#include "mongo/db/query/plan_explainer_factory.h"
#include "mongo/db/query/plan_ranker.h"
@@ -47,130 +45,6 @@ namespace mongo::plan_ranker {
std::unique_ptr<PlanScorer<PlanStageStats>> makePlanScorer();
/**
- * Takes a vector of pairs holding (score, planIndex).
- * Returns an iterator pointing to the first non-tying plan, or the end of the vector.
- */
-inline std::vector<std::pair<double, size_t>>::iterator findTopTiedPlans(
- std::vector<std::pair<double, size_t>>& plans) {
- return std::find_if(plans.begin(), plans.end(), [&plans](const auto& plan) {
- return plan.first != plans[0].first;
- });
-}
-
-/**
- * Holds information about tie breaking heuristic bonuses. It is used to update candidate plan score
- * and log the bonuses.
- */
-struct TieBreakingScores {
- TieBreakingScores(bool isPlanTied, double score)
- : isPlanTied(isPlanTied), score(score), docsExaminedBonus(0.0), indexPrefixBonus(0.0) {}
-
- double getTotalBonus() const {
- return docsExaminedBonus + indexPrefixBonus;
- }
-
- const bool isPlanTied;
- const double score;
- double docsExaminedBonus;
- double indexPrefixBonus;
-};
-
-/**
- * Apply docs examined tie breaking heuristic and return bonuses in 'scores' list. 'candidates' and
- * 'scores' are synchronized, with the i-th score from 'scores' corresponding to the i-th
- * 'candidate.'
- */
-template <typename PlanStageType, typename ResultType, typename Data>
-void calcDocsExaminedHeuristicBonus(
- const std::vector<std::pair<double, size_t>>& scoresAndCandidateIndices,
- size_t numberOfTiedPlans,
- const std::vector<BaseCandidatePlan<PlanStageType, ResultType, Data>>& candidates,
- const std::vector<size_t>& documentsExamined,
- std::vector<TieBreakingScores>& scores) {
- // The vector tiedPlans holds the number of documents and the plan's index.
- std::vector<std::pair<double, size_t>> tiedPlans{};
- tiedPlans.reserve(numberOfTiedPlans);
- for (size_t i = 0; i < numberOfTiedPlans; ++i) {
- const size_t candidateIndex = scoresAndCandidateIndices[i].second;
- tiedPlans.emplace_back(std::make_pair(documentsExamined[candidateIndex], candidateIndex));
- }
-
- // Sort top plans by least documents examined, and allocate a bonus to each of the top plans.
- std::stable_sort(tiedPlans.begin(), tiedPlans.end(), [](const auto& lhs, const auto& rhs) {
- return lhs.first < rhs.first;
- });
- auto stillTiedPlansEnd = findTopTiedPlans(tiedPlans);
- for (auto topPlan = tiedPlans.begin(); topPlan < stillTiedPlansEnd; ++topPlan) {
- scores[topPlan->second].docsExaminedBonus = kBonusEpsilon;
- }
-}
-
-/**
- * Apply best index prefix tie breaking heuristic and return bonuses in 'scores' list. 'candidates'
- * and 'scores' are synchronized, with the i-th score from 'scores' corresponding to the i-th
- * 'candidate.'
- */
-template <typename PlanStageType, typename ResultType, typename Data>
-void calcIndexPrefixHeuristicBonus(
- const std::vector<std::pair<double, size_t>>& scoresAndCandidateIndices,
- size_t numberOfTiedPlans,
- const std::vector<BaseCandidatePlan<PlanStageType, ResultType, Data>>& candidates,
- std::vector<TieBreakingScores>& scores) {
- std::vector<const QuerySolution*> solutions{};
- solutions.reserve(numberOfTiedPlans);
-
- for (size_t i = 0; i < numberOfTiedPlans; ++i) {
- const size_t candidateIndex = scoresAndCandidateIndices[i].second;
- solutions.emplace_back(candidates[candidateIndex].solution.get());
- }
-
- auto winIndices = applyIndexPrefixHeuristic(solutions);
- for (auto winIndex : winIndices) {
- const auto candidateIndex = scoresAndCandidateIndices[winIndex].second;
- scores[candidateIndex].indexPrefixBonus += 2 * kBonusEpsilon;
- }
-}
-
-/**
- * Apply tie-breaking hearistics and update candidate plan scores.
- */
-template <typename PlanStageType, typename ResultType, typename Data>
-void addTieBreakingHeuristicsBonuses(
- std::vector<std::pair<double, size_t>>& scoresAndCandidateIndices,
- const std::vector<BaseCandidatePlan<PlanStageType, ResultType, Data>>& candidates,
- const std::vector<size_t>& documentsExamined) {
- auto tiedPlansEnd = findTopTiedPlans(scoresAndCandidateIndices);
- int numberOfTiedPlans = std::distance(scoresAndCandidateIndices.begin(), tiedPlansEnd);
-
- if (numberOfTiedPlans > 1) {
- // Initialize 'scores' list. 'candidates' and 'scores' are synchronized, with the i-th score
- // from 'scores' corresponding to the i-th 'candidate.'
- std::vector<TieBreakingScores> scores{};
- scores.reserve(candidates.size());
- for (size_t i = 0; i < scoresAndCandidateIndices.size(); ++i) {
- scores.emplace_back(/* isPlanTied */ i < static_cast<size_t>(numberOfTiedPlans),
- /* score */ scoresAndCandidateIndices[i].first);
- }
-
- calcDocsExaminedHeuristicBonus(
- scoresAndCandidateIndices, numberOfTiedPlans, candidates, documentsExamined, scores);
-
- calcIndexPrefixHeuristicBonus(
- scoresAndCandidateIndices, numberOfTiedPlans, candidates, scores);
-
- // Log tie breaking bonuses.
- for (const auto& score : scores) {
- log_detail::logTieBreaking(
- score.score, score.docsExaminedBonus, score.indexPrefixBonus, score.isPlanTied);
- }
-
- for (auto& scoreAndIndex : scoresAndCandidateIndices) {
- scoreAndIndex.first += scores[scoreAndIndex.second].getTotalBonus();
- }
- }
-}
-
-/**
* Returns a PlanRankingDecision which has the ranking and the information about the ranking
* process with status OK if everything worked. 'candidateOrder' within the PlanRankingDecision
* holds indices into candidates ordered by score (winner in first element).
@@ -201,7 +75,6 @@ StatusWith<std::unique_ptr<PlanRankingDecision>> pickBestPlan(
// Used to derive scores and candidate ordering.
std::vector<std::pair<double, size_t>> scoresAndCandidateIndices;
std::vector<size_t> failed;
- std::vector<size_t> documentsExamined;
// Compute score for each tree. Record the best.
for (size_t i = 0; i < statTrees.size(); ++i) {
@@ -244,11 +117,6 @@ StatusWith<std::unique_ptr<PlanRankingDecision>> pickBestPlan(
candidates[i].solution->score = score;
scoresAndCandidateIndices.push_back(std::make_pair(score, i));
-
- // Collect some information about documents examined for tie breaking later.
- PlanSummaryStats stats;
- explainer->getSummaryStats(&stats);
- documentsExamined.push_back(stats.totalDocsExamined);
} else {
failed.push_back(i);
log_detail::logFailedPlan([&] { return explainer->getPlanSummary(); });
@@ -270,16 +138,6 @@ StatusWith<std::unique_ptr<PlanRankingDecision>> pickBestPlan(
return lhs.first > rhs.first;
});
- // Apply tie-breaking heuristics.
- if (internalQueryPlanTieBreakingWithIndexHeuristics.load()) {
- addTieBreakingHeuristicsBonuses(scoresAndCandidateIndices, candidates, documentsExamined);
-
- // Re-sort the candidates.
- std::stable_sort(scoresAndCandidateIndices.begin(),
- scoresAndCandidateIndices.end(),
- [](const auto& lhs, const auto& rhs) { return lhs.first > rhs.first; });
- }
-
auto why = std::make_unique<PlanRankingDecision>();
if constexpr (std::is_same_v<PlanStageStatsType, mongo::sbe::PlanStageStats>) {
diff --git a/src/mongo/db/query/plan_summary_stats.h b/src/mongo/db/query/plan_summary_stats.h
index 5c153ef3e39..ac80b6505f3 100644
--- a/src/mongo/db/query/plan_summary_stats.h
+++ b/src/mongo/db/query/plan_summary_stats.h
@@ -29,7 +29,6 @@
#pragma once
-#include "mongo/util/duration.h"
#include <optional>
#include <string>
diff --git a/src/mongo/db/query/plan_yield_policy.cpp b/src/mongo/db/query/plan_yield_policy.cpp
index 79816505026..58064f76d6e 100644
--- a/src/mongo/db/query/plan_yield_policy.cpp
+++ b/src/mongo/db/query/plan_yield_policy.cpp
@@ -32,7 +32,6 @@
#include "mongo/db/query/plan_yield_policy.h"
#include "mongo/db/catalog/collection.h"
-#include "mongo/db/concurrency/exception_util.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/operation_context.h"
#include "mongo/util/scopeguard.h"
@@ -40,43 +39,17 @@
namespace mongo {
-PlanYieldPolicy::PlanYieldPolicy(OperationContext* opCtx,
- YieldPolicy policy,
+PlanYieldPolicy::PlanYieldPolicy(YieldPolicy policy,
ClockSource* cs,
int yieldIterations,
Milliseconds yieldPeriod,
const Yieldable* yieldable,
std::unique_ptr<const YieldPolicyCallbacks> callbacks)
- : _policy(getPolicyOverrideForOperation(opCtx, policy)),
+ : _policy(policy),
_yieldable(yieldable),
_callbacks(std::move(callbacks)),
_elapsedTracker(cs, yieldIterations, yieldPeriod) {}
-PlanYieldPolicy::YieldPolicy PlanYieldPolicy::getPolicyOverrideForOperation(
- OperationContext* opCtx, PlanYieldPolicy::YieldPolicy desired) {
- // We may have a null opCtx in testing.
- if (MONGO_unlikely(!opCtx)) {
- return desired;
- }
- // Multi-document transactions cannot yield locks or snapshots. We convert to a non-yielding
- // interruptible plan.
- if (opCtx->inMultiDocumentTransaction() &&
- (desired == YieldPolicy::YIELD_AUTO || desired == YieldPolicy::YIELD_MANUAL ||
- desired == YieldPolicy::WRITE_CONFLICT_RETRY_ONLY)) {
- return YieldPolicy::INTERRUPT_ONLY;
- }
-
- // If the state of our locks held is not yieldable at all, we will assume this is an internal
- // operation that should not be interrupted or yielded.
- // TODO: SERVER-76238 Evaluate if we can make everything INTERRUPT_ONLY instead.
- if (!opCtx->lockState()->canSaveLockState() &&
- (desired == YieldPolicy::YIELD_AUTO || desired == YieldPolicy::YIELD_MANUAL)) {
- return YieldPolicy::NO_YIELD;
- }
-
- return desired;
-}
-
bool PlanYieldPolicy::shouldYieldOrInterrupt(OperationContext* opCtx) {
if (_policy == YieldPolicy::INTERRUPT_ONLY) {
return _elapsedTracker.intervalHasElapsed();
@@ -157,7 +130,7 @@ Status PlanYieldPolicy::yieldOrInterrupt(OperationContext* opCtx,
if (_callbacks) {
_callbacks->handledWriteConflict(opCtx);
}
- logWriteConflictAndBackoff(attempt, "query yield", ""_sd);
+ WriteConflictException::logAndBackoff(attempt, "query yield", ""_sd);
// Retry the yielding process.
} catch (...) {
// Errors other than write conflicts don't get retried, and should instead result in
@@ -174,34 +147,46 @@ void PlanYieldPolicy::performYield(OperationContext* opCtx,
std::function<void()> whileYieldingFn) {
// Things have to happen here in a specific order:
// * Release 'yieldable'.
- // * Abandon the current storage engine snapshot.
+ // * Release lock mgr locks.
// * Check for interrupt if the yield policy requires.
- // * Release lock manager locks.
- // * Reacquire lock manager locks.
+ // * Abondon the query's current storage engine snapshot.
+ // * Reacquire lock mgr locks.
// * Restore 'yieldable'.
- invariant(_policy == YieldPolicy::YIELD_AUTO || _policy == YieldPolicy::YIELD_MANUAL);
+ Locker* locker = opCtx->lockState();
+
+ if (locker->isGlobalLockedRecursively()) {
+ // No purpose in yielding if the locks are recursively held and cannot be released.
+ return;
+ }
- // If we are here, the caller has guaranteed locks are not recursively held. This is a top level
- // operation and we can safely clear the 'yieldable' state before unlocking and then
- // re-establish it after re-locking.
+ // Since the locks are not recursively held, this is a top level operation and we can safely
+ // clear the 'yieldable' state before unlocking and then re-establish it after re-locking.
if (yieldable) {
yieldable->yield();
}
- // Release any storage engine resources. This requires holding a global lock to correctly
- // synchronize with states such as shutdown and rollback.
- opCtx->recoveryUnit()->abandonSnapshot();
+ Locker::LockSnapshot snapshot;
+ auto unlocked = locker->saveLockStateAndUnlock(&snapshot);
- // Check for interrupt before releasing locks. This avoids the complexities of having to
- // re-acquire locks to clean up when we are interrupted. This is the main interrupt check during
- // query execution. Yield points and interrupt points are one and the same.
+ // After all steps to relinquish locks and save the execution plan have been taken, check
+ // for interrupt. This is the main interrupt check during query execution. Yield points and
+ // interrupt points are one and the same.
if (getPolicy() == PlanYieldPolicy::YieldPolicy::YIELD_AUTO) {
opCtx->checkForInterrupt(); // throws
}
- Locker* locker = opCtx->lockState();
- Locker::LockSnapshot snapshot;
- locker->saveLockStateAndUnlock(&snapshot);
+ if (!unlocked) {
+ // Nothing was unlocked. Recursively held locks are not the only reason locks cannot be
+ // released. Restore the 'yieldable' state before returning.
+ if (yieldable) {
+ yieldable->restore();
+ }
+ return;
+ }
+
+ // Top-level locks are freed, release any potential low-level (storage engine-specific
+ // locks). If we are yielding, we are at a safe place to do so.
+ opCtx->recoveryUnit()->abandonSnapshot();
if (_callbacks) {
_callbacks->duringYield(opCtx);
diff --git a/src/mongo/db/query/plan_yield_policy.h b/src/mongo/db/query/plan_yield_policy.h
index 05bbc5b40eb..9d44ba8e903 100644
--- a/src/mongo/db/query/plan_yield_policy.h
+++ b/src/mongo/db/query/plan_yield_policy.h
@@ -172,24 +172,14 @@ public:
}
/**
- * Returns the policy that this operation should use, accounting for any special circumstances,
- * and otherwise the desired policy. Should always be used when constructing a PlanYieldPolicy.
- */
- static YieldPolicy getPolicyOverrideForOperation(OperationContext* opCtx, YieldPolicy desired);
-
- /**
* Constructs a PlanYieldPolicy of the given 'policy' type. This class uses an ElapsedTracker
* to keep track of elapsed time, which is initialized from the parameters 'cs',
* 'yieldIterations' and 'yieldPeriod'.
*
* If provided, the given 'yieldable' is released and restored by the 'PlanYieldPolicy' (in
- * addition to releasing/restoring locks and the storage engine snapshot). The provided 'policy'
- * will be overridden depending on the nature of this operation. For example, multi-document
- * transactions will always downgrade to INTERRUPT_ONLY, and operations with recursively held
- * locks will downgrade to NO_YIELD.
+ * addition to releasing/restoring locks and the storage engine snapshot).
*/
- PlanYieldPolicy(OperationContext* opCtx,
- YieldPolicy policy,
+ PlanYieldPolicy(YieldPolicy policy,
ClockSource* cs,
int yieldIterations,
Milliseconds yieldPeriod,
diff --git a/src/mongo/db/query/plan_yield_policy_impl.cpp b/src/mongo/db/query/plan_yield_policy_impl.cpp
index 8b45e2998ed..e4f0688c97f 100644
--- a/src/mongo/db/query/plan_yield_policy_impl.cpp
+++ b/src/mongo/db/query/plan_yield_policy_impl.cpp
@@ -39,8 +39,9 @@ PlanYieldPolicyImpl::PlanYieldPolicyImpl(PlanExecutorImpl* exec,
PlanYieldPolicy::YieldPolicy policy,
const Yieldable* yieldable,
std::unique_ptr<YieldPolicyCallbacks> callbacks)
- : PlanYieldPolicy(exec->getOpCtx(),
- policy,
+ : PlanYieldPolicy(exec->getOpCtx()->lockState()->isGlobalLockedRecursively()
+ ? PlanYieldPolicy::YieldPolicy::NO_YIELD
+ : policy,
exec->getOpCtx()->getServiceContext()->getFastClockSource(),
internalQueryExecYieldIterations.load(),
Milliseconds{internalQueryExecYieldPeriodMS.load()},
diff --git a/src/mongo/db/query/plan_yield_policy_sbe.h b/src/mongo/db/query/plan_yield_policy_sbe.h
index 2286dcb6434..d0c0ea81fa7 100644
--- a/src/mongo/db/query/plan_yield_policy_sbe.h
+++ b/src/mongo/db/query/plan_yield_policy_sbe.h
@@ -36,21 +36,15 @@ namespace mongo {
class PlanYieldPolicySBE final : public PlanYieldPolicy {
public:
- PlanYieldPolicySBE(OperationContext* opCtx,
- YieldPolicy policy,
+ PlanYieldPolicySBE(YieldPolicy policy,
ClockSource* clockSource,
int yieldFrequency,
Milliseconds yieldPeriod,
const Yieldable* yieldable,
std::unique_ptr<YieldPolicyCallbacks> callbacks,
bool useExperimentalCommitTxnBehavior)
- : PlanYieldPolicy(opCtx,
- policy,
- clockSource,
- yieldFrequency,
- yieldPeriod,
- yieldable,
- std::move(callbacks)),
+ : PlanYieldPolicy(
+ policy, clockSource, yieldFrequency, yieldPeriod, yieldable, std::move(callbacks)),
_useExperimentalCommitTxnBehavior(useExperimentalCommitTxnBehavior) {
uassert(4822879,
"WRITE_CONFLICT_RETRY_ONLY yield policy is not supported in SBE",
diff --git a/src/mongo/db/query/planner_access.cpp b/src/mongo/db/query/planner_access.cpp
index de294ad661a..e05dfd8a5d1 100644
--- a/src/mongo/db/query/planner_access.cpp
+++ b/src/mongo/db/query/planner_access.cpp
@@ -45,21 +45,14 @@
#include "mongo/db/matcher/expression.h"
#include "mongo/db/matcher/expression_array.h"
#include "mongo/db/matcher/expression_geo.h"
-#include "mongo/db/matcher/expression_internal_expr_comparison.h"
-#include "mongo/db/matcher/expression_leaf.h"
#include "mongo/db/matcher/expression_text.h"
-#include "mongo/db/matcher/expression_tree.h"
-#include "mongo/db/namespace_string.h"
-#include "mongo/db/query/index_bounds.h"
#include "mongo/db/query/index_bounds_builder.h"
#include "mongo/db/query/index_tag.h"
#include "mongo/db/query/indexability.h"
-#include "mongo/db/query/parsed_find_command.h"
#include "mongo/db/query/planner_wildcard_helpers.h"
#include "mongo/db/query/query_knobs_gen.h"
#include "mongo/db/query/query_planner.h"
#include "mongo/db/query/query_planner_common.h"
-#include "mongo/db/query/record_id_range.h"
#include "mongo/db/record_id_helpers.h"
#include "mongo/logv2/log.h"
#include "mongo/util/transitional_tools_do_not_use/vector_spooling.h"
@@ -235,37 +228,27 @@ bool affectedByCollator(const BSONElement& element) {
}
}
-// Set 'curr' to 'newMin' if 'newMin' < 'curr'
-void setLowestRecord(boost::optional<RecordIdBound>& curr, const RecordIdBound& newMin) {
- if (!curr || newMin.recordId() < curr->recordId()) {
- curr = newMin;
+void setMinRecord(CollectionScanNode* collScan, const BSONObj& min) {
+ const auto newMinRecord = record_id_helpers::keyForObj(min);
+ if (!collScan->minRecord || newMinRecord > collScan->minRecord->recordId()) {
+ collScan->minRecord = RecordIdBound(newMinRecord, min);
}
}
-// Set 'curr' to 'newMax' if 'newMax' > 'curr'
-void setHighestRecord(boost::optional<RecordIdBound>& curr, const RecordIdBound& newMax) {
- if (!curr || newMax.recordId() > curr->recordId()) {
- curr = newMax;
+void setMaxRecord(CollectionScanNode* collScan, const BSONObj& max) {
+ const auto newMaxRecord = record_id_helpers::keyForObj(max);
+ if (!collScan->maxRecord || newMaxRecord < collScan->maxRecord->recordId()) {
+ collScan->maxRecord = RecordIdBound(newMaxRecord, max);
}
}
-// Set 'curr' to 'newMin' if 'newMin' < 'curr'
-void setLowestRecord(boost::optional<RecordIdBound>& curr, const BSONObj& newMin) {
- setLowestRecord(curr, RecordIdBound(record_id_helpers::keyForObj(newMin), newMin));
-}
-
-// Set 'curr' to 'newMax' if 'newMax' > 'curr'
-void setHighestRecord(boost::optional<RecordIdBound>& curr, const BSONObj& newMax) {
- setHighestRecord(curr, RecordIdBound(record_id_helpers::keyForObj(newMax), newMax));
-}
-
// Returns whether element is not affected by collators or query and collection collators are
// compatible.
bool compatibleCollator(const QueryPlannerParams& params,
const CollatorInterface* queryCollator,
const BSONElement& element) {
auto const collCollator = params.clusteredCollectionCollator;
- bool compatible = CollatorInterface::collatorsMatch(queryCollator, collCollator);
+ bool compatible = !queryCollator || (collCollator && *queryCollator == *collCollator);
return compatible || !affectedByCollator(element);
}
@@ -276,8 +259,7 @@ bool compatibleCollator(const QueryPlannerParams& params,
void handleRIDRangeMinMax(const CanonicalQuery& query,
CollectionScanNode* collScan,
const QueryPlannerParams& params,
- const CollatorInterface* collator,
- RecordIdRange& recordRange) {
+ const CollatorInterface* collator) {
BSONObj minObj = query.getFindCommandRequest().getMin();
BSONObj maxObj = query.getFindCommandRequest().getMax();
if (minObj.isEmpty() && maxObj.isEmpty()) {
@@ -297,170 +279,88 @@ void handleRIDRangeMinMax(const CanonicalQuery& query,
if (!maxObj.isEmpty() && compatibleCollator(params, collator, maxObj.firstElement())) {
// max() is exclusive.
// Assumes clustered collection scans are only supported with the forward direction.
- recordRange.maybeNarrowMax(
- IndexBoundsBuilder::objFromElement(maxObj.firstElement(), collator),
- false /* NOT inclusive*/);
+ collScan->boundInclusion =
+ CollectionScanParams::ScanBoundInclusion::kIncludeStartRecordOnly;
+ setMaxRecord(collScan, IndexBoundsBuilder::objFromElement(maxObj.firstElement(), collator));
}
if (!minObj.isEmpty() && compatibleCollator(params, collator, minObj.firstElement())) {
// The min() is inclusive as are bounded collection scans by default.
- recordRange.maybeNarrowMin(
- IndexBoundsBuilder::objFromElement(minObj.firstElement(), collator),
- true /* inclusive*/);
+ setMinRecord(collScan, IndexBoundsBuilder::objFromElement(minObj.firstElement(), collator));
}
}
/**
* Helper function to add an RID range to collection scans.
- * If the query solution tree contains a collection scan node with a suitable comparison predicate
- * on '_id', we add a minRecord and maxRecord on the collection node.
- *
- * Returns true if the MatchExpression is a comparison against the cluster key which either:
- * 1) is guaranteed to exclude values of the cluster key which are affected by collation or
- * 2) may return values of the cluster key which are affected by collation, but the query and
- * collection collations match.
- * Otherwise, returns false.
- *
- * For example, assuming the cluster key is "_id":
- * Given {a: {$eq: 2}}, we return false, because the comparison is not against the cluster key.
- * Given {_id: {$gte: 5}}, we return true, because this comparison against the cluster key excludes
- * keys which are affected by collations.
- * Given {_id: {$eq: "str"}}, we return true only if the query and collection collations match.
- *
+ * If the query solution tree contains a collection scan node with a suitable comparison
+ * predicate on '_id', we add a minRecord and maxRecord on the collection node.
*/
-[[nodiscard]] bool handleRIDRangeScan(const MatchExpression* conjunct,
- CollectionScanNode* collScan,
- const QueryPlannerParams& params,
- const CollatorInterface* collator,
- RecordIdRange& recordRange) {
+void handleRIDRangeScan(const MatchExpression* conjunct,
+ CollectionScanNode* collScan,
+ const QueryPlannerParams& params,
+ const CollatorInterface* collator) {
invariant(params.clusteredInfo);
if (conjunct == nullptr) {
- return false;
+ return;
}
auto* andMatchPtr = dynamic_cast<const AndMatchExpression*>(conjunct);
if (andMatchPtr != nullptr) {
- bool atLeastOneConjunctCompatibleCollation = false;
for (size_t index = 0; index < andMatchPtr->numChildren(); index++) {
- if (handleRIDRangeScan(
- andMatchPtr->getChild(index), collScan, params, collator, recordRange)) {
- atLeastOneConjunctCompatibleCollation = true;
- }
+ handleRIDRangeScan(andMatchPtr->getChild(index), collScan, params, collator);
}
-
- // If one of the conjuncts excludes values of the cluster key which are affected by
- // collation, then the entire $and will also exclude those values.
- return atLeastOneConjunctCompatibleCollation;
+ return;
}
if (conjunct->path() !=
clustered_util::getClusterKeyFieldName(params.clusteredInfo->getIndexSpec())) {
// No match on the cluster key.
- return false;
- }
-
- // TODO SERVER-62707: Allow $in with regex to use a clustered index.
- auto inMatch = dynamic_cast<const InMatchExpression*>(conjunct);
- if (inMatch && !inMatch->hasRegex()) {
- // Iterate through the $in equalities to find the min/max values. The min/max bounds for the
- // collscan need to be loose enough to cover all of these values.
- boost::optional<RecordIdBound> minBound;
- boost::optional<RecordIdBound> maxBound;
-
- bool allEltsCollationCompatible = true;
- for (const auto& element : inMatch->getEqualities()) {
- if (compatibleCollator(params, collator, element)) {
- const auto collated = IndexBoundsBuilder::objFromElement(element, collator);
- setLowestRecord(minBound, collated);
- setHighestRecord(maxBound, collated);
- } else {
- // Set coarse min/max bounds based on type when we can't set tight bounds.
- allEltsCollationCompatible = false;
-
- BSONObjBuilder bMin;
- bMin.appendMinForType("", element.type());
- setLowestRecord(minBound, bMin.obj());
-
- BSONObjBuilder bMax;
- bMax.appendMaxForType("", element.type());
- setHighestRecord(maxBound, bMax.obj());
- }
- }
-
- // {min,max}RecordId will bound the range of ids scanned to the highest and lowest present
- // in the InMatchExpression, but the filter is still required to filter to _exactly_ the
- // requested matches.
-
- // Finally, tighten the collscan bounds with the min/max bounds for the $in.
- recordRange.intersectRange(minBound, maxBound);
- return allEltsCollationCompatible;
+ return;
}
- auto match = dynamic_cast<const ComparisonMatchExpressionBase*>(conjunct);
+ auto match = dynamic_cast<const ComparisonMatchExpression*>(conjunct);
if (match == nullptr) {
- return false; // Not a comparison match expression.
+ return; // Not a comparison match expression.
}
const auto& element = match->getData();
- if (!ComparisonMatchExpressionBase::isInternalExprComparison(match->matchType())) {
- // Internal comparisons e.g., $_internalExprGt do _not_ carry type bracketing
- // semantics (consistent with `$expr{$gt:[a,b]}`).
- // For other comparisons which _do_ perform type bracketing, the RecordId bounds
- // may be tightened here.
- BSONObjBuilder minb;
- minb.appendMinForType("", element.type());
- recordRange.maybeNarrowMin(minb.obj(), true /* inclusive */);
+ // Set coarse min/max bounds based on type in case we can't set tight bounds.
+ BSONObjBuilder minb;
+ minb.appendMinForType("", element.type());
+ setMinRecord(collScan, minb.obj());
- BSONObjBuilder maxb;
- maxb.appendMaxForType("", element.type());
- recordRange.maybeNarrowMax(maxb.obj(), true /* inclusive */);
- }
+ BSONObjBuilder maxb;
+ maxb.appendMaxForType("", element.type());
+ setMaxRecord(collScan, maxb.obj());
bool compatible = compatibleCollator(params, collator, element);
if (!compatible) {
- return false; // Collator affects probe and it's not compatible with collection's collator.
+ return; // Collator affects probe and it's not compatible with collection's collator.
}
// Even if the collations don't match at this point, it's fine,
- // because the bounds exclude values that use it.
- const BSONObj collated = IndexBoundsBuilder::objFromElement(element, collator);
- using MType = MatchExpression::MatchType;
- switch (match->matchType()) {
- case MType::EQ:
- case MType::INTERNAL_EXPR_EQ:
- recordRange.maybeNarrowMin(collated, true /* inclusive */);
- recordRange.maybeNarrowMax(collated, true /* inclusive */);
- break;
- case MType::LT:
- case MType::INTERNAL_EXPR_LT:
- recordRange.maybeNarrowMax(collated, false /* EXclusive */);
- break;
- case MType::LTE:
- case MType::INTERNAL_EXPR_LTE:
- recordRange.maybeNarrowMax(collated, true /* inclusive */);
- break;
- case MType::GT:
- case MType::INTERNAL_EXPR_GT:
- recordRange.maybeNarrowMin(collated, false /* EXclusive */);
- break;
- case MType::GTE:
- case MType::INTERNAL_EXPR_GTE:
- recordRange.maybeNarrowMin(collated, true /* inclusive */);
- break;
- default:;
+ // because the bounds exclude values that use it
+ collScan->hasCompatibleCollation = true;
+
+ const auto collated = IndexBoundsBuilder::objFromElement(element, collator);
+ if (dynamic_cast<const EqualityMatchExpression*>(match)) {
+ setMinRecord(collScan, collated);
+ setMaxRecord(collScan, collated);
+ } else if (dynamic_cast<const LTMatchExpression*>(match) ||
+ dynamic_cast<const LTEMatchExpression*>(match)) {
+ setMaxRecord(collScan, collated);
+ } else if (dynamic_cast<const GTMatchExpression*>(match) ||
+ dynamic_cast<const GTEMatchExpression*>(match)) {
+ setMinRecord(collScan, collated);
}
- return true;
}
} // namespace
std::unique_ptr<QuerySolutionNode> QueryPlannerAccess::makeCollectionScan(
const CanonicalQuery& query, bool tailable, const QueryPlannerParams& params, int direction) {
- // The following are expensive to look up, so only do it once for each.
- const mongo::NamespaceString nss = query.nss();
- const bool isOplog = nss.isOplog();
// Make the (only) node, a collection scan.
auto csn = std::make_unique<CollectionScanNode>();
csn->name = query.ns();
@@ -471,7 +371,6 @@ std::unique_ptr<QuerySolutionNode> QueryPlannerAccess::makeCollectionScan(
csn->shouldWaitForOplogVisibility =
params.options & QueryPlannerParams::OPLOG_SCAN_WAIT_FOR_VISIBLE;
csn->direction = direction;
- csn->isOplog = isOplog;
if (params.clusteredInfo) {
csn->clusteredIndex = params.clusteredInfo->getIndexSpec();
@@ -548,29 +447,14 @@ std::unique_ptr<QuerySolutionNode> QueryPlannerAccess::makeCollectionScan(
auto queryCollator = query.getCollator();
auto collCollator = params.clusteredCollectionCollator;
- csn->hasCompatibleCollation = CollatorInterface::collatorsMatch(queryCollator, collCollator);
+ csn->hasCompatibleCollation =
+ !queryCollator || (collCollator && *queryCollator == *collCollator);
if (params.clusteredInfo && !csn->resumeAfterRecordId) {
// This is a clustered collection. Attempt to perform an efficient, bounded collection scan
- // via minRecord and maxRecord if applicable. During this process, we will check if the
- // query is guaranteed to exclude values of the cluster key which are affected by collation.
- // If so, then even if the query and collection collations differ, the collation difference
- // won't affect the query results. In that case, we can say hasCompatibleCollation is true.
-
- RecordIdRange recordRange;
- // min/max records may have been set if oplog or change collection.
- recordRange.intersectRange(csn->minRecord, csn->maxRecord);
- bool compatibleCollation =
- handleRIDRangeScan(csn->filter.get(), csn.get(), params, queryCollator, recordRange);
- csn->hasCompatibleCollation |= compatibleCollation;
-
- handleRIDRangeMinMax(query, csn.get(), params, queryCollator, recordRange);
-
- csn->minRecord = recordRange.getMin();
- csn->maxRecord = recordRange.getMax();
-
- csn->boundInclusion = CollectionScanParams::makeInclusion(recordRange.isMinInclusive(),
- recordRange.isMaxInclusive());
+ // via minRecord and maxRecord if applicable.
+ handleRIDRangeScan(csn->filter.get(), csn.get(), params, queryCollator);
+ handleRIDRangeMinMax(query, csn.get(), params, queryCollator);
}
return csn;
@@ -1230,13 +1114,47 @@ std::vector<std::unique_ptr<QuerySolutionNode>> QueryPlannerAccess::collapseEqui
}
/**
- * This helper determines if a query can be covered depending on the query projection.
+ * Returns true if this is a null query that can retrieve all the information it needs directly from
+ * the index, and so does not need a FETCH stage on top of it. Returns false otherwise.
*/
-bool projNeedsFetch(const CanonicalQuery& query, const QueryPlannerParams& params) {
+bool isCoveredNullQuery(const CanonicalQuery& query,
+ MatchExpression* root,
+ IndexTag* tag,
+ const vector<IndexEntry>& indices,
+ const QueryPlannerParams& params) {
+ // Sparse indexes and hashed indexes should not use this optimization as they will require a
+ // FETCH stage with a filter.
+ if (indices[tag->index].sparse || indices[tag->index].type == IndexType::INDEX_HASHED) {
+ return false;
+ }
+
+ // When the index is not multikey, we can support a query on an indexed field searching for null
+ // values. This optimization can only be done when the index is not multikey, otherwise empty
+ // arrays in the collection will be treated as null/undefined by the index. When the index is
+ // multikey, we can support a query searching for both null and empty array values.
+ const auto multikeyIndex = indices[tag->index].multikey;
+ if (root->matchType() == MatchExpression::MatchType::MATCH_IN) {
+ // Check that the query matches null values, if the index is not multikey, or null and empty
+ // array values, if the index is multikey. Note that the query may match values other than
+ // null (and empty array).
+ const auto node = static_cast<const InMatchExpression*>(root);
+ if (!node->hasNull() || (multikeyIndex && !node->hasEmptyArray())) {
+ return false;
+ }
+ } else if (ComparisonMatchExpressionBase::isEquality(root->matchType()) && !multikeyIndex) {
+ // Check that the query matches null values.
+ const auto node = static_cast<const ComparisonMatchExpressionBase*>(root);
+ if (node->getData().type() != BSONType::jstNULL) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+
// If nothing is being projected, the query is fully covered without a fetch.
// This is trivially true for a count query.
if (params.options & QueryPlannerParams::Options::IS_COUNT) {
- return false;
+ return true;
}
// This optimization can only be used for find when the index covers the projection completely.
@@ -1245,7 +1163,7 @@ bool projNeedsFetch(const CanonicalQuery& query, const QueryPlannerParams& param
// in the multikey case). Hence, only find queries projecting _id are covered.
auto proj = query.getProj();
if (!proj) {
- return true;
+ return false;
}
// We can cover projections on _id and generated fields and expressions depending only on _id.
@@ -1257,38 +1175,10 @@ bool projNeedsFetch(const CanonicalQuery& query, const QueryPlannerParams& param
// Note that it is not possible to project onto dotted paths of _id here, since they may be
// null or missing, and the index cannot differentiate between the two cases, so we would
// still need a FETCH stage.
- if (projFields.size() == 1 && *projFields.begin() == "_id") {
- return false;
- }
+ return projFields.size() == 1 && *projFields.begin() == "_id";
}
- return true;
-}
-
-/**
- * This helper updates a MAYBE_COVERED query tightness to one of EXACT, INEXACT_COVERED, or
- * INEXACT_FETCH, depending on whether we need a FETCH/filter to answer the query projection.
- */
-void refineTightnessForMaybeCoveredQuery(const CanonicalQuery& query,
- const QueryPlannerParams& params,
- IndexBoundsBuilder::BoundsTightness& tightnessOut) {
- // We need to refine the tightness in case we have a "MAYBE_COVERED" tightness bound which
- // depends on the query's projection. We will not have information about the projection
- // later on in order to make this determination, so we do it here.
- const bool noFetchNeededForProj = !projNeedsFetch(query, params);
- if (tightnessOut == IndexBoundsBuilder::EXACT_MAYBE_COVERED) {
- if (noFetchNeededForProj) {
- tightnessOut = IndexBoundsBuilder::EXACT;
- } else {
- tightnessOut = IndexBoundsBuilder::INEXACT_FETCH;
- }
- } else if (tightnessOut == IndexBoundsBuilder::INEXACT_MAYBE_COVERED) {
- if (noFetchNeededForProj) {
- tightnessOut = IndexBoundsBuilder::INEXACT_COVERED;
- } else {
- tightnessOut = IndexBoundsBuilder::INEXACT_FETCH;
- }
- }
+ return false;
}
bool QueryPlannerAccess::processIndexScans(const CanonicalQuery& query,
@@ -1332,6 +1222,11 @@ bool QueryPlannerAccess::processIndexScans(const CanonicalQuery& query,
// If we're here, we now know that 'child' can use an index directly and the index is
// over the child's field.
+ // We need to track if this is a covered null query so that we can have this information
+ // at hand when handling the filter on an indexed AND.
+ scanState.isCoveredNullQuery =
+ isCoveredNullQuery(query, child, scanState.ixtag, indices, params);
+
// If 'child' is a NOT, then the tag we're interested in is on the NOT's
// child node.
if (MatchExpression::NOT == child->matchType()) {
@@ -1364,7 +1259,6 @@ bool QueryPlannerAccess::processIndexScans(const CanonicalQuery& query,
verify(scanState.currentIndexNumber == scanState.ixtag->index);
scanState.tightness = IndexBoundsBuilder::INEXACT_FETCH;
mergeWithLeafNode(child, &scanState);
- refineTightnessForMaybeCoveredQuery(query, params, scanState.tightness);
handleFilter(&scanState);
} else {
if (nullptr != scanState.currentScan.get()) {
@@ -1384,7 +1278,6 @@ bool QueryPlannerAccess::processIndexScans(const CanonicalQuery& query,
&scanState.tightness,
scanState.getCurrentIETBuilder());
- refineTightnessForMaybeCoveredQuery(query, params, scanState.tightness);
handleFilter(&scanState);
}
}
@@ -1800,12 +1693,6 @@ std::unique_ptr<QuerySolutionNode> QueryPlannerAccess::_buildIndexedDataAccess(
return soln;
}
- // We may be able to avoid adding an extra fetch stage even though the bounds are
- // inexact, for instance if the query is counting null values on an indexed field
- // without projecting that field. We therefore convert "MAYBE_COVERED" bounds into
- // either EXACT or INEXACT, depending on the query projection.
- refineTightnessForMaybeCoveredQuery(query, params, tightness);
-
// If the bounds are exact, the set of documents that satisfy the predicate is
// exactly equal to the set of documents that the scan provides.
//
@@ -1813,7 +1700,11 @@ std::unique_ptr<QuerySolutionNode> QueryPlannerAccess::_buildIndexedDataAccess(
// superset of documents that satisfy the predicate, and we must check the
// predicate.
- if (tightness == IndexBoundsBuilder::EXACT) {
+ // We may also be able to avoid adding an extra fetch stage even though the bounds are
+ // inexact because the query is counting null values on an indexed field without
+ // projecting that field.
+ if (tightness == IndexBoundsBuilder::EXACT ||
+ isCoveredNullQuery(query, root, tag, indices, params)) {
return soln;
} else if (tightness == IndexBoundsBuilder::INEXACT_COVERED &&
!indices[tag->index].multikey) {
@@ -1959,9 +1850,10 @@ void QueryPlannerAccess::handleFilterAnd(ScanBuildingState* scanState) {
// should always be affixed as a filter. We keep 'curChild' in the $and
// for affixing later.
++scanState->curChild;
- } else if (scanState->tightness == IndexBoundsBuilder::EXACT) {
- // The tightness of the bounds is exact. We want to remove this child so that when control
- // returns to handleIndexedAnd we know that we don't need it to create a FETCH stage.
+ } else if (scanState->tightness == IndexBoundsBuilder::EXACT || scanState->isCoveredNullQuery) {
+ // The tightness of the bounds is exact or we are dealing with a covered null query.
+ // Either way, we want to remove this child so that when control returns to handleIndexedAnd
+ // we know that we don't need it to create a FETCH stage.
root->getChildVector()->erase(root->getChildVector()->begin() + scanState->curChild);
} else if (scanState->tightness == IndexBoundsBuilder::INEXACT_COVERED &&
(INDEX_TEXT == index.type || !index.multikey)) {
diff --git a/src/mongo/db/query/planner_access.h b/src/mongo/db/query/planner_access.h
index a5bfc04bb51..6ea44830415 100644
--- a/src/mongo/db/query/planner_access.h
+++ b/src/mongo/db/query/planner_access.h
@@ -106,7 +106,7 @@ public:
static std::unique_ptr<QuerySolutionNode> makeCollectionScan(const CanonicalQuery& query,
bool tailable,
const QueryPlannerParams& params,
- int direction);
+ int direction = 1);
/**
* Return a plan that uses the provided index as a proxy for a collection scan.
@@ -126,7 +126,7 @@ public:
const BSONObj& endKey);
/**
- * Constructs a data access plan for 'query' which answers the predicate contained in 'root'.
+ * Consructs a data access plan for 'query' which answers the predicate contained in 'root'.
* Assumes the presence of the passed in indices. Planning behavior is controlled by the
* settings in 'params'.
*/
@@ -145,9 +145,11 @@ private:
struct ScanBuildingState {
ScanBuildingState(MatchExpression* theRoot,
const std::vector<IndexEntry>& indexList,
- bool inArrayOp)
+ bool inArrayOp,
+ bool isCoveredNull = false)
: root(theRoot),
inArrayOperator(inArrayOp),
+ isCoveredNullQuery(isCoveredNull),
indices(indexList),
currentScan(nullptr),
curChild(0),
@@ -186,6 +188,9 @@ private:
// Are we inside an array operator such as $elemMatch or $all?
bool inArrayOperator;
+ // Is this a covered null query?
+ bool isCoveredNullQuery;
+
// A list of relevant indices which 'root' may be tagged to use.
const std::vector<IndexEntry>& indices;
diff --git a/src/mongo/db/query/planner_access_test.cpp b/src/mongo/db/query/planner_access_test.cpp
index d30544b43fa..52df32c871e 100644
--- a/src/mongo/db/query/planner_access_test.cpp
+++ b/src/mongo/db/query/planner_access_test.cpp
@@ -38,7 +38,9 @@ namespace mongo {
namespace {
BSONObj serializeMatcher(Matcher* matcher) {
- return matcher->getMatchExpression()->serialize();
+ BSONObjBuilder builder;
+ matcher->getMatchExpression()->serialize(&builder);
+ return builder.obj();
}
TEST(PlannerAccessTest, PrepareForAccessPlanningSortsEqualNodesByTheirChildren) {
diff --git a/src/mongo/db/query/planner_analysis.cpp b/src/mongo/db/query/planner_analysis.cpp
index ee86d426560..40d8d7b0d0d 100644
--- a/src/mongo/db/query/planner_analysis.cpp
+++ b/src/mongo/db/query/planner_analysis.cpp
@@ -337,7 +337,7 @@ void geoSkipValidationOn(const std::set<StringData>& twoDSphereFields,
/**
* If any field is missing from the list of fields the projection wants, we are not covered.
*/
-auto providesAllFields(const OrderedPathSet& fields, const QuerySolutionNode& solnRoot) {
+auto providesAllFields(const std::set<std::string>& fields, const QuerySolutionNode& solnRoot) {
for (auto&& field : fields) {
if (!solnRoot.hasField(field))
return false;
@@ -580,10 +580,6 @@ void removeProjectSimpleBelowGroupRecursive(QuerySolutionNode* solnRoot) {
if (solnRoot->getType() == StageType::STAGE_GROUP) {
auto groupNode = static_cast<GroupNode*>(solnRoot);
- if (groupNode->needWholeDocument) {
- // The sub expression needs the whole document.
- return;
- }
auto projectNodeCandidate = groupNode->children[0];
if (projectNodeCandidate->getType() == StageType::STAGE_GROUP) {
// Multiple $group stages may be pushed down. So, if the child is a GROUP, then recurse.
diff --git a/src/mongo/db/query/planner_ixselect.cpp b/src/mongo/db/query/planner_ixselect.cpp
index 755077517af..518da370750 100644
--- a/src/mongo/db/query/planner_ixselect.cpp
+++ b/src/mongo/db/query/planner_ixselect.cpp
@@ -245,7 +245,7 @@ static bool boundsGeneratingNodeContainsComparisonToType(MatchExpression* node,
// static
void QueryPlannerIXSelect::getFields(const MatchExpression* node,
string prefix,
- RelevantFieldIndexMap* out) {
+ stdx::unordered_set<string>* out) {
// Do not traverse tree beyond a NOR negation node
MatchExpression::MatchType exprtype = node->matchType();
if (exprtype == MatchExpression::NOR) {
@@ -254,12 +254,16 @@ void QueryPlannerIXSelect::getFields(const MatchExpression* node,
// Leaf nodes with a path and some array operators.
if (Indexability::nodeCanUseIndexOnOwnField(node)) {
- bool supportSparse = Indexability::nodeSupportedBySparseIndex(node);
- (*out)[prefix + node->path().toString()] = {supportSparse};
- } else if (Indexability::isBoundsGeneratingElemMatchObject(node)) {
+ out->insert(prefix + node->path().toString());
+ } else if (Indexability::arrayUsesIndexOnChildren(node)) {
// If the array uses an index on its children, it's something like
// {foo : {$elemMatch: {bar: 1}}}, in which case the predicate is really over foo.bar.
- prefix += node->path().toString() + ".";
+ //
+ // When we have {foo: {$all: [{$elemMatch: {a: 1}}], the path of the embedded elemMatch
+ // is empty. We don't want to append a dot in that case as the field would be foo..a.
+ if (!node->path().empty()) {
+ prefix += node->path().toString() + ".";
+ }
for (size_t i = 0; i < node->numChildren(); ++i) {
getFields(node->getChild(i), prefix, out);
@@ -271,7 +275,8 @@ void QueryPlannerIXSelect::getFields(const MatchExpression* node,
}
}
-void QueryPlannerIXSelect::getFields(const MatchExpression* node, RelevantFieldIndexMap* out) {
+void QueryPlannerIXSelect::getFields(const MatchExpression* node,
+ stdx::unordered_set<string>* out) {
getFields(node, "", out);
}
@@ -311,40 +316,26 @@ std::vector<IndexEntry> QueryPlannerIXSelect::findIndexesByHint(
// static
std::vector<IndexEntry> QueryPlannerIXSelect::findRelevantIndices(
- const RelevantFieldIndexMap& fields, const std::vector<IndexEntry>& allIndices) {
+ const stdx::unordered_set<std::string>& fields, const std::vector<IndexEntry>& allIndices) {
std::vector<IndexEntry> out;
- for (auto&& index : allIndices) {
- BSONObjIterator it(index.keyPattern);
+ for (auto&& entry : allIndices) {
+ BSONObjIterator it(entry.keyPattern);
BSONElement elt = it.next();
- const std::string fieldName = elt.fieldNameStringData().toString();
-
- // If the index is non-sparse we can use the field regardless its sparsity, otherwise we
- // should find the field that can be answered by a sparse index.
- if (fields.contains(fieldName) &&
- (!index.sparse || fields.find(fieldName)->second.isSparse)) {
- out.push_back(index);
+ if (fields.end() != fields.find(elt.fieldName())) {
+ out.push_back(entry);
}
}
return out;
}
-std::vector<IndexEntry> QueryPlannerIXSelect::expandIndexes(const RelevantFieldIndexMap& fields,
- std::vector<IndexEntry> relevantIndices,
- bool indexHinted) {
+std::vector<IndexEntry> QueryPlannerIXSelect::expandIndexes(
+ const stdx::unordered_set<std::string>& fields, std::vector<IndexEntry> relevantIndices) {
std::vector<IndexEntry> out;
- // Filter out fields that cannot be answered by any sparse index. We know wildcard indexes are
- // sparse, so we don't want to expand the wildcard index based on such fields.
- stdx::unordered_set<std::string> sparseIncompatibleFields;
- for (auto&& [fieldName, idxProperty] : fields) {
- if (idxProperty.isSparse || indexHinted) {
- sparseIncompatibleFields.insert(fieldName);
- }
- }
for (auto&& entry : relevantIndices) {
if (entry.type == IndexType::INDEX_WILDCARD) {
- wcp::expandWildcardIndexEntry(entry, sparseIncompatibleFields, &out);
+ wcp::expandWildcardIndexEntry(entry, fields, &out);
} else {
out.push_back(std::move(entry));
}
@@ -789,8 +780,7 @@ void QueryPlannerIXSelect::_rateIndices(MatchExpression* node,
childRt->path = rt->path;
node->getChild(0)->setTag(childRt);
}
- } else if (Indexability::arrayUsesIndexOnChildren(node) && !node->path().empty()) {
- // Note we skip empty path components since they are not allowed in index key patterns.
+ } else if (Indexability::arrayUsesIndexOnChildren(node)) {
const auto newPath = prefix + node->path().toString();
ElemMatchContext newContext;
// Note this StringData is unowned and references the string declared on the stack here.
@@ -801,7 +791,12 @@ void QueryPlannerIXSelect::_rateIndices(MatchExpression* node,
// If the array uses an index on its children, it's something like
// {foo: {$elemMatch: {bar: 1}}}, in which case the predicate is really over foo.bar.
- prefix += node->path().toString() + ".";
+ //
+ // When we have {foo: {$all: [{$elemMatch: {a: 1}}], the path of the embedded elemMatch
+ // is empty. We don't want to append a dot in that case as the field would be foo..a.
+ if (!node->path().empty()) {
+ prefix += node->path().toString() + ".";
+ }
for (size_t i = 0; i < node->numChildren(); ++i) {
_rateIndices(node->getChild(i), prefix, indices, collator, newContext);
}
@@ -924,10 +919,7 @@ void QueryPlannerIXSelect::stripUnneededAssignments(MatchExpression* node,
*/
static void removeIndexRelevantTag(MatchExpression* node, size_t idx) {
RelevantTag* tag = static_cast<RelevantTag*>(node->getTag());
- if (!tag) {
- return;
- }
-
+ verify(tag);
vector<size_t>::iterator firstIt = std::find(tag->first.begin(), tag->first.end(), idx);
if (firstIt != tag->first.end()) {
tag->first.erase(firstIt);
@@ -952,8 +944,9 @@ void stripInvalidAssignmentsToPartialIndexNode(MatchExpression* node,
size_t idxNo,
const IndexEntry& idxEntry,
bool inNegationOrElemMatchObj) {
- removeIndexRelevantTag(node, idxNo);
-
+ if (node->getTag()) {
+ removeIndexRelevantTag(node, idxNo);
+ }
inNegationOrElemMatchObj |= nodeIsNegationOrElemMatchObj(node);
for (size_t i = 0; i < node->numChildren(); ++i) {
// If 'node' is an OR and our current clause satisfies the filter expression, then we may be
diff --git a/src/mongo/db/query/planner_ixselect.h b/src/mongo/db/query/planner_ixselect.h
index 15f1e135d5e..0ef2d480953 100644
--- a/src/mongo/db/query/planner_ixselect.h
+++ b/src/mongo/db/query/planner_ixselect.h
@@ -38,24 +38,16 @@ namespace mongo {
class CollatorInterface;
-struct IndexProperties {
- bool isSparse = false; // 'true' if a sparse index can answer the field.
-};
-
-// A relevant field to index requirement map.
-using RelevantFieldIndexMap = stdx::unordered_map<std::string, IndexProperties>;
-
/**
* Methods for determining what fields and predicates can use indices.
*/
class QueryPlannerIXSelect {
public:
/**
- * Return all the fields in the tree rooted at 'node' that we can use an index to answer the
- * query. The output, 'RelevantFieldIndexMap', contains the requirements of the index that can
- * answer the field. e.g. Some fields can be supported only by a non-sparse index.
+ * Return all the fields in the tree rooted at 'node' that we can use an index on
+ * in order to answer the query.
*/
- static void getFields(const MatchExpression* node, RelevantFieldIndexMap* out);
+ static void getFields(const MatchExpression* node, stdx::unordered_set<std::string>* out);
/**
* Similar to other getFields() method, but with 'prefix' argument which is a path prefix to be
@@ -65,7 +57,7 @@ public:
*/
static void getFields(const MatchExpression* node,
std::string prefix,
- RelevantFieldIndexMap* out);
+ stdx::unordered_set<std::string>* out);
/**
* Finds all indices that correspond to the hinted index. Matches the index both by name and by
@@ -78,8 +70,8 @@ public:
* Finds all indices prefixed by fields we have predicates over. Only these indices are
* useful in answering the query.
*/
- static std::vector<IndexEntry> findRelevantIndices(const RelevantFieldIndexMap& fields,
- const std::vector<IndexEntry>& allIndices);
+ static std::vector<IndexEntry> findRelevantIndices(
+ const stdx::unordered_set<std::string>& fields, const std::vector<IndexEntry>& allIndices);
/**
* Determine how useful all of our relevant 'indices' are to all predicates in the subtree
@@ -138,12 +130,9 @@ public:
/**
* Given a list of IndexEntries and fields used by a query's match expression, return a list
* "expanded" indexes (where the $** indexes in the given list have been expanded).
- * 'hintedIndexBson' indicates that the indexes in 'relevantIndices' are the results of the
- * user's hint.
*/
- static std::vector<IndexEntry> expandIndexes(const RelevantFieldIndexMap& fields,
- std::vector<IndexEntry> relevantIndices,
- bool hintedIndexBson = false);
+ static std::vector<IndexEntry> expandIndexes(const stdx::unordered_set<std::string>& fields,
+ std::vector<IndexEntry> relevantIndices);
/**
* Check if this match expression is a leaf and is supported by a wildcard index.
diff --git a/src/mongo/db/query/planner_ixselect_test.cpp b/src/mongo/db/query/planner_ixselect_test.cpp
index 1df4d714e67..93c4f12c821 100644
--- a/src/mongo/db/query/planner_ixselect_test.cpp
+++ b/src/mongo/db/query/planner_ixselect_test.cpp
@@ -65,20 +65,6 @@ unique_ptr<MatchExpression> parseMatchExpression(const BSONObj& obj) {
return std::move(status.getValue());
}
-using FieldIter = RelevantFieldIndexMap::iterator;
-string toString(FieldIter begin, FieldIter end) {
- str::stream ss;
- ss << "[";
- for (FieldIter i = begin; i != end; i++) {
- if (i != begin) {
- ss << " ";
- }
- ss << i->first;
- }
- ss << "]";
- return ss;
-}
-
/**
* Utility function to join elements in iterator range with comma
*/
@@ -102,13 +88,10 @@ string toString(Iter begin, Iter end) {
* to QueryPlannerIXSelect::getFields()
* Results are compared with expected fields (parsed from expectedFieldsStr)
*/
-void testGetFields(const char* query,
- const char* prefix,
- const char* expectedFieldsStr,
- bool sparseSupported = true) {
+void testGetFields(const char* query, const char* prefix, const char* expectedFieldsStr) {
BSONObj obj = fromjson(query);
unique_ptr<MatchExpression> expr(parseMatchExpression(obj));
- RelevantFieldIndexMap fields;
+ stdx::unordered_set<string> fields;
QueryPlannerIXSelect::getFields(expr.get(), prefix, &fields);
// Verify results
@@ -116,7 +99,7 @@ void testGetFields(const char* query,
vector<string> expectedFields = StringSplitter::split(expectedFieldsStr, ",");
for (vector<string>::const_iterator i = expectedFields.begin(); i != expectedFields.end();
i++) {
- if (fields[*i].isSparse != sparseSupported) {
+ if (fields.find(*i) == fields.end()) {
str::stream ss;
ss << "getFields(query=" << query << ", prefix=" << prefix << "): unable to find " << *i
<< " in result: " << toString(fields.begin(), fields.end());
@@ -176,12 +159,6 @@ TEST(QueryPlannerIXSelectTest, GetFieldsArrayNegation) {
testGetFields("{a: {$all: [{$elemMatch: {b: {$ne: 1}}}]}}", "", "a.b");
}
-TEST(QueryPlannerIXSelectTest, GetFieldsInternalExpr) {
- testGetFields("{$expr: {$lt: ['$a', 'r']}}", "", "", false /* sparse supported */);
- testGetFields("{$expr: {$eq: ['$a', null]}}", "", "", false /* sparse supported */);
- testGetFields("{$expr: {$eq: ['$a', 1]}}", "", "", false /* sparse supported */);
-}
-
/**
* Performs a pre-order traversal of expression tree. Validates
* that all tagged nodes contain an instance of RelevantTag.
@@ -1176,6 +1153,25 @@ TEST(QueryPlannerIXSelectTest, InternalExprEqCanUseTextIndexSuffix) {
"{a: {$_internalExprEq: 1}}", "", kSimpleCollator, indices, "a", expectedIndices);
}
+TEST(QueryPlannerIXSelectTest, InternalExprEqCanUseSparseIndexWithComparisonToNull) {
+ auto entry = buildSimpleIndexEntry(BSON("a" << 1));
+ entry.sparse = true;
+ std::vector<IndexEntry> indices;
+ indices.push_back(entry);
+ std::set<size_t> expectedIndices = {0};
+ testRateIndices(
+ "{a: {$_internalExprEq: null}}", "", kSimpleCollator, indices, "a", expectedIndices);
+}
+
+TEST(QueryPlannerIXSelectTest, InternalExprEqCanUseSparseIndexWithComparisonToNonNull) {
+ auto entry = buildSimpleIndexEntry(BSON("a" << 1));
+ entry.sparse = true;
+ std::vector<IndexEntry> indices;
+ indices.push_back(entry);
+ std::set<size_t> expectedIndices = {0};
+ testRateIndices(
+ "{a: {$_internalExprEq: 1}}", "", kSimpleCollator, indices, "a", expectedIndices);
+}
TEST(QueryPlannerIXSelectTest, NotEqualsNullCanUseIndex) {
auto entry = buildSimpleIndexEntry(BSON("a" << 1));
std::set<size_t> expectedIndices = {0};
@@ -1361,17 +1357,18 @@ TEST(QueryPlannerIXSelectTest, ExpandWildcardIndices) {
const auto indexEntry = makeIndexEntry(BSON("$**" << 1), {});
// Case where no fields are specified.
- std::vector<IndexEntry> result = QueryPlannerIXSelect::expandIndexes({}, {indexEntry.first});
+ std::vector<IndexEntry> result =
+ QueryPlannerIXSelect::expandIndexes(stdx::unordered_set<string>(), {indexEntry.first});
ASSERT_TRUE(result.empty());
- RelevantFieldIndexMap fields = {{"fieldA", {true}}, {"fieldB", {true}}};
+ stdx::unordered_set<string> fields = {"fieldA", "fieldB"};
result = QueryPlannerIXSelect::expandIndexes(fields, {indexEntry.first});
std::vector<BSONObj> expectedKeyPatterns = {BSON("fieldA" << 1), BSON("fieldB" << 1)};
ASSERT_TRUE(indexEntryKeyPatternsMatch(&expectedKeyPatterns, &result));
const auto wildcardIndexWithSubpath = makeIndexEntry(BSON("a.b.$**" << 1), {});
- fields = {{"a.b", {true}}, {"a.b.c", {true}}, {"a.d", {true}}};
+ fields = {"a.b", "a.b.c", "a.d"};
result = QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexWithSubpath.first});
expectedKeyPatterns = {BSON("a.b" << 1), BSON("a.b.c" << 1)};
ASSERT_TRUE(indexEntryKeyPatternsMatch(&expectedKeyPatterns, &result));
@@ -1383,8 +1380,7 @@ TEST(QueryPlannerIXSelectTest, ExpandWildcardIndicesInPresenceOfOtherIndices) {
auto bIndexEntry = makeIndexEntry(BSON("fieldB" << 1), {});
auto abIndexEntry = makeIndexEntry(BSON("fieldA" << 1 << "fieldB" << 1), {});
- const RelevantFieldIndexMap fields = {
- {"fieldA", {true}}, {"fieldB", {true}}, {"fieldC", {true}}};
+ const stdx::unordered_set<string> fields = {"fieldA", "fieldB", "fieldC"};
std::vector<BSONObj> expectedKeyPatterns = {
BSON("fieldA" << 1), BSON("fieldA" << 1), BSON("fieldB" << 1), BSON("fieldC" << 1)};
@@ -1422,7 +1418,7 @@ TEST(QueryPlannerIXSelectTest, ExpandWildcardIndicesInPresenceOfOtherIndices) {
TEST(QueryPlannerIXSelectTest, ExpandedIndexEntriesAreCorrectlyMarkedAsMultikeyOrNonMultikey) {
auto wildcardIndexEntry = makeIndexEntry(BSON("$**" << 1), {}, {FieldRef{"a"}});
- RelevantFieldIndexMap fields = {{"a.b", {true}}, {"c.d", {true}}};
+ const stdx::unordered_set<string> fields = {"a.b", "c.d"};
std::vector<BSONObj> expectedKeyPatterns = {BSON("a.b" << 1), BSON("c.d" << 1)};
auto result = QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexEntry.first});
@@ -1446,7 +1442,7 @@ TEST(QueryPlannerIXSelectTest, ExpandedIndexEntriesAreCorrectlyMarkedAsMultikeyO
TEST(QueryPlannerIXSelectTest, WildcardIndexExpansionExcludesIdField) {
const auto indexEntry = makeIndexEntry(BSON("$**" << 1), {});
- RelevantFieldIndexMap fields = {{"_id", {true}}, {"abc", {true}}, {"def", {true}}};
+ stdx::unordered_set<string> fields = {"_id", "abc", "def"};
std::vector<IndexEntry> result =
QueryPlannerIXSelect::expandIndexes(fields, {indexEntry.first});
@@ -1458,7 +1454,7 @@ TEST(QueryPlannerIXSelectTest, WildcardIndicesExpandedEntryHasCorrectProperties)
auto wildcardIndexEntry = makeIndexEntry(BSON("$**" << 1), {});
wildcardIndexEntry.first.identifier = IndexEntry::Identifier("someIndex");
- RelevantFieldIndexMap fields = {{"abc", {true}}, {"def", {true}}};
+ stdx::unordered_set<string> fields = {"abc", "def"};
std::vector<IndexEntry> result =
QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexEntry.first});
@@ -1488,11 +1484,7 @@ TEST(QueryPlannerIXSelectTest, WildcardIndicesExpandedEntryHasCorrectProperties)
TEST(QueryPlannerIXSelectTest, WildcardIndicesExcludeNonMatchingKeySubpath) {
auto wildcardIndexEntry = makeIndexEntry(BSON("subpath.$**" << 1), {});
- RelevantFieldIndexMap fields = {{"abc", {true}},
- {"def", {true}},
- {"subpath.abc", {true}},
- {"subpath.def", {true}},
- {"subpath", {true}}};
+ stdx::unordered_set<string> fields = {"abc", "def", "subpath.abc", "subpath.def", "subpath"};
std::vector<IndexEntry> result =
QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexEntry.first});
@@ -1508,11 +1500,7 @@ TEST(QueryPlannerIXSelectTest, WildcardIndicesExcludeNonMatchingPathsWithInclusi
{},
BSON("wildcardProjection" << BSON("abc" << 1 << "subpath.abc" << 1)));
- RelevantFieldIndexMap fields = {{"abc", {true}},
- {"def", {true}},
- {"subpath.abc", {true}},
- {"subpath.def", {true}},
- {"subpath", {true}}};
+ stdx::unordered_set<string> fields = {"abc", "def", "subpath.abc", "subpath.def", "subpath"};
std::vector<IndexEntry> result =
QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexEntry.first});
@@ -1527,11 +1515,7 @@ TEST(QueryPlannerIXSelectTest, WildcardIndicesExcludeNonMatchingPathsWithExclusi
{},
BSON("wildcardProjection" << BSON("abc" << 0 << "subpath.abc" << 0)));
- RelevantFieldIndexMap fields = {{"abc", {true}},
- {"def", {true}},
- {"subpath.abc", {true}},
- {"subpath.def", {true}},
- {"subpath", {true}}};
+ stdx::unordered_set<string> fields = {"abc", "def", "subpath.abc", "subpath.def", "subpath"};
std::vector<IndexEntry> result =
QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexEntry.first});
@@ -1547,12 +1531,8 @@ TEST(QueryPlannerIXSelectTest, WildcardIndicesWithInclusionProjectionAllowIdExcl
{},
BSON("wildcardProjection" << BSON("_id" << 0 << "abc" << 1 << "subpath.abc" << 1)));
- RelevantFieldIndexMap fields = {{"_id", {true}},
- {"abc", {true}},
- {"def", {true}},
- {"subpath.abc", {true}},
- {"subpath.def", {true}},
- {"subpath", {true}}};
+ stdx::unordered_set<string> fields = {
+ "_id", "abc", "def", "subpath.abc", "subpath.def", "subpath"};
std::vector<IndexEntry> result =
QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexEntry.first});
@@ -1567,12 +1547,8 @@ TEST(QueryPlannerIXSelectTest, WildcardIndicesWithInclusionProjectionAllowIdIncl
{},
BSON("wildcardProjection" << BSON("_id" << 1 << "abc" << 1 << "subpath.abc" << 1)));
- RelevantFieldIndexMap fields = {{"_id", {true}},
- {"abc", {true}},
- {"def", {true}},
- {"subpath.abc", {true}},
- {"subpath.def", {true}},
- {"subpath", {true}}};
+ stdx::unordered_set<string> fields = {
+ "_id", "abc", "def", "subpath.abc", "subpath.def", "subpath"};
std::vector<IndexEntry> result =
QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexEntry.first});
@@ -1588,12 +1564,8 @@ TEST(QueryPlannerIXSelectTest, WildcardIndicesWithExclusionProjectionAllowIdIncl
{},
BSON("wildcardProjection" << BSON("_id" << 1 << "abc" << 0 << "subpath.abc" << 0)));
- RelevantFieldIndexMap fields = {{"_id", {true}},
- {"abc", {true}},
- {"def", {true}},
- {"subpath.abc", {true}},
- {"subpath.def", {true}},
- {"subpath", {true}}};
+ stdx::unordered_set<string> fields = {
+ "_id", "abc", "def", "subpath.abc", "subpath.def", "subpath"};
std::vector<IndexEntry> result =
QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexEntry.first});
@@ -1606,12 +1578,8 @@ TEST(QueryPlannerIXSelectTest, WildcardIndicesIncludeMatchingInternalNodes) {
auto wildcardIndexEntry = makeIndexEntry(
BSON("$**" << 1), {}, {}, BSON("wildcardProjection" << BSON("_id" << 1 << "subpath" << 1)));
- RelevantFieldIndexMap fields = {{"_id", {true}},
- {"abc", {true}},
- {"def", {true}},
- {"subpath.abc", {true}},
- {"subpath.def", {true}},
- {"subpath", {true}}};
+ stdx::unordered_set<string> fields = {
+ "_id", "abc", "def", "subpath.abc", "subpath.def", "subpath"};
std::vector<IndexEntry> result =
QueryPlannerIXSelect::expandIndexes(fields, {wildcardIndexEntry.first});
diff --git a/src/mongo/db/query/projection.cpp b/src/mongo/db/query/projection.cpp
index 2646b39bd8c..af5fedfe780 100644
--- a/src/mongo/db/query/projection.cpp
+++ b/src/mongo/db/query/projection.cpp
@@ -49,7 +49,7 @@ struct DepsAnalysisData {
fieldDependencyTracker.fields.insert(fieldName);
}
- OrderedPathSet requiredFields() const {
+ std::set<std::string> requiredFields() const {
return fieldDependencyTracker.fields;
}
};
@@ -227,13 +227,6 @@ void optimizeProjection(ProjectionPathASTNode* root) {
Projection::Projection(ProjectionPathASTNode root, ProjectType type)
: _root(std::move(root)), _type(type), _deps(analyzeProjection(&_root, type)) {}
-void Projection::optimize() {
- if (!_projOptimized) {
- optimizeProjection(&_root);
- _deps = analyzeProjection(&_root, _type);
- _projOptimized = true;
- }
-}
namespace {
/**
diff --git a/src/mongo/db/query/projection.h b/src/mongo/db/query/projection.h
index 95d41caba02..914567e87ed 100644
--- a/src/mongo/db/query/projection.h
+++ b/src/mongo/db/query/projection.h
@@ -49,7 +49,7 @@ struct ProjectionDependencies {
bool containsElemMatch = false;
// Which fields are necessary to perform the projection, or boost::none if all are required.
- boost::optional<OrderedPathSet> requiredFields;
+ boost::optional<std::set<std::string>> requiredFields;
bool hasDottedPath = false;
@@ -95,7 +95,7 @@ public:
* Return which fields are required to compute the projection, assuming the entire document is
* not needed.
*/
- const OrderedPathSet& getRequiredFields() const {
+ const std::set<std::string>& getRequiredFields() const {
invariant(_type == ProjectType::kInclusion);
return *_deps.requiredFields;
}
@@ -142,25 +142,14 @@ public:
return _deps.containsElemMatch;
}
- /**
- * Optimizes the projection tree. Additionally, re-computes dependencies in case anything
- * changes as in projection {x: {$and: [false, "$b"]}} - which when optimized will no longer
- * depend on "b".
- */
- void optimize();
-
private:
ProjectionPathASTNode _root;
ProjectType _type;
ProjectionDependencies _deps;
- bool _projOptimized = false;
};
/**
- * Walks the projection AST and optimizes each node. Note if you have a 'Projection' instance you
- * should prefer to use Projection::optimize() since it will additionally re-compute dependencies in
- * case anything changes as in projection {x: {$and: [false, "$b"]}} - which when optimized will no
- * longer depend on "b".
+ * Walks the projection AST and optimizes each node.
*/
void optimizeProjection(ProjectionPathASTNode* root);
diff --git a/src/mongo/db/query/projection_ast.h b/src/mongo/db/query/projection_ast.h
index 1304c68efe5..247a91537f3 100644
--- a/src/mongo/db/query/projection_ast.h
+++ b/src/mongo/db/query/projection_ast.h
@@ -275,7 +275,7 @@ public:
ExpressionASTNode(boost::intrusive_ptr<Expression> expr) : _expr(expr) {}
ExpressionASTNode(const ExpressionASTNode& other) : ASTNode(other) {
BSONObjBuilder bob;
- bob << "" << other._expr->serialize();
+ bob << "" << other._expr->serialize(false);
// TODO SERVER-31003: add a clone() method to Expression.
// Temporary stop expression counters while processing the cloned expression.
diff --git a/src/mongo/db/query/projection_ast_test.cpp b/src/mongo/db/query/projection_ast_test.cpp
index ea1eb2b683d..619bd9a1dde 100644
--- a/src/mongo/db/query/projection_ast_test.cpp
+++ b/src/mongo/db/query/projection_ast_test.cpp
@@ -41,7 +41,6 @@
#include "mongo/db/query/projection_ast_util.h"
#include "mongo/db/query/projection_parser.h"
#include "mongo/db/query/query_planner_test_fixture.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
namespace {
@@ -773,62 +772,4 @@ TEST_F(ProjectionASTTest, ShouldThrowWithPositionalOnExclusion) {
DBException,
31395);
}
-
-TEST_F(ProjectionASTTest, TestASTRedaction) {
- SerializationOptions options = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST;
-
- auto proj = fromjson("{'a.b': 1}");
- BSONObj output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options);
- ASSERT_BSONOBJ_EQ_AUTO( //
- R"({"HASH<a>":{"HASH<b>":true},"HASH<_id>":true})",
- output);
-
- proj = fromjson("{'a.b': 0}");
- output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options);
- ASSERT_BSONOBJ_EQ_AUTO( //
- R"({"HASH<a>":{"HASH<b>":false}})",
- output);
-
- proj = fromjson("{a: 1, b: 1}");
- output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options);
- ASSERT_BSONOBJ_EQ_AUTO( //
- R"({"HASH<a>":true,"HASH<b>":true,"HASH<_id>":true})",
- output);
-
- // ElemMatch projection
- proj = fromjson("{f: {$elemMatch: {foo: 'bar'}}}");
- output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options);
- ASSERT_BSONOBJ_EQ_AUTO( //
- R"({"HASH<f>":{"$elemMatch":{"HASH<foo>":{"$eq":"?string"}}},"HASH<_id>":true})",
- output);
-
- // Positional projection
- proj = fromjson("{'x.$': 1}");
- output = projection_ast::serialize(
- *parseWithFindFeaturesEnabled(proj, fromjson("{'x.a': 2}")).root(), {});
- ASSERT_BSONOBJ_EQ_AUTO( //
- R"({"x.$":true,"_id":true})",
- output);
-
- // Slice (first form)
- proj = fromjson("{a: {$slice: 1}}");
- output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options);
- ASSERT_BSONOBJ_EQ_AUTO( //
- R"({"HASH<a>":{"$slice":"?number"}})",
- output);
-
- // Slice (second form)
- proj = fromjson("{a: {$slice: [1, 3]}}");
- output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options);
- ASSERT_BSONOBJ_EQ_AUTO( //
- R"({"HASH<a>":{"$slice":["?number","?number"]}})",
- output);
-
- /// $meta projection
- proj = fromjson("{foo: {$meta: 'indexKey'}}");
- output = projection_ast::serialize(*parseWithFindFeaturesEnabled(proj).root(), options);
- ASSERT_BSONOBJ_EQ_AUTO( //
- R"({"HASH<foo>":{"$meta":"indexKey"}})",
- output);
-}
} // namespace
diff --git a/src/mongo/db/query/projection_ast_util.cpp b/src/mongo/db/query/projection_ast_util.cpp
index 23c7b6d9582..e5b4cc1a9c4 100644
--- a/src/mongo/db/query/projection_ast_util.cpp
+++ b/src/mongo/db/query/projection_ast_util.cpp
@@ -29,26 +29,28 @@
#include "mongo/platform/basic.h"
-#include "mongo/db/query/projection_ast_path_tracking_visitor.h"
#include "mongo/db/query/projection_ast_util.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
+
+#include "mongo/db/query/projection_ast_path_tracking_visitor.h"
#include "mongo/db/query/tree_walker.h"
namespace mongo::projection_ast {
namespace {
struct BSONVisitorContext {
std::stack<BSONObjBuilder> builders;
- bool underElemMatch = false;
};
class BSONPreVisitor : public ProjectionASTConstVisitor {
public:
- using ProjectionASTConstVisitor::visit;
- BSONPreVisitor(PathTrackingVisitorContext<BSONVisitorContext>* context,
- SerializationOptions options)
- : _context(context), _builders(context->data().builders), _options(std::move(options)) {}
+ BSONPreVisitor(PathTrackingVisitorContext<BSONVisitorContext>* context)
+ : _context(context), _builders(context->data().builders) {}
+
+ virtual void visit(const MatchExpressionASTNode* node) {
+ static_cast<const MatchExpressionASTNode*>(node)->matchExpression()->serialize(
+ &_builders.top(), true);
+ }
- void visit(const ProjectionPathASTNode* node) override {
+ virtual void visit(const ProjectionPathASTNode* node) {
if (!node->parent()) {
// No root of the tree, thus this node has no field name.
_builders.push(BSONObjBuilder());
@@ -57,46 +59,47 @@ public:
}
}
- void visit(const ProjectionSliceASTNode* node) override {
+ virtual void visit(const ProjectionPositionalASTNode* node) {
+ // ProjectionPositional always has the original query's match expression node as its
+ // child. Serialize as: {"positional.projection.field.$": <original match expression>}.
+ _context->data().builders.push(_builders.top().subobjStart(getFieldName() + ".$"));
+ }
+
+ virtual void visit(const ProjectionSliceASTNode* node) {
BSONObjBuilder sub(_builders.top().subobjStart(getFieldName()));
if (node->skip()) {
- sub.appendArray("$slice",
- BSON_ARRAY(_options.serializeLiteral(*node->skip())
- << _options.serializeLiteral(node->limit())));
+ sub.appendArray("$slice", BSON_ARRAY(*node->skip() << node->limit()));
} else {
- _options.appendLiteral(&sub, "$slice", node->limit());
+ sub.appendNumber("$slice", node->limit());
}
}
+ virtual void visit(const ProjectionElemMatchASTNode* node) {
+ // Defer to the child, match expression node.
+ }
- void visit(const ExpressionASTNode* node) override {
- node->expression()->serialize(_options).addToBsonObj(&_builders.top(), getFieldName());
+ virtual void visit(const ExpressionASTNode* node) {
+ node->expression()->serialize(false).addToBsonObj(&_builders.top(), getFieldName());
}
- void visit(const BooleanConstantASTNode* node) override {
+ virtual void visit(const BooleanConstantASTNode* node) {
_builders.top().append(getFieldName(), node->value());
}
- void visit(const ProjectionPositionalASTNode* node) override = 0;
- void visit(const ProjectionElemMatchASTNode* node) override = 0;
- void visit(const MatchExpressionASTNode* node) override = 0;
-
-protected:
+private:
std::string getFieldName() {
- return _options.serializeFieldPathFromString(_context->childPath());
+ return _context->childPath();
}
PathTrackingVisitorContext<BSONVisitorContext>* _context;
std::stack<BSONObjBuilder>& _builders;
- SerializationOptions _options;
};
class BSONPostVisitor : public ProjectionASTConstVisitor {
public:
- using ProjectionASTConstVisitor::visit;
BSONPostVisitor(BSONVisitorContext* context) : _context(context) {}
- void visit(const ProjectionPathASTNode* node) override {
+ virtual void visit(const ProjectionPathASTNode* node) {
// Don't pop the top builder.
if (node->parent()) {
// Pop the BSONObjBuilder that was added in the pre visitor.
@@ -104,97 +107,25 @@ public:
}
}
- void visit(const ProjectionSliceASTNode* node) override {}
- void visit(const ExpressionASTNode* node) override {}
- void visit(const BooleanConstantASTNode* node) override {}
- void visit(const MatchExpressionASTNode* node) override {}
-
- void visit(const ProjectionPositionalASTNode* node) override = 0;
- void visit(const ProjectionElemMatchASTNode* node) override = 0;
-
-protected:
- BSONVisitorContext* _context;
-};
-
-class DebugPreVisitor : public BSONPreVisitor {
-public:
- using BSONPreVisitor::visit;
- DebugPreVisitor(PathTrackingVisitorContext<BSONVisitorContext>* context)
- : BSONPreVisitor(context, SerializationOptions{}) {}
-
- void visit(const ProjectionPositionalASTNode* node) override {
- // ProjectionPositional always has the original query's match expression node as its
- // child. Serialize as: {"positional.projection.field.$": <original match expression>}.
- _context->data().builders.push(_builders.top().subobjStart(getFieldName() + ".$"));
- }
-
- void visit(const ProjectionElemMatchASTNode* node) override {
- // Defer to the child, match expression node.
- }
-
- void visit(const MatchExpressionASTNode* node) override {
- static_cast<const MatchExpressionASTNode*>(node)->matchExpression()->serialize(
- &_builders.top(), {});
- }
-};
-
-class DebugPostVisitor : public BSONPostVisitor {
-public:
- using BSONPostVisitor::visit;
- DebugPostVisitor(BSONVisitorContext* context) : BSONPostVisitor(context) {}
-
- void visit(const ProjectionPositionalASTNode* node) override {
+ virtual void visit(const ProjectionPositionalASTNode* node) {
_context->builders.pop();
}
- void visit(const ProjectionElemMatchASTNode* node) override {}
-};
+ virtual void visit(const MatchExpressionASTNode* node) {}
+ virtual void visit(const ProjectionSliceASTNode* node) {}
+ virtual void visit(const ProjectionElemMatchASTNode* node) {}
+ virtual void visit(const ExpressionASTNode* node) {}
+ virtual void visit(const BooleanConstantASTNode* node) {}
-class SerializationPreVisitor : public BSONPreVisitor {
-public:
- using BSONPreVisitor::visit;
- SerializationPreVisitor(PathTrackingVisitorContext<BSONVisitorContext>* context,
- const SerializationOptions& options)
- : BSONPreVisitor(context, options) {}
-
- void visit(const ProjectionPositionalASTNode* node) override {
- tassert(73488,
- "Positional projection should not appear below an $elemMatch projection.",
- !_context->data().underElemMatch);
- _builders.top().append(getFieldName() + ".$", true);
- }
-
- void visit(const ProjectionElemMatchASTNode* node) override {
- // The child match expression node should begin with $elemMatch.
- _context->data().underElemMatch = true;
- }
-
- void visit(const MatchExpressionASTNode* node) override {
- if (_context->data().underElemMatch) {
- static_cast<const MatchExpressionASTNode*>(node)->matchExpression()->serialize(
- &_builders.top(), _options);
- }
- }
-};
-
-class SerializationPostVisitor : public BSONPostVisitor {
-public:
- using BSONPostVisitor::visit;
- SerializationPostVisitor(BSONVisitorContext* context) : BSONPostVisitor(context) {}
-
- void visit(const ProjectionPositionalASTNode* node) override {}
- void visit(const ProjectionElemMatchASTNode* node) override {
- _context->underElemMatch = false;
- }
+private:
+ BSONVisitorContext* _context;
};
-
} // namespace
BSONObj astToDebugBSON(const ASTNode* root) {
PathTrackingVisitorContext<BSONVisitorContext> context;
- DebugPreVisitor preVisitor{&context};
- DebugPostVisitor postVisitor{&context.data()};
-
+ BSONPreVisitor preVisitor{&context};
+ BSONPostVisitor postVisitor{&context.data()};
PathTrackingWalker walker{&context, {&preVisitor}, {&postVisitor}};
tree_walker::walk<true, projection_ast::ASTNode>(root, &walker);
@@ -202,15 +133,4 @@ BSONObj astToDebugBSON(const ASTNode* root) {
invariant(context.data().builders.size() == 1);
return context.data().builders.top().obj();
}
-
-BSONObj serialize(const ProjectionPathASTNode& root, const SerializationOptions& options) {
- PathTrackingVisitorContext<BSONVisitorContext> context;
- SerializationPreVisitor preVisitor{&context, options};
- SerializationPostVisitor postVisitor{&context.data()};
- PathTrackingWalker walker{&context, {&preVisitor}, {&postVisitor}};
- tree_walker::walk<true, projection_ast::ASTNode>(&root, &walker);
-
- invariant(context.data().builders.size() == 1);
- return context.data().builders.top().obj();
-}
} // namespace mongo::projection_ast
diff --git a/src/mongo/db/query/projection_ast_util.h b/src/mongo/db/query/projection_ast_util.h
index de102dfbbd7..af89254a9a1 100644
--- a/src/mongo/db/query/projection_ast_util.h
+++ b/src/mongo/db/query/projection_ast_util.h
@@ -29,7 +29,6 @@
#pragma once
-#include "mongo/db/query/projection.h"
#include "mongo/db/query/projection_ast.h"
namespace mongo {
@@ -38,7 +37,5 @@ namespace projection_ast {
* This is intended to be used for debug output, not for serialization.
*/
BSONObj astToDebugBSON(const ASTNode* root);
-
-BSONObj serialize(const ProjectionPathASTNode& root, const SerializationOptions& options);
} // namespace projection_ast
} // namespace mongo
diff --git a/src/mongo/db/query/projection_parser.cpp b/src/mongo/db/query/projection_parser.cpp
index 3495f99f8f2..afb3c12556e 100644
--- a/src/mongo/db/query/projection_parser.cpp
+++ b/src/mongo/db/query/projection_parser.cpp
@@ -510,16 +510,7 @@ void parseSubObject(ParseContext* ctx,
// It was likely intended to be an expression. Check if it's a valid field path or not to
// confirm.
try {
- const auto elementFieldName = obj.firstElementFieldNameStringData();
- if (!hasPositionalOperator(elementFieldName)) {
- FieldPath fp(elementFieldName);
- } else {
- // The 'FieldPath' parser doesn't take positional operators into account, but those
- // are valid path projections so trim it off for this validation.
- StringData pathWithoutPositionalOperator =
- elementFieldName.substr(0, elementFieldName.size() - 2);
- FieldPath fp(pathWithoutPositionalOperator);
- }
+ FieldPath fp(obj.firstElementFieldNameStringData());
} catch (const DBException&) {
uasserted(31325,
str::stream()
@@ -640,8 +631,10 @@ Projection parseAndAnalyze(boost::intrusive_ptr<ExpressionContext> expCtx,
invariant(root.removeChild("_id"));
}
- // Optimize the projection expression if requested.
- if (shouldOptimize) {
+ // Optimize the projection expression if requested and as long as not explicitly disabled
+ // pipeline optimization.
+ auto fp = globalFailPointRegistry().find("disablePipelineOptimization");
+ if (shouldOptimize && !(fp && fp->shouldFail())) {
optimizeProjection(&root);
}
diff --git a/src/mongo/db/query/query_feature_flags.idl b/src/mongo/db/query/query_feature_flags.idl
index 7e5995a022f..5ac4a341a2c 100644
--- a/src/mongo/db/query/query_feature_flags.idl
+++ b/src/mongo/db/query/query_feature_flags.idl
@@ -107,12 +107,6 @@ feature_flags:
default: true
version: 5.3
- featureFlagChangeStreamsFurtherEnrichedEvents:
- description: "Feature flag for enabling the reshardCollection and refineCollectionShardKey events, as well as enhancements to the updateDescription field for update events"
- cpp_varname: gFeatureFlagChangeStreamsFurtherEnrichedEvents
- default: true
- version: 6.0
-
featureFlagCommonQueryFramework:
description: "Feature flag for allowing use of Cascades-based query optimizer"
cpp_varname: gfeatureFlagCommonQueryFramework
@@ -151,27 +145,3 @@ feature_flags:
description: "Feature flag to enable using SBE for a larger number of queries"
cpp_varname: gFeatureFlagSbeFull
default: false
-
- featureFlagQueryStats:
- description: "Feature flag for enabling full queryStats collection."
- cpp_varname: gFeatureFlagQueryStats
- default: true
- version: 6.0
-
- featureFlagShardedSearchCustomSort:
- description: "Feature flag to enable user specified sort for sharded $search queries."
- cpp_varname: gFeatureFlagShardedSearchCustomSort
- default: true
- version: 6.0
-
- featureFlagSearchBatchSizeLimit:
- description: "Feature flag to enable the search batchsize and limit optimization."
- cpp_varname: gFeatureFlagSearchBatchSizeLimit
- default: true
- version: 6.0
-
- featureFlagVectorSearchPublicPreview:
- description: "Feature flag to enable vector search for public preview."
- cpp_varname: gFeatureFlagVectorSearchPublicPreview
- default: true
- version: 6.0
diff --git a/src/mongo/db/query/query_knobs.idl b/src/mongo/db/query/query_knobs.idl
index f0f24d3c3dd..f894629037f 100644
--- a/src/mongo/db/query/query_knobs.idl
+++ b/src/mongo/db/query/query_knobs.idl
@@ -29,9 +29,8 @@
global:
cpp_namespace: "mongo"
cpp_includes:
+ - "mongo/db/query/plan_cache_size_parameter.h"
- "mongo/db/query/sbe_plan_cache_on_parameter_change.h"
- - "mongo/db/query/util/memory_util.h"
- - "mongo/db/query/query_stats/query_stats_on_parameter_change.h"
- "mongo/platform/atomic_proxy.h"
- "mongo/platform/atomic_word.h"
@@ -101,15 +100,6 @@ server_parameters:
gte: 0
on_update: plan_cache_util::clearSbeCacheOnParameterChange
- internalQueryPlanTieBreakingWithIndexHeuristics:
- description: "Tie-breaking rules which choose the plan that fetches the smallest number of
- documents or examines fewest index keys, as well as one that can be assigned the longest
- contiguous prefix of equality predicates in any index."
- set_at: [ startup, runtime ]
- cpp_varname: "internalQueryPlanTieBreakingWithIndexHeuristics"
- cpp_vartype: AtomicWord<bool>
- default: true
-
internalQueryForceIntersectionPlans:
description: "Gives a large ranking bonus to index intersection plans, forcing intersection
plans to be chosen when possible."
@@ -421,7 +411,7 @@ server_parameters:
cpp_varname: "internalInsertMaxBatchSize"
cpp_vartype: AtomicWord<int>
default:
- expr: 64
+ expr: internalQueryExecYieldIterations.load() / 2
is_constexpr: false
validator:
gt: 0
@@ -636,13 +626,6 @@ server_parameters:
validator:
gt: 0
- internalQuerySlotBasedExecutionHashAggForceIncreasedSpilling:
- description: "If true, the HashAgg stage will perform increased spilling."
- set_at: [ startup, runtime ]
- cpp_varname: "internalQuerySlotBasedExecutionHashAggForceIncreasedSpilling"
- cpp_vartype: AtomicWord<bool>
- default: false
-
internalQuerySlotBasedExecutionHashLookupApproxMemoryUseInBytesBeforeSpill:
description: "The max size in bytes that the hash table in a HashLookup stage can be estimated to
be before we spill to disk."
@@ -676,7 +659,7 @@ server_parameters:
set_at: [ startup, runtime ]
cpp_varname: "internalQueryForceClassicEngine"
cpp_vartype: AtomicWord<bool>
- default: true
+ default: false
internalQueryAppendIdToSetWindowFieldsSort:
description: "If true, appends _id to the sort stage generated by desugaring $setWindowFields to
@@ -888,88 +871,6 @@ server_parameters:
default:
expr: false
- internalQueryGlobalProfilingFilter:
- description: "Enables the setProfilingFilterGlobally command."
- set_at: [ startup ]
- cpp_varname: internalQueryGlobalProfilingFilter
- cpp_vartype: AtomicWord<bool>
- default: false
-
-
- internalQueryDocumentSourceWriterBatchExtraReservedBytes:
- description: "Space to reserve in document source writer batches for miscellaneous metadata"
- set_at: [ startup, runtime ]
- cpp_vartype: AtomicWord<int>
- cpp_varname: internalQueryDocumentSourceWriterBatchExtraReservedBytes
- validator:
- gte: 0
- lte:
- expr: 8 * 1024 * 1024 # 8MB
- default: 0
-
- internalQueryStatsRateLimit:
- description: "The maximum number of queries per second that are sampled for query stats.
- If the rate of queries goes above this number, then rate limiting will kick in, and any
- further queries will not be sampled. To sample all queries, this can be set to -1. This can be
- set to 0 to turn queryStats off completely."
- set_at: [ startup, runtime ]
- cpp_varname: "internalQueryStatsRateLimit"
- cpp_vartype: AtomicWord<int>
- default: 0
- validator:
- gte: -1
- on_update: query_stats_util::onQueryStatsSamplingRateUpdate
-
- internalQueryStatsCacheSize:
- description: "The maximum amount of memory that the system will allocate for the query queryStats
- cache. This will accept values in either of the following formats:
- 1. <number>% indicates a percentage of the physical memory available to the process. E.g.: 15%.
- 2. <number>(MB|GB), indicates the amount of memory in MB or GB. E.g.: 1.5GB, 100MB.
- The default value is 1%, which means 1% of the physical memory available to the process."
- set_at: [ startup, runtime ]
- cpp_varname: "internalQueryStatsCacheSize"
- cpp_vartype: synchronized_value<std::string>
- default: "1%"
- on_update: query_stats_util::onQueryStatsStoreSizeUpdate
- validator:
- callback: query_stats_util::validateQueryStatsStoreSize
-
- internalQueryStatsErrorsAreCommandFatal:
- description: "Whether errors in the $queryStats stage cause the aggregation pipeline to
- immediately fail and report the error. Note that this is always the case for debug builds."
- set_at: [ startup, runtime ]
- cpp_varname: "internalQueryStatsErrorsAreCommandFatal"
- cpp_vartype: AtomicWord<bool>
- default: false
-
- internalQueryAggMulticastTimeoutMS:
- description: "Timeout in MS for requests to shard servers when aggregations are sent to all shard servers"
- set_at: [ startup ]
- cpp_vartype: int
- cpp_varname: internalQueryAggMulticastTimeoutMS
- default: 60000
- validator:
- gte: 0
-
- internalQueryAggMulticastMaxConcurrency:
- description: "Max number of concurrent requests when aggregations are sent to all shard servers"
- set_at: startup
- cpp_vartype: int
- cpp_varname: internalQueryAggMulticastMaxConcurrency
- default: 100
- validator:
- gte: 1
-
- internalQuerySpillingMaxWaitTimeout:
- description: "Timeout in MS that the storage engine will block a spilling operation when the
- cache is under pressure."
- set_at: [ startup, runtime ]
- cpp_vartype: AtomicWord<int>
- cpp_varname: internalQuerySpillingMaxWaitTimeout
- default: 1000
- validator:
- gte: 0
-
# Note for adding additional query knobs:
#
# When adding a new query knob, you should consider whether or not you need to add an 'on_update'
diff --git a/src/mongo/db/query/query_planner.cpp b/src/mongo/db/query/query_planner.cpp
index 32af6762a17..c258c9e6867 100644
--- a/src/mongo/db/query/query_planner.cpp
+++ b/src/mongo/db/query/query_planner.cpp
@@ -252,11 +252,6 @@ void tryToAddColumnScan(const QueryPlannerParams& params,
// collection scan. Add that solution.
out.push_back(QueryPlannerAnalysis::analyzeDataAccess(query, params, std::move(columnScan)));
}
-
-bool collscanIsBounded(const CollectionScanNode* collscan) {
- return collscan->minRecord || collscan->maxRecord;
-}
-
} // namespace
using std::numeric_limits;
@@ -339,9 +334,6 @@ string optionString(size_t options) {
case QueryPlannerParams::RETURN_OWNED_DATA:
ss << "RETURN_OWNED_DATA ";
break;
- case QueryPlannerParams::STRICT_NO_TABLE_SCAN:
- ss << "STRICT_NO_TABLE_SCAN ";
- break;
case QueryPlannerParams::DEFAULT:
MONGO_UNREACHABLE;
break;
@@ -477,57 +469,13 @@ static BSONObj finishMaxObj(const IndexEntry& indexEntry,
}
}
-bool providesSort(const CanonicalQuery& query, const BSONObj& kp) {
- return query.getFindCommandRequest().getSort().isPrefixOf(
- kp, SimpleBSONElementComparator::kInstance);
-}
-
-/**
- * Determine whether this query has a sort that can be provided by the clustered index, if so, which
- * direction the scan should be. If the collection is not clustered, or the sort cannot be provided,
- * returns 'boost::none'.
- */
-boost::optional<int> determineClusteredScanDirection(const CanonicalQuery& query,
- const QueryPlannerParams& params) {
- if (params.clusteredInfo && query.getSortPattern() &&
- CollatorInterface::collatorsMatch(params.clusteredCollectionCollator,
- query.getCollator())) {
- auto kp = clustered_util::getSortPattern(params.clusteredInfo->getIndexSpec());
- if (providesSort(query, kp)) {
- return 1;
- } else if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) {
- return -1;
- }
- }
-
- return boost::none;
-}
-
-/**
- * Determine the direction of the scan needed for the query. Defaults to 1 unless this is a
- * clustered collection and we have a sort that can be provided by the clustered index.
- */
-int determineCollscanDirection(const CanonicalQuery& query, const QueryPlannerParams& params) {
- return determineClusteredScanDirection(query, params).value_or(1);
-}
-
-std::pair<std::unique_ptr<QuerySolution>, const CollectionScanNode*> buildCollscanSolnWithNode(
- const CanonicalQuery& query,
- bool tailable,
- const QueryPlannerParams& params,
- boost::optional<int> direction = boost::none) {
- std::unique_ptr<QuerySolutionNode> solnRoot(QueryPlannerAccess::makeCollectionScan(
- query, tailable, params, direction.value_or(determineCollscanDirection(query, params))));
- const auto* collscanNode = checked_cast<const CollectionScanNode*>(solnRoot.get());
- return std::make_pair(
- QueryPlannerAnalysis::analyzeDataAccess(query, params, std::move(solnRoot)), collscanNode);
-}
-
std::unique_ptr<QuerySolution> buildCollscanSoln(const CanonicalQuery& query,
bool tailable,
const QueryPlannerParams& params,
- boost::optional<int> direction = boost::none) {
- return buildCollscanSolnWithNode(query, tailable, params, direction).first;
+ int direction = 1) {
+ std::unique_ptr<QuerySolutionNode> solnRoot(
+ QueryPlannerAccess::makeCollectionScan(query, tailable, params, direction));
+ return QueryPlannerAnalysis::analyzeDataAccess(query, params, std::move(solnRoot));
}
std::unique_ptr<QuerySolution> buildWholeIXSoln(
@@ -543,6 +491,11 @@ std::unique_ptr<QuerySolution> buildWholeIXSoln(
return QueryPlannerAnalysis::analyzeDataAccess(query, params, std::move(solnRoot));
}
+bool providesSort(const CanonicalQuery& query, const BSONObj& kp) {
+ return query.getFindCommandRequest().getSort().isPrefixOf(
+ kp, SimpleBSONElementComparator::kInstance);
+}
+
StatusWith<std::unique_ptr<PlanCacheIndexTree>> QueryPlanner::cacheDataFromTaggedTree(
const MatchExpression* const taggedTree, const vector<IndexEntry>& relevantIndices) {
if (!taggedTree) {
@@ -708,7 +661,7 @@ StatusWith<std::unique_ptr<QuerySolution>> QueryPlanner::planFromCache(
} else if (SolutionCacheData::COLLSCAN_SOLN == winnerCacheData.solnType) {
// The cached solution is a collection scan. We don't cache collscans
// with tailable==true, hence the false below.
- auto soln = buildCollscanSoln(query, false, params, winnerCacheData.wholeIXSolnDir);
+ auto soln = buildCollscanSoln(query, false, params);
if (!soln) {
return Status(ErrorCodes::NoQueryExecutionPlans,
"plan cache error: collection scan soln");
@@ -730,11 +683,10 @@ StatusWith<std::unique_ptr<QuerySolution>> QueryPlanner::planFromCache(
"filter"_attr = redact(clone->debugString()),
"cacheData"_attr = redact(winnerCacheData.toString()));
- RelevantFieldIndexMap fields;
+ stdx::unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), &fields);
- // We will not cache queries with 'hint'.
std::vector<IndexEntry> expandedIndexes =
- QueryPlannerIXSelect::expandIndexes(fields, params.indices, false /* indexHinted */);
+ QueryPlannerIXSelect::expandIndexes(fields, params.indices);
// Map from index name to index number.
map<IndexEntry::Identifier, size_t> indexMap;
@@ -785,38 +737,6 @@ StatusWith<std::unique_ptr<QuerySolution>> QueryPlanner::planFromCache(
return {std::move(soln)};
}
-// If no table scan option is set the planner may not return any plan containing a collection scan.
-// Yet clusteredIdxScans are still allowed as they are not a full collection scan but a bounded
-// collection scan.
-bool noTableScan(const QueryPlannerParams& params) {
- return (params.options & QueryPlannerParams::NO_TABLE_SCAN);
-}
-
-// Used internally if the planner should also avoid retruning a plan containing a clusteredIDX scan.
-bool noTableAndClusteredIDXScan(const QueryPlannerParams& params) {
- return (params.options & QueryPlannerParams::STRICT_NO_TABLE_SCAN);
-}
-
-bool isClusteredScan(QuerySolutionNode* node) {
- if (node->getType() == STAGE_COLLSCAN) {
- auto collectionScanSolnNode = dynamic_cast<CollectionScanNode*>(node);
- return (collectionScanSolnNode->doClusteredCollectionScan());
- }
- return false;
-}
-
-// Check if this is a real coll scan or a hidden ClusteredIDX scan.
-bool isColusteredIDXScanSoln(QuerySolution* collscanSoln) {
- if (collscanSoln->root()->getType() == STAGE_SHARDING_FILTER) {
- auto child = collscanSoln->root()->children.begin();
- return isClusteredScan(*child);
- }
- if (collscanSoln->root()->getType() == STAGE_COLLSCAN) {
- return isClusteredScan(collscanSoln->root());
- }
- return false;
-}
-
StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
const CanonicalQuery& query, const QueryPlannerParams& params) {
// It's a little silly to ask for a count and for owned data. This could indicate a bug
@@ -870,6 +790,7 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
// any $natural sort to have been normalized to a $natural hint upstream.
// Additionally, if the hint matches the collection's cluster key, we also output a
// collscan utilizing the cluster key.
+
if (naturalHint) {
// Perform validation specific to $natural.
LOGV2_DEBUG(20969, 5, "Forcing a table scan due to hinted $natural");
@@ -929,7 +850,7 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
out.push_back(std::move(soln));
return {std::move(out)};
}
- } // namespace mongo
+ }
// Hints require us to only consider the hinted index. If index filters in the query
// settings were used to override the allowed indices for planning, we should not use the
@@ -967,14 +888,13 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
}
// Figure out what fields we care about.
- RelevantFieldIndexMap fields;
+ stdx::unordered_set<string> fields;
QueryPlannerIXSelect::getFields(query.root(), &fields);
for (auto&& field : fields) {
- LOGV2_DEBUG(20970, 5, "Predicate over field", "field"_attr = field.first);
+ LOGV2_DEBUG(20970, 5, "Predicate over field", "field"_attr = field);
}
- fullIndexList = QueryPlannerIXSelect::expandIndexes(
- fields, std::move(fullIndexList), !hintedIndex.isEmpty());
+ fullIndexList = QueryPlannerIXSelect::expandIndexes(fields, std::move(fullIndexList));
std::vector<IndexEntry> relevantIndices;
if (!hintedIndexEntry) {
@@ -1337,6 +1257,37 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
}
}
}
+
+ // The base index is sorted on some key, so it's possible we might want to use
+ // a collection scan to provide the sort requested
+ if (params.clusteredInfo) {
+ if (CollatorInterface::collatorsMatch(params.clusteredCollectionCollator,
+ query.getCollator())) {
+ auto kp = clustered_util::getSortPattern(params.clusteredInfo->getIndexSpec());
+ int direction = 0;
+ if (providesSort(query, kp)) {
+ direction = 1;
+ } else if (providesSort(query, QueryPlannerCommon::reverseSortObj(kp))) {
+ direction = -1;
+ }
+
+ if (direction != 0) {
+ auto soln = buildCollscanSoln(query, isTailable, params, direction);
+ if (soln) {
+ LOGV2_DEBUG(6082401,
+ 5,
+ "Planner: outputting soln that uses clustered index to "
+ "provide sort");
+ SolutionCacheData* scd = new SolutionCacheData();
+ scd->solnType = SolutionCacheData::COLLSCAN_SOLN;
+ scd->wholeIXSolnDir = direction;
+
+ soln->cacheData.reset(scd);
+ out.push_back(std::move(soln));
+ }
+ }
+ }
+ }
}
// If a projection exists, there may be an index that allows for a covered plan, even if
@@ -1383,13 +1334,11 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
// No indexed plans? We must provide a collscan if possible or else we can't run the query.
bool collScanRequired = 0 == out.size();
- if (collScanRequired && noTableAndClusteredIDXScan(params)) {
+ if (collScanRequired && !canTableScan) {
return Status(ErrorCodes::NoQueryExecutionPlans,
"No indexed plans available, and running with 'notablescan'");
}
- bool clusteredCollection = params.clusteredInfo.has_value();
-
// geoNear and text queries *require* an index.
// Also, if a hint is specified it indicates that we MUST use it.
bool possibleToCollscan =
@@ -1399,53 +1348,31 @@ StatusWith<std::vector<std::unique_ptr<QuerySolution>>> QueryPlanner::plan(
return Status(ErrorCodes::NoQueryExecutionPlans, "No query solutions");
}
- bool isClusteredIDXScan = false;
- if (possibleToCollscan && (collscanRequested || collScanRequired || clusteredCollection)) {
- auto clusteredScanDirection = determineClusteredScanDirection(query, params);
- auto direction = clusteredScanDirection.value_or(1);
- auto [collscanSoln, collscanNode] =
- buildCollscanSolnWithNode(query, isTailable, params, direction);
- if (!collscanSoln && collScanRequired) {
+ if (possibleToCollscan && (collscanRequested || collScanRequired)) {
+ auto collscan = buildCollscanSoln(query, isTailable, params);
+ if (!collscan && collScanRequired) {
return Status(ErrorCodes::NoQueryExecutionPlans,
"Failed to build collection scan soln");
}
- isClusteredIDXScan = isColusteredIDXScanSoln(collscanSoln.get());
- // We consider collection scan in the following cases:
- // 1. collScanRequested - specifically requested by caller.
- // 2. collScanRequired - there are no other possible plans, so we fallback to full scan.
- // 3. collscanIsBounded - collection is clustered and clustered index is used.
- // 4. clusteredScanDirection - collection is clustered and sort, provided by clustered
- // index, is used
- if (collscanSoln &&
- (collscanRequested || collScanRequired || collscanIsBounded(collscanNode) ||
- clusteredScanDirection)) {
+ if (collscan) {
LOGV2_DEBUG(20984,
5,
"Planner: outputting a collection scan",
- "collectionScan"_attr = redact(collscanSoln->toString()));
+ "collectionScan"_attr = redact(collscan->toString()));
SolutionCacheData* scd = new SolutionCacheData();
scd->solnType = SolutionCacheData::COLLSCAN_SOLN;
- scd->wholeIXSolnDir = direction;
- collscanSoln->cacheData.reset(scd);
- out.push_back(std::move(collscanSoln));
+ collscan->cacheData.reset(scd);
+ out.push_back(std::move(collscan));
}
}
- // Make sure to respect the notablescan option. A clustered IDX scan is allowed even under a
- // NOTABLE option. Only in the case of a strict NOTABLE scan option a clustered IDX scan is not
- // allowed. This option is used in mongoS for shardPruning.
invariant(out.size() > 0);
- if (collScanRequired && noTableScan(params) && !isClusteredIDXScan) {
- return Status(ErrorCodes::NoQueryExecutionPlans,
- "No indexed plans available, and running with 'notablescan'");
- }
return {std::move(out)};
}
/**
- * The 'query' might contain parts of aggregation pipeline. For now, we plan those separately
- * and later attach the agg portion of the plan to the solution(s) for the "find" part of the
- * query.
+ * The 'query' might contain parts of aggregation pipeline. For now, we plan those separately and
+ * later attach the agg portion of the plan to the solution(s) for the "find" part of the query.
*/
std::unique_ptr<QuerySolution> QueryPlanner::extendWithAggPipeline(
const CanonicalQuery& query,
@@ -1625,13 +1552,13 @@ StatusWith<QueryPlanner::SubqueriesPlanningResult> QueryPlanner::planSubqueries(
planningResult.branches.push_back(
std::make_unique<SubqueriesPlanningResult::BranchPlanningResult>());
auto branchResult = planningResult.branches.back().get();
+ auto orChild = planningResult.orExpression->getChild(i);
// Turn the i-th child into its own query.
- auto statusWithCQ = CanonicalQuery::makeForSubplanner(opCtx, query, i);
+ auto statusWithCQ = CanonicalQuery::canonicalize(opCtx, query, orChild);
if (!statusWithCQ.isOK()) {
str::stream ss;
- ss << "Can't canonicalize subchild "
- << planningResult.orExpression->getChild(i)->debugString() << " "
+ ss << "Can't canonicalize subchild " << orChild->debugString() << " "
<< statusWithCQ.getStatus().reason();
return Status(ErrorCodes::BadValue, ss);
}
diff --git a/src/mongo/db/query/query_planner_array_test.cpp b/src/mongo/db/query/query_planner_array_test.cpp
index ff39e98c3d0..19dfc99dbcd 100644
--- a/src/mongo/db/query/query_planner_array_test.cpp
+++ b/src/mongo/db/query/query_planner_array_test.cpp
@@ -2110,13 +2110,13 @@ TEST_F(QueryPlannerTest, CanHoistNegatedPredFromElemMatchIntoSiblingOrWithMultik
"{fetch: {filter: {arr: {$elemMatch: {a: {$ne: 1}, b: {$in: [2, 3]}}}},"
"node: {"
" or: {nodes: ["
- " {fetch: {filter: {'arr.a': {$ne: 1}},"
+ " {fetch: {filter: {a: {$ne: 1}},"
" node: {ixscan: {pattern: {'arr.a': 1, 'arr.b': 1, c: 1, d: 1},"
" bounds: {'arr.a': [['MinKey', 1, true, false], [1, 'MaxKey', false, true]],"
" 'arr.b': [[2, 2, true, true], [3, 3, true, true]],"
" c: [[4, 4, true, true]],"
" d: [[5, 5, true, true]]}}}}},"
- " {fetch: {filter: {'arr.a': {$ne: 1}},"
+ " {fetch: {filter: {a: {$ne: 1}},"
" node: {ixscan: {pattern: {'arr.a': 1, 'arr.b': 1, c: 1, d: 1},"
" bounds: {'arr.a': [['MinKey', 1, true, false],[1, 'MaxKey', false, true]],"
" 'arr.b': [[2, 2, true, true], [3, 3, true, true]],"
diff --git a/src/mongo/db/query/query_planner_common.h b/src/mongo/db/query/query_planner_common.h
index 97e94bb0a69..6d441155b54 100644
--- a/src/mongo/db/query/query_planner_common.h
+++ b/src/mongo/db/query/query_planner_common.h
@@ -65,20 +65,6 @@ public:
}
/**
- * Returns a count of 'type' nodes in expression tree.
- */
- static size_t countNodes(const MatchExpression* root, MatchExpression::MatchType type) {
- size_t sum = 0;
- if (type == root->matchType()) {
- sum = 1;
- }
- for (size_t i = 0; i < root->numChildren(); ++i) {
- sum += countNodes(root->getChild(i), type);
- }
- return sum;
- }
-
- /**
* Assumes the provided BSONObj is of the form {field1: -+1, ..., field2: -+1}
* Returns a BSONObj with the values negated.
*/
diff --git a/src/mongo/db/query/query_planner_index_test.cpp b/src/mongo/db/query/query_planner_index_test.cpp
index 24631b9091e..045fa35dbe1 100644
--- a/src/mongo/db/query/query_planner_index_test.cpp
+++ b/src/mongo/db/query/query_planner_index_test.cpp
@@ -198,20 +198,26 @@ TEST_F(QueryPlannerTest, SparseIndexForQuery) {
"{filter: null, pattern: {a: 1}}}}}");
}
-TEST_F(QueryPlannerTest, ExprEqCannotUseSparseIndex) {
+TEST_F(QueryPlannerTest, ExprEqCanUseSparseIndex) {
params.options &= ~QueryPlannerParams::INCLUDE_COLLSCAN;
addIndex(fromjson("{a: 1}"), false, true);
runQuery(fromjson("{a: {$_internalExprEq: 1}}"));
- assertHasOnlyCollscan();
+ assertNumSolutions(1U);
+ assertSolutionExists(
+ "{fetch: {filter: null, node: {ixscan: "
+ "{filter: null, pattern: {a: 1}, bounds: {a: [[1,1,true,true]]}}}}}");
}
-TEST_F(QueryPlannerTest, ExprEqCannotUseSparseIndexForEqualityToNull) {
+TEST_F(QueryPlannerTest, ExprEqCanUseSparseIndexForEqualityToNull) {
params.options &= ~QueryPlannerParams::INCLUDE_COLLSCAN;
addIndex(fromjson("{a: 1}"), false, true);
runQuery(fromjson("{a: {$_internalExprEq: null}}"));
- assertHasOnlyCollscan();
+ assertNumSolutions(1U);
+ assertSolutionExists(
+ "{fetch: {filter: {a: {$_internalExprEq: null}}, node: {ixscan: {filter: null, pattern: "
+ "{a: 1}, bounds: {a: [[undefined,undefined,true,true], [null,null,true,true]]}}}}}");
}
TEST_F(QueryPlannerTest, NegationCannotUseSparseIndex) {
diff --git a/src/mongo/db/query/query_planner_params.h b/src/mongo/db/query/query_planner_params.h
index ab07388710f..c8542cda90e 100644
--- a/src/mongo/db/query/query_planner_params.h
+++ b/src/mongo/db/query/query_planner_params.h
@@ -153,11 +153,6 @@ struct QueryPlannerParams {
// Ensure that any plan generated returns data that is "owned." That is, all BSONObjs are
// in an "owned" state and are not pointing to data that belongs to the storage engine.
RETURN_OWNED_DATA = 1 << 12,
-
- // This is an extension to the NO_TABLE_SCAN parameter. This more stricter option will also
- // avoid a CLUSTEREDIDX_SCAN which comes built into a collection scan when the collection is
- // clustered.
- STRICT_NO_TABLE_SCAN = 1 << 13,
};
// See Options enum above.
diff --git a/src/mongo/db/query/query_planner_test_lib.cpp b/src/mongo/db/query/query_planner_test_lib.cpp
index 849ce6fb6e8..bec219226b5 100644
--- a/src/mongo/db/query/query_planner_test_lib.cpp
+++ b/src/mongo/db/query/query_planner_test_lib.cpp
@@ -1234,10 +1234,7 @@ Status QueryPlannerTestLib::solutionMatches(const BSONObj& testSoln,
}
BSONObjBuilder bob;
- actualGroupNode->groupByExpression
- ->serialize(SerializationOptions{
- boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})
- .addToBsonObj(&bob, "_id");
+ actualGroupNode->groupByExpression->serialize(true).addToBsonObj(&bob, "_id");
auto actualGroupByObj = bob.done();
if (!SimpleBSONObjComparator::kInstance.evaluate(actualGroupByObj ==
expectedGroupByElem.Obj())) {
@@ -1250,10 +1247,7 @@ Status QueryPlannerTestLib::solutionMatches(const BSONObj& testSoln,
BSONArrayBuilder actualAccs;
for (auto& acc : actualGroupNode->accumulators) {
BSONObjBuilder bob;
- acc.expr.argument
- ->serialize(SerializationOptions{
- boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})
- .addToBsonObj(&bob, acc.expr.name);
+ acc.expr.argument->serialize(true).addToBsonObj(&bob, acc.expr.name);
actualAccs.append(BSON(acc.fieldName << bob.done()));
}
auto expectedAccsObj = expectedGroupObj["accs"].Obj();
diff --git a/src/mongo/db/query/query_planner_tree_test.cpp b/src/mongo/db/query/query_planner_tree_test.cpp
index 5b7055dd720..9d403989792 100644
--- a/src/mongo/db/query/query_planner_tree_test.cpp
+++ b/src/mongo/db/query/query_planner_tree_test.cpp
@@ -32,7 +32,6 @@
#include "mongo/db/query/collation/collator_interface_mock.h"
#include "mongo/db/query/query_planner.h"
#include "mongo/db/query/query_planner_test_fixture.h"
-#include "mongo/idl/server_parameter_test_util.h"
#include "mongo/unittest/unittest.h"
namespace mongo {
@@ -2465,32 +2464,6 @@ TEST_F(QueryPlannerTest, LockstepOrEnumerationSanityCheckTwoChildrenTwoIndexesEa
"{ixscan: {pattern: {a: 1, c: 1}}}}}}}");
}
-TEST_F(QueryPlannerTest, TotalPossibleLockstepOrEnumerationReachesTheOrLimit) {
- params.options =
- QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::ENUMERATE_OR_CHILDREN_LOCKSTEP;
- addIndex(BSON("a" << 1 << "b" << 1));
- addIndex(BSON("a" << 1 << "c" << 1));
-
- BSONArrayBuilder orBuilder;
- // This max number has a value of 65 in order to potentillay triger any overflow of the possible
- // enumeration count, because each predicate in $or has two possible indexes, allowing for 2^65
- // possible enumerations.
- const int maxPredicates = 65;
- for (int i = 0; i < maxPredicates; i++) {
- orBuilder.append(BSON("b" << i << "c" << i));
- }
-
- auto cmd = BSON("find"
- << "testns"
- << "filter" << BSON("a" << 1 << "$or" << orBuilder.arr()));
-
- // Ensure that the query runs fine.
- runQueryAsCommand(cmd);
-
- // internalQueryMaxOrSolutions.load() + 2.
- assertNumSolutions(12U);
-}
-
// Test that we enumerate the expected plans with the special parameter set. In this test we have
// two branches of an $or, each with one possible indexed solution.
TEST_F(QueryPlannerTest, LockstepOrEnumerationSanityCheckTwoChildrenOneIndexEach) {
@@ -2847,89 +2820,6 @@ TEST_F(QueryPlannerTest, LockstepOrEnumerationApplysToEachOrInTree) {
"]}}");
}
-// This test was designed to reproduce SERVER-83091, a case in which an implementation error in the
-// lockstep $or enumeration algorithm could result in an infinite loop. This could happen only if
-// there were nested $or nodes and the inner $or hit the maximum number of plans that it is willing
-// to generate.
-TEST_F(QueryPlannerTest, LockstepOrEnumerationWithNestedOrWhereInnerOrHitsEnumerationLimit) {
- // Disable match expression optimixation, since when enabled it will collapse nested $or nodes
- // into a single $or.
- FailPointEnableBlock failPoint("disableMatchExpressionOptimization");
-
- // The repro depends on the inner $or hitting its enumeration limit. The original problem from
- // SERVER-83091 can be reproduced with a simpler query if we lower the limit on the number of
- // plans that the 'PlanEnumerator' is allowed to generate for any $or node.
- RAIIServerParameterControllerForTest maxOrPlansController(
- "internalQueryEnumerationMaxOrSolutions", 3);
-
- params.options =
- QueryPlannerParams::NO_TABLE_SCAN | QueryPlannerParams::ENUMERATE_OR_CHILDREN_LOCKSTEP;
- addIndex(BSON("a" << 1));
- addIndex(BSON("b" << 1));
- addIndex(BSON("c" << 1));
-
- runQueryAsCommand(fromjson(R"(
- {find: 'testns', filter: {
- $or: [
- {$or: [
- {a: 1, b: 2},
- {a: 3}
- ]},
- {c: 4}
- ]
- }})"));
-
- // There are two plans, the only difference between the two being whether the nested $and
- // {a: 1, b: 2} uses the index on "a" or the index on "b".
- assertNumSolutions(2U);
-
- // Plan using the {a: 1} index for the innermost conjunction.
- assertSolutionExists(R"(
- {
- fetch: {
- node: {
- or: {
- nodes: [
- {
- or: {
- nodes: [
- {fetch: {filter: {b: 2}, node: {ixscan: {pattern: {a: 1}}}}},
- {ixscan: {pattern: {a: 1}}}
- ]
- }
- },
- {ixscan: {pattern: {c: 1}}}
- ]
- }
- }
- }
- }
- )");
-
- // Alternative plan using the {b: 1} index for the innermost conjunction.
- assertSolutionExists(R"(
- {
- fetch: {
- node: {
- or: {
- nodes: [
- {
- or: {
- nodes: [
- {fetch: {filter: {a: 1}, node: {ixscan: {pattern: {b: 1}}}}},
- {ixscan: {pattern: {a: 1}}}
- ]
- }
- },
- {ixscan: {pattern: {c: 1}}}
- ]
- }
- }
- }
- }
- )");
-}
-
TEST_F(QueryPlannerTest, NoOrSolutionsIfMaxOrSolutionsIsZero) {
auto defaultMaxOr = internalQueryEnumerationMaxOrSolutions.load();
ON_BLOCK_EXIT([&] { internalQueryEnumerationMaxOrSolutions.store(defaultMaxOr); });
diff --git a/src/mongo/db/query/query_planner_wildcard_index_test.cpp b/src/mongo/db/query/query_planner_wildcard_index_test.cpp
index eca110ffac0..cd943126a17 100644
--- a/src/mongo/db/query/query_planner_wildcard_index_test.cpp
+++ b/src/mongo/db/query/query_planner_wildcard_index_test.cpp
@@ -56,7 +56,7 @@ protected:
}
void addWildcardIndex(BSONObj keyPattern,
- const OrderedPathSet& multikeyPathSet = {},
+ const std::set<std::string>& multikeyPathSet = {},
BSONObj wildcardProjection = BSONObj{},
MatchExpression* partialFilterExpr = nullptr,
CollatorInterface* collator = nullptr,
@@ -418,18 +418,25 @@ TEST_F(QueryPlannerWildcardTest, EqualityIndexScanOverNestedField) {
"bounds: {'$_path': [['a.b','a.b',true,true]], 'a.b': [[5,5,true,true]]}}}}}");
}
-TEST_F(QueryPlannerWildcardTest, ExprEqCannotUseIndex) {
+TEST_F(QueryPlannerWildcardTest, ExprEqCanUseIndex) {
addWildcardIndex(BSON("$**" << 1));
runQuery(fromjson("{a: {$_internalExprEq: 1}}"));
- assertHasOnlyCollscan();
+ assertNumSolutions(1U);
+ assertSolutionExists(
+ "{fetch: {filter: null, node: {ixscan: {pattern: {'$_path': 1, a: 1},"
+ "bounds: {'$_path': [['a','a',true,true]], a: [[1,1,true,true]]}}}}}");
}
-TEST_F(QueryPlannerWildcardTest, ExprEqCannotUseSparseIndexForEqualityToNull) {
+TEST_F(QueryPlannerWildcardTest, ExprEqCanUseSparseIndexForEqualityToNull) {
addWildcardIndex(BSON("$**" << 1));
runQuery(fromjson("{a: {$_internalExprEq: null}}"));
- assertHasOnlyCollscan();
+ assertNumSolutions(1U);
+ assertSolutionExists(
+ "{fetch: {filter: {a: {$_internalExprEq: null}}, node: {ixscan: {pattern: {'$_path': 1, a: "
+ "1}, bounds: {'$_path': [['a','a',true,true]], a: [[undefined,undefined,true,true], "
+ "[null,null,true,true]]}}}}}");
}
TEST_F(QueryPlannerWildcardTest, PrefixRegex) {
diff --git a/src/mongo/db/query/query_request_helper.cpp b/src/mongo/db/query/query_request_helper.cpp
index b713b9d8000..410c05fcaf0 100644
--- a/src/mongo/db/query/query_request_helper.cpp
+++ b/src/mongo/db/query/query_request_helper.cpp
@@ -85,35 +85,6 @@ Status validateGetMoreCollectionName(StringData collectionName) {
return Status::OK();
}
-Status validateResumeAfter(const mongo::BSONObj& resumeAfter, bool isClusteredCollection) {
- if (resumeAfter.isEmpty()) {
- return Status::OK();
- }
-
- BSONType recordIdType = resumeAfter["$recordId"].type();
- if (resumeAfter.nFields() != 1 ||
- (recordIdType != BSONType::NumberLong && recordIdType != BSONType::BinData &&
- recordIdType != BSONType::jstNULL)) {
- return Status(ErrorCodes::BadValue,
- "Malformed resume token: the '_resumeAfter' object must contain"
- " exactly one field named '$recordId', of type NumberLong, BinData "
- "or jstNULL.");
- }
-
- // Clustered collections can only have accept '$_resumeAfter' parameter of type
- // BinData. Non clustered collections should only accept '$_resumeAfter' of type
- // Long.
- if ((isClusteredCollection && recordIdType == BSONType::NumberLong) ||
- (!isClusteredCollection && recordIdType == BSONType::BinData)) {
- return Status(ErrorCodes::Error(7738600),
- "The '$_resumeAfter parameter must match collection type. Clustered "
- "collections only have BinData recordIds, and all other collections"
- "have Long recordId.");
- }
-
- return Status::OK();
-}
-
Status validateFindCommandRequest(const FindCommandRequest& findCommand) {
// Min and Max objects must have the same fields.
if (!findCommand.getMin().isEmpty() && !findCommand.getMax().isEmpty()) {
@@ -156,8 +127,17 @@ Status validateFindCommandRequest(const FindCommandRequest& findCommand) {
return Status(ErrorCodes::BadValue,
"sort must be unset or {$natural:1} if 'requestResumeToken' is enabled");
}
- // The $_resumeAfter parameter is checked in 'validateResumeAfter()'.
-
+ if (!findCommand.getResumeAfter().isEmpty()) {
+ if (findCommand.getResumeAfter().nFields() != 1 ||
+ (findCommand.getResumeAfter()["$recordId"].type() != BSONType::NumberLong &&
+ findCommand.getResumeAfter()["$recordId"].type() != BSONType::BinData &&
+ findCommand.getResumeAfter()["$recordId"].type() != BSONType::jstNULL)) {
+ return Status(ErrorCodes::BadValue,
+ "Malformed resume token: the '_resumeAfter' object must contain"
+ " exactly one field named '$recordId', of type NumberLong, BinData "
+ "or jstNULL.");
+ }
+ }
} else if (!findCommand.getResumeAfter().isEmpty()) {
return Status(ErrorCodes::BadValue,
"'requestResumeToken' must be true if 'resumeAfter' is"
diff --git a/src/mongo/db/query/query_request_helper.h b/src/mongo/db/query/query_request_helper.h
index c0d8968f563..4d3ec6143c8 100644
--- a/src/mongo/db/query/query_request_helper.h
+++ b/src/mongo/db/query/query_request_helper.h
@@ -63,12 +63,6 @@ static constexpr auto kNaturalSortField = "$natural";
Status validateGetMoreCollectionName(StringData collectionName);
/**
- * Returns a non-OK status if '$_resumeAfter' is set to an unexpected value, or the wrong type
- * determined by the collection type.
- */
-Status validateResumeAfter(const mongo::BSONObj& resumeAfter, bool isClusteredCollection);
-
-/**
* Returns a non-OK status if any property of the QR has a bad value (e.g. a negative skip
* value) or if there is a bad combination of options (e.g. awaitData is illegal without
* tailable).
diff --git a/src/mongo/db/query/query_request_test.cpp b/src/mongo/db/query/query_request_test.cpp
index 52c6dd4aca7..80ed0325802 100644
--- a/src/mongo/db/query/query_request_test.cpp
+++ b/src/mongo/db/query/query_request_test.cpp
@@ -36,7 +36,6 @@
#include "mongo/base/error_codes.h"
#include "mongo/db/catalog/collection_catalog.h"
#include "mongo/db/catalog/collection_mock.h"
-#include "mongo/db/cursor_id.h"
#include "mongo/db/dbmessage.h"
#include "mongo/db/json.h"
#include "mongo/db/namespace_string.h"
@@ -279,13 +278,10 @@ TEST(QueryRequestTest, InvalidResumeAfterWrongRecordIdType) {
findCommand.setRequestResumeToken(true);
// Hint must be explicitly set for the query request to validate.
findCommand.setHint(fromjson("{$natural: 1}"));
- ASSERT_NOT_OK(query_request_helper::validateResumeAfter(findCommand.getResumeAfter(),
- false /* isClusteredCollection */));
+ ASSERT_NOT_OK(query_request_helper::validateFindCommandRequest(findCommand));
resumeAfter = BSON("$recordId" << 1LL);
findCommand.setResumeAfter(resumeAfter);
ASSERT_OK(query_request_helper::validateFindCommandRequest(findCommand));
- ASSERT_OK(query_request_helper::validateResumeAfter(findCommand.getResumeAfter(),
- false /* isClusteredCollection */));
}
TEST(QueryRequestTest, InvalidResumeAfterExtraField) {
@@ -295,8 +291,7 @@ TEST(QueryRequestTest, InvalidResumeAfterExtraField) {
findCommand.setRequestResumeToken(true);
// Hint must be explicitly set for the query request to validate.
findCommand.setHint(fromjson("{$natural: 1}"));
- ASSERT_NOT_OK(query_request_helper::validateResumeAfter(findCommand.getResumeAfter(),
- false /* isClusteredCollection */));
+ ASSERT_NOT_OK(query_request_helper::validateFindCommandRequest(findCommand));
}
TEST(QueryRequestTest, ResumeAfterWithHint) {
@@ -319,8 +314,6 @@ TEST(QueryRequestTest, ResumeAfterWithSort) {
// Hint must be explicitly set for the query request to validate.
findCommand.setHint(fromjson("{$natural: 1}"));
ASSERT_OK(query_request_helper::validateFindCommandRequest(findCommand));
- ASSERT_OK(query_request_helper::validateResumeAfter(findCommand.getResumeAfter(),
- false /* isClusteredCollection */));
findCommand.setSort(fromjson("{a: 1}"));
ASSERT_NOT_OK(query_request_helper::validateFindCommandRequest(findCommand));
findCommand.setSort(fromjson("{$natural: 1}"));
@@ -336,8 +329,6 @@ TEST(QueryRequestTest, ResumeNoSpecifiedRequestResumeToken) {
ASSERT_NOT_OK(query_request_helper::validateFindCommandRequest(findCommand));
findCommand.setRequestResumeToken(true);
ASSERT_OK(query_request_helper::validateFindCommandRequest(findCommand));
- ASSERT_OK(query_request_helper::validateResumeAfter(findCommand.getResumeAfter(),
- false /* isClusteredCollection */));
}
TEST(QueryRequestTest, ExplicitEmptyResumeAfter) {
@@ -349,8 +340,6 @@ TEST(QueryRequestTest, ExplicitEmptyResumeAfter) {
ASSERT_OK(query_request_helper::validateFindCommandRequest(findCommand));
findCommand.setRequestResumeToken(true);
ASSERT_OK(query_request_helper::validateFindCommandRequest(findCommand));
- ASSERT_OK(query_request_helper::validateResumeAfter(findCommand.getResumeAfter(),
- false /* isClusteredCollection */));
}
//
diff --git a/src/mongo/db/query/query_shape.cpp b/src/mongo/db/query/query_shape.cpp
deleted file mode 100644
index 02d4c97d25f..00000000000
--- a/src/mongo/db/query/query_shape.cpp
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_shape.h"
-
-#include "mongo/base/status.h"
-#include "mongo/db/query/find_command_gen.h"
-#include "mongo/db/query/projection_ast_util.h"
-#include "mongo/db/query/query_request_helper.h"
-#include "mongo/db/query/query_shape_gen.h"
-#include "mongo/db/query/query_stats/query_stats.h"
-#include "mongo/db/query/shape_helpers.h"
-#include "mongo/db/query/sort_pattern.h"
-
-namespace mongo::query_shape {
-
-BSONObj debugPredicateShape(const MatchExpression* predicate) {
- SerializationOptions opts;
- opts.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString;
- return predicate->serialize(opts);
-}
-BSONObj representativePredicateShape(const MatchExpression* predicate) {
- SerializationOptions opts;
- opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue;
- return predicate->serialize(opts);
-}
-
-BSONObj debugPredicateShape(const MatchExpression* predicate,
- std::function<std::string(StringData)> transformIdentifiersCallback) {
- SerializationOptions opts;
- opts.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString;
- opts.transformIdentifiersCallback = transformIdentifiersCallback;
- opts.transformIdentifiers = true;
- return predicate->serialize(opts);
-}
-
-BSONObj representativePredicateShape(
- const MatchExpression* predicate,
- std::function<std::string(StringData)> transformIdentifiersCallback) {
- SerializationOptions opts;
- opts.literalPolicy = LiteralSerializationPolicy::kToRepresentativeParseableValue;
- opts.transformIdentifiersCallback = transformIdentifiersCallback;
- opts.transformIdentifiers = true;
- return predicate->serialize(opts);
-}
-
-BSONObj extractSortShape(const BSONObj& sortSpec,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const SerializationOptions& opts) {
- if (sortSpec.isEmpty()) {
- return sortSpec;
- }
- auto natural = sortSpec[query_request_helper::kNaturalSortField];
-
- if (!natural) {
- return SortPattern{sortSpec, expCtx}
- .serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts)
- .toBson();
- }
- // This '$natural' will fail to parse as a valid SortPattern since it is not a valid field
- // path - it is usually considered and converted into a hint. For the query shape, we'll
- // keep it unmodified.
- BSONObjBuilder bob;
- for (auto&& elem : sortSpec) {
- if (elem.isABSONObj()) {
- // We expect this won't work or parse on the main command path, but for shapification we
- // don't really care, just treat it as a literal and don't bother parsing.
- opts.appendLiteral(
- &bob, opts.serializeFieldPathFromString(elem.fieldNameStringData()), elem);
- } else if (elem.fieldNameStringData() == natural.fieldNameStringData()) {
- bob.append(elem);
- } else {
- bob.appendAs(elem, opts.serializeFieldPathFromString(elem.fieldNameStringData()));
- }
- }
- return bob.obj();
-}
-
-void addShapeLiterals(BSONObjBuilder* bob,
- const FindCommandRequest& findCommand,
- const SerializationOptions& opts) {
- if (auto limit = findCommand.getLimit()) {
- opts.appendLiteral(
- bob, FindCommandRequest::kLimitFieldName, static_cast<long long>(*limit));
- }
- if (auto skip = findCommand.getSkip()) {
- opts.appendLiteral(bob, FindCommandRequest::kSkipFieldName, static_cast<long long>(*skip));
- }
-}
-
-static std::vector<
- std::pair<StringData, std::function<const OptionalBool(const FindCommandRequest&)>>>
- boolArgMap = {
- {FindCommandRequest::kSingleBatchFieldName, &FindCommandRequest::getSingleBatch},
- {FindCommandRequest::kAllowDiskUseFieldName, &FindCommandRequest::getAllowDiskUse},
- {FindCommandRequest::kReturnKeyFieldName, &FindCommandRequest::getReturnKey},
- {FindCommandRequest::kShowRecordIdFieldName, &FindCommandRequest::getShowRecordId},
- {FindCommandRequest::kTailableFieldName, &FindCommandRequest::getTailable},
- {FindCommandRequest::kAwaitDataFieldName, &FindCommandRequest::getAwaitData},
-};
-std::vector<std::pair<StringData, std::function<const BSONObj(const FindCommandRequest&)>>>
- objArgMap = {
- {FindCommandRequest::kCollationFieldName, &FindCommandRequest::getCollation},
-
-};
-
-void addRemainingFindCommandFields(BSONObjBuilder* bob,
- const FindCommandRequest& findCommand,
- const SerializationOptions& opts) {
- for (auto [fieldName, getterFunction] : boolArgMap) {
- auto optBool = getterFunction(findCommand);
- optBool.serializeToBSON(fieldName, bob);
- }
-
- if (auto optOplogReplay = findCommand.getOplogReplay()) {
- if (optOplogReplay.has_value()) {
- opts.appendLiteral(
- bob, FindCommandRequest::kOplogReplayFieldName, optOplogReplay.value_or(false));
- }
- }
-
- auto collation = findCommand.getCollation();
- if (!collation.isEmpty()) {
- bob->append(FindCommandRequest::kCollationFieldName, collation);
- }
-}
-
-
-/**
- * In a let specification all field names are variable names, and all values are either
- * expressions or constants.
- */
-BSONObj extractLetSpecShape(BSONObj letSpec,
- const SerializationOptions& opts,
- const boost::intrusive_ptr<ExpressionContext>& expCtx) {
-
- BSONObjBuilder bob;
- for (BSONElement elem : letSpec) {
- auto expr = Expression::parseOperand(expCtx.get(), elem, expCtx->variablesParseState);
- auto redactedValue = expr->serialize(opts);
- // Note that this will throw on deeply nested let variables.
- redactedValue.addToBsonObj(&bob, opts.serializeFieldPathFromString(elem.fieldName()));
- }
- return bob.obj();
-}
-
-void appendCmdNs(BSONObjBuilder& bob,
- const NamespaceString& nss,
- const SerializationOptions& opts) {
- BSONObjBuilder nsObj = bob.subobjStart("cmdNs");
- shape_helpers::appendNamespaceShape(nsObj, nss, opts);
- nsObj.doneFast();
-}
-
-BSONObj extractQueryShape(const ParsedFindCommand& findRequest,
- const SerializationOptions& opts,
- const boost::intrusive_ptr<ExpressionContext>& expCtx) {
- const auto& findCmd = *findRequest.findCommandRequest;
- BSONObjBuilder bob;
- // Serialize the namespace as part of the query shape.
- {
- auto ns = findCmd.getNamespaceOrUUID();
- if (ns.nss().has_value()) {
- appendCmdNs(bob, *ns.nss(), opts);
- } else {
- BSONObjBuilder cmdNs = bob.subobjStart("cmdNs");
- cmdNs.append("uuid", opts.serializeIdentifier(ns.uuid()->toString()));
- cmdNs.append("db", opts.serializeIdentifier(ns.db()));
- cmdNs.doneFast();
- }
- }
-
- bob.append("command", "find");
- std::unique_ptr<MatchExpression> filterExpr;
- // Filter.
- bob.append(FindCommandRequest::kFilterFieldName, findRequest.filter->serialize(opts));
- // Let Spec.
- if (auto letSpec = findCmd.getLet()) {
- auto redactedObj = extractLetSpecShape(letSpec.get(), opts, expCtx);
- auto ownedObj = redactedObj.getOwned();
- bob.append(FindCommandRequest::kLetFieldName, std::move(ownedObj));
- }
-
- if (findRequest.proj) {
- bob.append(FindCommandRequest::kProjectionFieldName,
- projection_ast::serialize(*findRequest.proj->root(), opts));
- }
-
- if (!findCmd.getMax().isEmpty()) {
- bob.append(FindCommandRequest::kMaxFieldName,
- shape_helpers::extractMinOrMaxShape(findCmd.getMax(), opts));
- }
- if (!findCmd.getMin().isEmpty()) {
- bob.append(FindCommandRequest::kMinFieldName,
- shape_helpers::extractMinOrMaxShape(findCmd.getMin(), opts));
- }
-
- // Sort.
- if (findRequest.sort) {
- bob.append(
- FindCommandRequest::kSortFieldName,
- findRequest.sort
- ->serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts)
- .toBson());
- }
-
- // Fields for literal redaction. Adds limit and skip.
- addShapeLiterals(&bob, findCmd, opts);
-
- // Add the fields that require no redaction.
- addRemainingFindCommandFields(&bob, findCmd, opts);
-
- return bob.obj();
-}
-
-BSONObj extractQueryShape(const AggregateCommandRequest& aggregateCommand,
- const Pipeline& pipeline,
- const SerializationOptions& opts,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const NamespaceString& nss) {
- BSONObjBuilder bob;
-
- // namespace
- appendCmdNs(bob, nss, opts);
- bob.append("command", "aggregate");
-
- // pipeline
- {
- BSONArrayBuilder pipelineBab(
- bob.subarrayStart(AggregateCommandRequest::kPipelineFieldName));
- auto serializedPipeline = pipeline.serializeToBson(opts);
- for (const auto& stage : serializedPipeline) {
- pipelineBab.append(stage);
- }
- pipelineBab.doneFast();
- }
-
- // explain
- if (aggregateCommand.getExplain().has_value()) {
- bob.append(AggregateCommandRequest::kExplainFieldName, true);
- }
-
- // allowDiskUse
- if (auto param = aggregateCommand.getAllowDiskUse(); param.has_value()) {
- bob.append(AggregateCommandRequest::kAllowDiskUseFieldName, param.value_or(false));
- }
-
- // collation
- if (auto param = aggregateCommand.getCollation()) {
- bob.append(AggregateCommandRequest::kCollationFieldName, param.get());
- }
-
- // let
- if (auto letSpec = aggregateCommand.getLet()) {
- auto redactedObj = extractLetSpecShape(letSpec.get(), opts, expCtx);
- auto ownedObj = redactedObj.getOwned();
- bob.append(FindCommandRequest::kLetFieldName, std::move(ownedObj));
- }
- return bob.obj();
-}
-
-QueryShapeHash hash(const BSONObj& queryShape) {
- return QueryShapeHash::computeHash(reinterpret_cast<const uint8_t*>(queryShape.objdata()),
- queryShape.objsize());
-}
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/README.md b/src/mongo/db/query/query_shape/README.md
deleted file mode 100644
index b3c02d28e69..00000000000
--- a/src/mongo/db/query/query_shape/README.md
+++ /dev/null
@@ -1,69 +0,0 @@
-# Query Shape
-A query shape is a transformed version of a command with literal values replaced by a "canonical"
-BSON Type placeholder. Hence, different instances of a command would be considered to have the same
-query shape if they are identical once their literal values are abstracted.
-
-For example, these two queries would have the same shape:
-```js
-db.example.findOne({x: 24});
-db.example.findOne({x: 53});
-```
-While these queries would each have a distinct shape:
-```js
-db.example.findOne({x: 53, y: 1});
-db.example.findOne({x: 53});
-db.example.findOne({x: "string"});
-```
-While different literal _values_ result in the same shape (matching `x` for 23 vs 53), different
-BSON _types_ of the literal are considered distinct shapes (matching `x` for 53 vs "string").
-
-The concept of a query shape exists not just for the find command, but for many of the CRUD commands
-and aggregate. It also includes most (but not all) components of these commands, not just the query
-predicate (MatchExpresssion). In these ways, "query" is meant more generally. While some components
-included in the query shape are shared across the different types of commands (e.g., the "hint"
-field), some are unique. For example, a find command would include a `filter` while an aggregate
-command would have a `pipeline`.
-
-You can see which components are considered part of the query shape or not for each specific shape
-type in their respective "shape component" classes, whose purpose is to determine which components
-are relevant and should be included for determining the shape for specific type of command. The
-structure is as follows:
-- [`CmdSpecificShapeComponents`](query_shape.h#L65)
- - [`LetShapeComponent`](cmd_with_let_shape.h#L48)
- - [`AggCmdShapeComponents`](agg_cmd_shape.h#L82)
- - [`FindCmdShapeComponents`](find_cmd_shape.h#L48)
-
-See more information for the different shapes in their respective classes, structured as follows:
-- [`Shape`](query_shape.h)
- - [`CmdWithLetShape`](cmd_with_let_shape.h)
- - [`AggCmdShape`](agg_cmd_shape.h)
- - [`FindCmdShape`](find_cmd_shape.h)
-
-## Serialization Options
-`SerializationOptions` describes the way we serialize literal values.
-
-There are 3 different serialization options:
-- `kUnchanged`: literals are serialized unmodified
- - `{x: 5, y: "hello"}` -> `{x: 5, y: "hello"}`
-- `kToDebugTypeString`: human readable format, type string of the literal is serialized
- - `{x: 5, y: "hello"}` -> `{x: "?number", y: "?string"}`
-- `kToRepresentativeParseableValue`: literal serialized to one canonical value for given type, which
- must be parseable
- - `{x: 5, y: "hello"}` -> `{x: 1, y: "?"}`
- - An example of a query which is serialized differently due to the parseable requirement is `{x:
- {$regex: "^p.*"}}`. If we serialized the pattern as if it were a normal string we would end up
- with `{x: {$regex: "?"}}` however `"?"` is not a valid regex pattern, so this would fail
- parsing. Instead we will serialize it this way to maintain parseability, `{x: {$regex:
- "\\?"}}`, since `"\\?"` is valid regex.
-
-See [serialization_options.h](serialization_options.h) for more details.
-
-When we compute the [query shape hash](query_shape.cpp#L99-107), we use the
-`kToRepresentativeParseableValue`, since all literals of the same type will become the same value.
-This allows us to group together queries that have the same structure but different literal values
-into the same shape, since they will result in the same hash. The term we use to refer to this is
-"shapify", as we simplify the queries into their query shape.
-
-When shapifying, we try to get as close as possible to the original user input, but there are some
-stages like `$jsonSchema` and `$setWindowFields` that output "internal" stages that are already
-transformed from user input.
diff --git a/src/mongo/db/query/query_shape/SConscript b/src/mongo/db/query/query_shape/SConscript
deleted file mode 100644
index d4bddba4934..00000000000
--- a/src/mongo/db/query/query_shape/SConscript
+++ /dev/null
@@ -1,42 +0,0 @@
-# -*- mode: python -*-
-
-Import([
- "env",
- "get_option",
-])
-
-env = env.Clone()
-
-env.Library(
- target='query_shape', source=['query_shape.cpp', 'shape_helpers.cpp'], LIBDEPS=[
- '$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/db/exec/document_value/document_value',
- '$BUILD_DIR/mongo/db/pipeline/field_path',
- 'query_shape_common',
- ], LIBDEPS_PRIVATE=[
- ])
-
-env.Library(
- target='query_shape_common', source=[
- 'query_shape.idl',
- 'serialization_options.cpp',
- ], LIBDEPS=[
- '$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/db/exec/document_value/document_value',
- '$BUILD_DIR/mongo/db/pipeline/field_path',
- ], LIBDEPS_PRIVATE=[
- ])
-
-env.CppUnitTest(
- target="db_query_query_shape_test",
- source=[
- "query_shape_test.cpp",
- "query_shape_test.idl",
- ],
- LIBDEPS=[
- "$BUILD_DIR/mongo/db/auth/authmocks",
- "$BUILD_DIR/mongo/db/query/query_test_service_context",
- "$BUILD_DIR/mongo/db/service_context_d_test_fixture",
- "query_shape",
- ],
-)
diff --git a/src/mongo/db/query/query_shape/agg_cmd_shape.cpp b/src/mongo/db/query/query_shape/agg_cmd_shape.cpp
deleted file mode 100644
index e997150ecc6..00000000000
--- a/src/mongo/db/query/query_shape/agg_cmd_shape.cpp
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_shape/agg_cmd_shape.h"
-
-#include "mongo/db/query/query_shape/shape_helpers.h"
-
-namespace mongo::query_shape {
-
-AggCmdShapeComponents::AggCmdShapeComponents(
- const AggregateCommandRequest& aggRequest,
- stdx::unordered_set<NamespaceString> involvedNamespaces_,
- std::vector<BSONObj> pipeline)
- : allowDiskUse(aggRequest.getAllowDiskUse()),
- involvedNamespaces(std::move(involvedNamespaces_)),
- representativePipeline(std::move(pipeline)) {}
-
-AggCmdShapeComponents::AggCmdShapeComponents(
- OptionalBool allowDiskUse,
- stdx::unordered_set<NamespaceString> involvedNamespaces_,
- std::vector<BSONObj> pipeline)
- : allowDiskUse(allowDiskUse),
- involvedNamespaces(std::move(involvedNamespaces_)),
- representativePipeline(std::move(pipeline)) {}
-
-void AggCmdShapeComponents::HashValue(absl::HashState state) const {
- state = absl::HashState::combine(std::move(state), allowDiskUse);
- for (auto&& shapifiedStage : representativePipeline) {
- state = absl::HashState::combine(std::move(state), simpleHash(shapifiedStage));
- }
-}
-
-void AggCmdShape::appendLetCmdSpecificShapeComponents(
- BSONObjBuilder& bob,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const SerializationOptions& opts) const {
- tassert(7633000,
- "We don't support serializing to the unmodified shape here, since we have already "
- "shapified and stored the representative query - we've lost the original literals",
- opts.literalPolicy != LiteralSerializationPolicy::kUnchanged);
-
- if (opts == SerializationOptions::kRepresentativeQueryShapeSerializeOptions) {
- // We have this copy stored already!
- return _components.appendTo(bob);
- } else {
- // The cached pipeline shape doesn't match the requested options, so we have to
- // re-parse the pipeline from the initial request.
- expCtx->inMongos = _inMongos;
- expCtx->addResolvedNamespaces(_components.involvedNamespaces);
- auto reparsed = Pipeline::parse(_components.representativePipeline, expCtx);
- auto serializedPipeline = reparsed->serializeToBson(opts);
- AggCmdShapeComponents{
- _components.allowDiskUse, _components.involvedNamespaces, serializedPipeline}
- .appendTo(bob);
- }
-}
-
-void AggCmdShapeComponents::appendTo(BSONObjBuilder& bob) const {
- bob.append("command", "aggregate");
-
- // pipeline
- bob.append(AggregateCommandRequest::kPipelineFieldName, representativePipeline);
-
- // allowDiskUse
- if (allowDiskUse.has_value()) {
- bob.append(AggregateCommandRequest::kAllowDiskUseFieldName, bool(allowDiskUse));
- }
-}
-
-// As part of the size, we must track the allocation of elements in the representative
-// pipeline, as well as the elements in the unordered set of involved namespaces.
-size_t AggCmdShapeComponents::size() const {
- return sizeof(AggCmdShapeComponents) + shape_helpers::containerSize(representativePipeline) +
- shape_helpers::containerSize(involvedNamespaces);
-}
-
-AggCmdShape::AggCmdShape(const AggregateCommandRequest& aggregateCommand,
- NamespaceString origNss,
- stdx::unordered_set<NamespaceString> involvedNamespaces_,
- const Pipeline& pipeline,
- const boost::intrusive_ptr<ExpressionContext>& expCtx)
- : CmdWithLetShape(aggregateCommand.getLet(),
- expCtx,
- _components,
- std::move(origNss),
- aggregateCommand.getCollation().value_or(BSONObj())),
- _components(aggregateCommand,
- std::move(involvedNamespaces_),
- pipeline.serializeToBson(
- SerializationOptions::kRepresentativeQueryShapeSerializeOptions)),
- _inMongos(expCtx->inMongos) {}
-
-size_t AggCmdShape::extraSize() const {
- // To account for possible padding, we calculate the extra space with the difference instead of
- // using sizeof(bool);
- return sizeof(AggCmdShape) - sizeof(CmdWithLetShape) - sizeof(AggCmdShapeComponents);
-}
-
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/agg_cmd_shape.h b/src/mongo/db/query/query_shape/agg_cmd_shape.h
deleted file mode 100644
index c0ef5a7b06f..00000000000
--- a/src/mongo/db/query/query_shape/agg_cmd_shape.h
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include <boost/intrusive_ptr.hpp>
-
-#include "mongo/db/pipeline/aggregate_command_gen.h"
-#include "mongo/db/pipeline/expression_context.h"
-#include "mongo/db/pipeline/pipeline.h"
-#include "mongo/db/query/query_shape/cmd_with_let_shape.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-
-namespace mongo::query_shape {
-
-/**
- * A struct representing the aggregate command's specific components that are to be considered part
- * of the query shape.
- *
- * This struct stores the shapified version of the pipeline as a memory optimization. We'll need to
- * store the BSON version in either case, since often the parsed version needs that BSON to survive
- * as backing memory, so we store the representative pipeline shape so that we are able to parse the
- * pipeline again if we need to compute a different shape.
- */
-struct AggCmdShapeComponents : public query_shape::CmdSpecificShapeComponents {
- AggCmdShapeComponents(const AggregateCommandRequest&,
- stdx::unordered_set<NamespaceString> involvedNamespaces,
- std::vector<BSONObj> shapifiedPipeline);
-
- AggCmdShapeComponents(OptionalBool allowDiskUse,
- stdx::unordered_set<NamespaceString> involvedNamespaces,
- std::vector<BSONObj> shapifiedPipeline);
-
- size_t size() const final;
-
- void appendTo(BSONObjBuilder&) const;
-
- void HashValue(absl::HashState state) const final;
-
- OptionalBool allowDiskUse;
-
- stdx::unordered_set<NamespaceString> involvedNamespaces;
-
- // The representative query shape of the pipeline.
- std::vector<BSONObj> representativePipeline;
-};
-
-/**
- * A class representing the query shape of an aggregate command. The components are listed above.
- * This class knows how to utilize those components to serialize to BSON with any
- * SerializationOptions. Mostly this involves correctly setting up an ExpressionContext to re-parse
- * the request if needed.
- */
-class AggCmdShape : public CmdWithLetShape {
-public:
- AggCmdShape(const AggregateCommandRequest&,
- NamespaceString origNss,
- stdx::unordered_set<NamespaceString> involvedNamespaces,
- const Pipeline&,
- const boost::intrusive_ptr<ExpressionContext>&);
-
- void appendLetCmdSpecificShapeComponents(BSONObjBuilder& bob,
- const boost::intrusive_ptr<ExpressionContext>&,
- const SerializationOptions&) const final;
- size_t extraSize() const final override;
-
-private:
- AggCmdShapeComponents _components;
- // Flag to denote if the query was run on mongos. Needed to rebuild the "dummy" expression
- // context for re-parsing.
- bool _inMongos;
-};
-static_assert(sizeof(AggCmdShape) <=
- sizeof(CmdWithLetShape) + sizeof(AggCmdShapeComponents) + 8 /* bool and padding*/,
- "If the class' members have changed, this assert and the extraSize() calculation may "
- "need to be updated with a new value.");
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp b/src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp
deleted file mode 100644
index c617391f4e7..00000000000
--- a/src/mongo/db/query/query_shape/agg_cmd_shape_test.cpp
+++ /dev/null
@@ -1,266 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/bson/json.h"
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/query_shape/agg_cmd_shape.h"
-#include "mongo/db/query/query_shape/cmd_with_let_shape.h"
-#include "mongo/db/query/query_test_service_context.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo::query_shape {
-
-namespace {
-static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl");
-
-class AggCmdShapeTest : public unittest::Test {
-public:
- void setUp() final {
- _queryTestServiceContext = std::make_unique<QueryTestServiceContext>();
- _operationContext = _queryTestServiceContext->makeOperationContext();
- _expCtx = make_intrusive<ExpressionContextForTest>();
- }
-
- std::unique_ptr<AggregateCommandRequest> makeAggregateCommandRequest(
- std::vector<StringData> stagesJson,
- boost::optional<StringData> letJson = boost::none,
- boost::optional<StringData> collationJson = boost::none) {
- std::vector<BSONObj> pipeline;
- for (auto&& stage : stagesJson) {
- pipeline.push_back(fromjson(stage.rawData()));
- }
-
- auto aggRequest =
- std::make_unique<AggregateCommandRequest>(kDefaultTestNss, std::move(pipeline));
- if (letJson) {
- aggRequest->setLet(fromjson(letJson->rawData()));
- }
- if (collationJson) {
- aggRequest->setCollation(fromjson(collationJson->rawData()));
- }
- return aggRequest;
- }
-
- std::unique_ptr<AggCmdShape> makeShapeFromPipeline(
- std::vector<StringData> stagesJson,
- boost::optional<StringData> letJson = boost::none,
- boost::optional<StringData> collationJson = boost::none) {
-
- auto aggRequest = makeAggregateCommandRequest(
- std::move(stagesJson), std::move(letJson), std::move(collationJson));
-
- auto parsedPipeline = Pipeline::parse(aggRequest->getPipeline(), _expCtx);
- return std::make_unique<AggCmdShape>(*aggRequest,
- kDefaultTestNss,
- stdx::unordered_set<NamespaceString>{kDefaultTestNss},
- *parsedPipeline,
- _expCtx);
- }
- std::unique_ptr<AggCmdShapeComponents> makeShapeComponentsFromPipeline(
- std::vector<StringData> stagesJson, OptionalBool allowDiskUse = {}) {
- auto aggRequest = makeAggregateCommandRequest(std::move(stagesJson));
-
- auto parsedPipeline = Pipeline::parse(aggRequest->getPipeline(), _expCtx);
- return std::make_unique<AggCmdShapeComponents>(
- *aggRequest,
- stdx::unordered_set<NamespaceString>{kDefaultTestNss},
- parsedPipeline->serializeToBson(
- SerializationOptions::kRepresentativeQueryShapeSerializeOptions));
- }
-
- std::unique_ptr<QueryTestServiceContext> _queryTestServiceContext;
-
- ServiceContext::UniqueOperationContext _operationContext;
- boost::intrusive_ptr<ExpressionContext> _expCtx;
-};
-
-TEST_F(AggCmdShapeTest, BasicPipelineShape) {
- auto shape =
- makeShapeFromPipeline({R"({$match: {x: 3, y: {$lte: 3}}})"_sd,
- R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"});
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "cmdNs": {
- "db": "testDB",
- "coll": "testColl"
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$match": {
- "$and": [
- {
- "x": {
- "$eq": "?number"
- }
- },
- {
- "y": {
- "$lte": "?number"
- }
- }
- ]
- }
- },
- {
- "$group": {
- "_id": "$y",
- "z": {
- "$max": "$z"
- },
- "w": {
- "$avg": "$w"
- }
- }
- }
- ]
- })",
- shape->toBson(_operationContext.get(),
- SerializationOptions::kDebugQueryShapeSerializeOptions));
-}
-
-TEST_F(AggCmdShapeTest, IncludesLet) {
- auto shape = makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd},
- R"({x: 4, y: "str"})"_sd);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "cmdNs": {
- "db": "testDB",
- "coll": "testColl"
- },
- "let": {
- "x": "?number",
- "y": "?string"
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$match": {
- "x": {
- "$eq": "?number"
- }
- }
- },
- {
- "$limit": "?number"
- }
- ]
- })",
- shape->toBson(_operationContext.get(),
- SerializationOptions::kDebugQueryShapeSerializeOptions));
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "cmdNs": {
- "db": "testDB",
- "coll": "testColl"
- },
- "let": {
- "x": {
- "$const": 1
- },
- "y": {
- "$const": "?"
- }
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$match": {
- "x": {
- "$eq": 1
- }
- }
- },
- {
- "$limit": 1
- }
- ]
- })",
- shape->toBson(_operationContext.get(),
- SerializationOptions::kRepresentativeQueryShapeSerializeOptions));
-}
-
-TEST_F(AggCmdShapeTest, SizeOfAggCmdShapeComponents) {
- auto aggComponents = makeShapeComponentsFromPipeline(
- {R"({$match: {x: 3, y: {$lte: 3}}})"_sd,
- R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"},
- false /*allowDiskUse*/);
-
- // The sizes of any members of AggCmdShapeComponents are typically accounted for by
- // sizeof(AggCmdShapeComponents). The important part of the test here is to ensure that any
- // additional memory allocations are also included in the size() operation. In our case,
- // we expect additional memory use from the representative pipeline and the involved
- // namespaces set.
- const auto pipelineSize = shape_helpers::containerSize(aggComponents->representativePipeline);
- const auto involvedNamespacesSize = sizeof(kDefaultTestNss) +
- kDefaultTestNss.size(); // kDefaultTestNss is the only value in the unordered set.
-
- ASSERT_EQ(aggComponents->size(),
- sizeof(AggCmdShapeComponents) + pipelineSize + involvedNamespacesSize);
-}
-
-TEST_F(AggCmdShapeTest, EquivalentAggCmdShapeComponentSizes) {
- auto aggComponentsDiskUseFalse = makeShapeComponentsFromPipeline(
- {R"({$match: {x: 3, y: {$lte: 3}}})"_sd,
- R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"},
- false /*allowDiskUse*/);
- auto aggComponentsDiskUseTrue = makeShapeComponentsFromPipeline(
- {R"({$match: {x: 3, y: {$lte: 3}}})"_sd,
- R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"},
- true /*allowDiskUse*/);
- ASSERT_EQ(aggComponentsDiskUseFalse->size(), aggComponentsDiskUseTrue->size());
-}
-
-TEST_F(AggCmdShapeTest, DifferentAggCmdShapeComponentSizes) {
- auto smallAggComponents = makeShapeComponentsFromPipeline({R"({$match: {x: 3, y: {$lte: 3}}})"},
- false /*allowDiskUse*/);
- auto largeAggComponents = makeShapeComponentsFromPipeline(
- {R"({$match: {x: 3, y: {$lte: 3}}})"_sd,
- R"({$group: {_id: "$y", z: {$max: "$z"}, w: {$avg: "$w"}}})"},
- false /*allowDiskUse*/);
- ASSERT_LT(smallAggComponents->size(), largeAggComponents->size());
-}
-
-TEST_F(AggCmdShapeTest, SizeOfAggCmdShapeWithAndWithoutLet) {
- auto shapeWithoutLet = makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd});
- auto shapeWithLet = makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd},
- R"({x: 4, y: "str"})"_sd);
- ASSERT_LT(shapeWithoutLet->size(), shapeWithLet->size());
-}
-
-TEST_F(AggCmdShapeTest, SizeOfAggCmdShapeWithAndWithoutCollation) {
- auto shapeWithoutCollation =
- makeShapeFromPipeline({R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd});
- auto shapeWithCollation = makeShapeFromPipeline(
- {R"({$match: {x: 3}})"_sd, R"({$limit: 2})"_sd}, boost::none, R"({locale: "en_US"})"_sd);
- ASSERT_LT(shapeWithoutCollation->size(), shapeWithCollation->size());
-}
-} // namespace
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/cmd_with_let_shape.cpp b/src/mongo/db/query/query_shape/cmd_with_let_shape.cpp
deleted file mode 100644
index 2bbb6dfeadc..00000000000
--- a/src/mongo/db/query/query_shape/cmd_with_let_shape.cpp
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_shape/cmd_with_let_shape.h"
-
-namespace mongo::query_shape {
-
-namespace {
-BSONObj extractLetShape(BSONObj letSpec,
- const SerializationOptions& opts,
- const boost::intrusive_ptr<ExpressionContext>& expCtx) {
- if (letSpec.isEmpty()) {
- // Fast path for the common case.
- return letSpec;
- }
-
- BSONObjBuilder bob;
- for (BSONElement elem : letSpec) {
- auto expr = Expression::parseOperand(expCtx.get(), elem, expCtx->variablesParseState);
- auto redactedValue = expr->serialize(opts);
- // Note that this will throw on deeply nested let variables.
- redactedValue.addToBsonObj(&bob, opts.serializeFieldPathFromString(elem.fieldName()));
- }
- return bob.obj();
-}
-
-auto representativeLetShape(boost::optional<BSONObj> let,
- const boost::intrusive_ptr<ExpressionContext>& expCtx) {
- return let ? extractLetShape(
- *let, SerializationOptions::kRepresentativeQueryShapeSerializeOptions, expCtx)
- : BSONObj();
-}
-} // namespace
-
-LetShapeComponent::LetShapeComponent(boost::optional<BSONObj> let,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const CmdSpecificShapeComponents& unownedInnerComponents_)
- : shapifiedLet(representativeLetShape(let, expCtx)),
- hasLet(bool(let)),
- unownedInnerComponents(unownedInnerComponents_) {}
-
-void LetShapeComponent::HashValue(absl::HashState state) const {
- state = absl::HashState::combine(
- std::move(state), hasLet, simpleHash(shapifiedLet), unownedInnerComponents);
-}
-
-size_t LetShapeComponent::size() const {
- return sizeof(LetShapeComponent) + shapifiedLet.objsize() + unownedInnerComponents.size();
-}
-
-void LetShapeComponent::addLetBson(BSONObjBuilder& bob,
- const SerializationOptions& opts,
- const boost::intrusive_ptr<ExpressionContext>& expCtx) const {
- if (hasLet) {
- auto shapeToAppend = shapifiedLet;
- if (opts != SerializationOptions::kRepresentativeQueryShapeSerializeOptions) {
- // We have the representative query cached/stored here, but the caller is asking for a
- // different format, so we must re-compute.
- shapeToAppend = extractLetShape(shapifiedLet, opts, expCtx);
- }
- bob.append(FindCommandRequest::kLetFieldName, shapeToAppend);
- }
-}
-
-void CmdWithLetShape::appendCmdSpecificShapeComponents(BSONObjBuilder& bob,
- OperationContext* opCtx,
- const SerializationOptions& opts) const {
- auto expCtx =
- ExpressionContext::makeBlankExpressionContext(opCtx, nssOrUUID, _let.shapifiedLet);
- _let.addLetBson(bob, opts, expCtx);
- appendLetCmdSpecificShapeComponents(bob, expCtx, opts);
-}
-
-CmdWithLetShape::CmdWithLetShape(boost::optional<BSONObj> let,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const CmdSpecificShapeComponents& unownedInnerComponents,
- NamespaceStringOrUUID nssOrUUID,
- BSONObj collation)
- : Shape(nssOrUUID, collation), _let(let, expCtx, unownedInnerComponents) {}
-
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/cmd_with_let_shape.h b/src/mongo/db/query/query_shape/cmd_with_let_shape.h
deleted file mode 100644
index a1c127b9999..00000000000
--- a/src/mongo/db/query/query_shape/cmd_with_let_shape.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/bson/bsonobj.h"
-#include "mongo/db/pipeline/expression_context.h"
-#include "mongo/db/query/parsed_find_command.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
-
-namespace mongo::query_shape {
-
-/**
- * This struct is bit of a weird one. We want to use it as the shape's _entire_ "specific
- * components" (rather than introduce more virtual functions to that interface). So, we track here
- * the let component (as the name suggests) but we also keep an unowned reference to the specific
- * components of CmdWithLetShape sub-classes. This class doesn't really do all that much with those
- * components except track a reference to them and ensure their size is accounted for and their hash
- * value is incorporated.
- */
-struct LetShapeComponent : public CmdSpecificShapeComponents {
- LetShapeComponent(boost::optional<BSONObj> let,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const CmdSpecificShapeComponents& unownedInnerComponents);
-
- /**
- * Hashes to include the shapified let parameters and also the hash of 'unownedInnerComponents'.
- */
- void HashValue(absl::HashState state) const final;
-
- /**
- * Includes the size of the let parameters and the size of 'unownedInnerComponents.'
- */
- size_t size() const final;
-
- /**
- * Adds _only_ the let params.
- */
- void addLetBson(BSONObjBuilder&,
- const SerializationOptions&,
- const boost::intrusive_ptr<ExpressionContext>&) const;
-
- BSONObj shapifiedLet;
- bool hasLet;
- // Tracked so that this can be hash combined correctly.
- const CmdSpecificShapeComponents& unownedInnerComponents;
-};
-
-/**
- * The 'let' command argument is semi-generic in that it is supported in a couple commands. However
- * it is treated specially since it supports using expressions as the let constants. Using
- * expressions induces a library dependency that we don't want in the Shape interface itself. So
- * this class handles tracking and adding the 'let' component of the shape for sub-classes.
- */
-class CmdWithLetShape : public Shape {
-public:
- CmdWithLetShape(boost::optional<BSONObj> let,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const CmdSpecificShapeComponents& unownedInnerComponents,
- NamespaceStringOrUUID,
- BSONObj collation_);
-
- const CmdSpecificShapeComponents& specificComponents() const final {
- return _let;
- }
-
-protected:
- void appendCmdSpecificShapeComponents(BSONObjBuilder&,
- OperationContext* opCtx,
- const SerializationOptions& opts) const final;
- virtual void appendLetCmdSpecificShapeComponents(
- BSONObjBuilder&,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const SerializationOptions&) const = 0;
-
- LetShapeComponent _let;
-};
-static_assert(sizeof(CmdWithLetShape) == sizeof(Shape) + sizeof(LetShapeComponent),
- "If the class' members have changed, this assert and the extraSize() calculation may "
- "need to be updated with a new value.");
-
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp b/src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp
deleted file mode 100644
index 21812d97fc7..00000000000
--- a/src/mongo/db/query/query_shape/cmd_with_let_shape_test.cpp
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Copyright (C) 2024-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/query_shape/cmd_with_let_shape.h"
-#include "mongo/db/service_context_test_fixture.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo::query_shape {
-
-namespace {
-static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl");
-
-class CmdWithLetShapeTest : public unittest::Test {};
-
-
-struct DummyInnerComponent : public CmdSpecificShapeComponents {
- DummyInnerComponent(){};
- void HashValue(absl::HashState state) const {}
- size_t size() const final {
- return sizeof(*this);
- }
-};
-
-TEST_F(CmdWithLetShapeTest, SizeOfLetShapeComponent) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
- auto let = fromjson(R"({x: 4, y: "str"})");
- auto innerComponents = std::make_unique<DummyInnerComponent>();
- auto components = std::make_unique<LetShapeComponent>(let, expCtx, *innerComponents);
-
- const auto minimumSize = sizeof(CmdSpecificShapeComponents) + sizeof(BSONObj) + sizeof(bool) +
- sizeof(void*) /*CmdSpecificShapeComponents&*/ +
- static_cast<size_t>(components->shapifiedLet.objsize()) +
- components->unownedInnerComponents.size();
-
- ASSERT_GTE(components->size(), minimumSize);
- ASSERT_LTE(components->size(), minimumSize + 8 /*padding*/);
-}
-
-TEST_F(CmdWithLetShapeTest, SizeOfComponentWithAndWithoutLet) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
- auto let = fromjson(R"({x: 4, y: "str"})");
- auto innerComponents = std::make_unique<DummyInnerComponent>();
- auto componentsWithLet = std::make_unique<LetShapeComponent>(let, expCtx, *innerComponents);
- auto componentsWithNoLet =
- std::make_unique<LetShapeComponent>(boost::none, expCtx, *innerComponents);
-
- ASSERT_LT(componentsWithNoLet->size(), componentsWithLet->size());
-}
-
-} // namespace
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/find_cmd_shape.cpp b/src/mongo/db/query/query_shape/find_cmd_shape.cpp
deleted file mode 100644
index 2d018de2619..00000000000
--- a/src/mongo/db/query/query_shape/find_cmd_shape.cpp
+++ /dev/null
@@ -1,227 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_shape/find_cmd_shape.h"
-
-#include "mongo/db/query/projection_ast_util.h"
-#include "mongo/db/query/query_shape/shape_helpers.h"
-
-namespace mongo::query_shape {
-namespace {
-
-BSONObj projectionShape(const boost::optional<projection_ast::Projection>& proj,
- const SerializationOptions& opts =
- SerializationOptions::kRepresentativeQueryShapeSerializeOptions) {
- return proj ? projection_ast::serialize(*proj->root(), opts) : BSONObj();
-}
-
-BSONObj sortShape(const boost::optional<SortPattern>& sort,
- const SerializationOptions& opts =
- SerializationOptions::kRepresentativeQueryShapeSerializeOptions) {
- return sort
- ? sort->serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts)
- .toBson()
- : BSONObj();
-}
-
-void maybeAddWithName(const OptionalBool& optBool, BSONObjBuilder& bob, StringData name) {
- if (optBool.has_value()) {
- bob.append(name, bool(optBool));
- }
-}
-
-void addRemainingFindCommandFields(const FindCmdShapeComponents& components, BSONObjBuilder& bob) {
- maybeAddWithName(components.singleBatch, bob, FindCommandRequest::kSingleBatchFieldName);
- maybeAddWithName(components.allowDiskUse, bob, FindCommandRequest::kAllowDiskUseFieldName);
- maybeAddWithName(components.returnKey, bob, FindCommandRequest::kReturnKeyFieldName);
- maybeAddWithName(components.showRecordId, bob, FindCommandRequest::kShowRecordIdFieldName);
- maybeAddWithName(components.tailable, bob, FindCommandRequest::kTailableFieldName);
- maybeAddWithName(components.awaitData, bob, FindCommandRequest::kAwaitDataFieldName);
- maybeAddWithName(components.oplogReplay, bob, FindCommandRequest::kOplogReplayFieldName);
-}
-
-} // namespace
-
-FindCmdShapeComponents::FindCmdShapeComponents(
- const ParsedFindCommand& request,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const SerializationOptions& opts)
- : filter(request.filter->serialize(opts)),
- projection(projectionShape(request.proj, opts)),
- sort(sortShape(request.sort, opts)),
- min(shape_helpers::extractMinOrMaxShape(request.findCommandRequest->getMin(), opts)),
- max(shape_helpers::extractMinOrMaxShape(request.findCommandRequest->getMax(), opts)),
- singleBatch(request.findCommandRequest->getSingleBatch()),
- allowDiskUse(request.findCommandRequest->getAllowDiskUse().has_value()
- ? boost::optional<bool>(bool(request.findCommandRequest->getAllowDiskUse()))
- : boost::none),
- returnKey(request.findCommandRequest->getReturnKey()),
- showRecordId(request.findCommandRequest->getShowRecordId()),
- tailable(request.findCommandRequest->getTailable()),
- awaitData(request.findCommandRequest->getAwaitData()),
- oplogReplay(request.findCommandRequest->getOplogReplay()),
- hasField(),
- serializationOpts(opts) {
- hasField.projection = request.proj.has_value();
- hasField.sort = request.sort.has_value();
- hasField.limit = request.findCommandRequest->getLimit().has_value();
- hasField.skip = request.findCommandRequest->getSkip().has_value();
-}
-
-void FindCmdShapeComponents::appendTo(BSONObjBuilder& bob) const {
-
- bob.append("command", "find");
-
- std::unique_ptr<MatchExpression> filterExpr;
- // Filter.
- bob.append(FindCommandRequest::kFilterFieldName, filter);
-
- if (hasField.projection) {
- bob.append(FindCommandRequest::kProjectionFieldName, projection);
- }
-
- if (!max.isEmpty()) {
- bob.append(FindCommandRequest::kMaxFieldName, max);
- }
- if (!min.isEmpty()) {
- bob.append(FindCommandRequest::kMinFieldName, min);
- }
-
- // Sort.
- if (hasField.sort) {
- bob.append(FindCommandRequest::kSortFieldName, sort);
- }
-
- // The values here don't matter (assuming we're not using the 'kUnchanged' policy).
- tassert(7973601,
- "Serialization policy not supported - original values have been discarded",
- serializationOpts.literalPolicy != LiteralSerializationPolicy::kUnchanged);
- if (hasField.limit) {
- serializationOpts.appendLiteral(&bob, FindCommandRequest::kLimitFieldName, 1ll);
- }
- if (hasField.skip) {
- serializationOpts.appendLiteral(&bob, FindCommandRequest::kSkipFieldName, 1ll);
- }
-
- // Add the fields that require no transformation.
- addRemainingFindCommandFields(*this, bob);
-}
-
-void FindCmdShapeComponents::HashValue(absl::HashState state) const {
- absl::HashState::combine(std::move(state),
- simpleHash(filter),
- simpleHash(projection),
- simpleHash(sort),
- simpleHash(min),
- simpleHash(max),
- singleBatch,
- allowDiskUse,
- returnKey,
- showRecordId,
- tailable,
- awaitData,
- oplogReplay,
- hasField);
-}
-
-std::unique_ptr<FindCommandRequest> FindCmdShape::toFindCommandRequest() const {
- auto fcr = std::make_unique<FindCommandRequest>(nssOrUUID);
-
- fcr->setFilter(components.filter);
- if (components.hasField.projection)
- fcr->setProjection(components.projection);
- if (components.hasField.sort)
- fcr->setSort(components.sort);
-
- fcr->setMin(components.min);
- fcr->setMax(components.max);
-
- // Doesn't matter what value to use for limit and skip in the context of a shape.
- if (components.hasField.limit)
- fcr->setLimit(1ll);
- if (components.hasField.skip)
- fcr->setSkip(1ll);
-
- // All the booleans.
- if (components.singleBatch.has_value())
- fcr->setSingleBatch(bool(components.singleBatch));
- if (components.allowDiskUse.has_value())
- fcr->setAllowDiskUse(bool(components.allowDiskUse));
- if (components.returnKey.has_value())
- fcr->setReturnKey(bool(components.returnKey));
- if (components.showRecordId.has_value())
- fcr->setShowRecordId(bool(components.showRecordId));
- if (components.tailable.has_value())
- fcr->setTailable(bool(components.tailable));
- if (components.awaitData.has_value())
- fcr->setAwaitData(bool(components.awaitData));
- if (components.oplogReplay.has_value())
- fcr->setOplogReplay(bool(components.oplogReplay));
-
- // Common shape components.
- if (_let.hasLet)
- fcr->setLet(_let.shapifiedLet);
- if (!collation.isEmpty())
- fcr->setCollation(collation);
-
-
- return fcr;
-}
-
-FindCmdShape::FindCmdShape(const ParsedFindCommand& findRequest,
- const boost::intrusive_ptr<ExpressionContext>& expCtx)
- : CmdWithLetShape(findRequest.findCommandRequest->getLet(),
- expCtx,
- components,
- findRequest.findCommandRequest->getNamespaceOrUUID(),
- findRequest.findCommandRequest->getCollation()),
- components(findRequest, expCtx) {}
-
-void FindCmdShape::appendLetCmdSpecificShapeComponents(
- BSONObjBuilder& bob,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const SerializationOptions& opts) const {
- if (opts == SerializationOptions::kRepresentativeQueryShapeSerializeOptions) {
- // Fast path: we already have this.
- return components.appendTo(bob);
- } else {
- // Slow path: we need to re-parse from our representative shapes.
- auto request = uassertStatusOKWithContext(
- parsed_find_command::parse(expCtx,
- toFindCommandRequest(),
- ExtensionsCallbackNoop(),
- MatchExpressionParser::kAllowAllSpecialFeatures),
- "Could not re-parse a representative query shape");
-
- // This constructor will shapify according to the options.
- FindCmdShapeComponents{*request, expCtx, opts}.appendTo(bob);
- }
-}
-
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/find_cmd_shape.h b/src/mongo/db/query/query_shape/find_cmd_shape.h
deleted file mode 100644
index 49d70b8ec27..00000000000
--- a/src/mongo/db/query/query_shape/find_cmd_shape.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/db/query/query_shape/cmd_with_let_shape.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_shape/shape_helpers.h"
-
-namespace mongo::query_shape {
-
-/**
- * This struct tracks the components of a find command which are important for the find query shape.
- * It attempts to only track those which are _unique_ to a find command - common elements should go
- * on some super class.
- *
- * Data elements which are shapified like 'filter' are stored in their shapified form. By default
- * and in most cases this will be the representative query shape form so that it can be re-parsed,
- * but as a convenience for serializing it is also supported to construct and serialize this with
- * other options.
- */
-struct FindCmdShapeComponents : public CmdSpecificShapeComponents {
-
- FindCmdShapeComponents(const ParsedFindCommand& request,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const SerializationOptions& opts =
- SerializationOptions::kRepresentativeQueryShapeSerializeOptions);
-
- /**
- * Appends using the SerializationOptions given in the constructor.
- */
- void appendTo(BSONObjBuilder&) const;
-
- size_t size() const final {
- return sizeof(FindCmdShapeComponents) + filter.objsize() + projection.objsize() +
- sort.objsize() + min.objsize() + max.objsize();
- }
-
- BSONObj filter;
- BSONObj projection;
- BSONObj sort;
- BSONObj min;
- BSONObj max;
-
- OptionalBool singleBatch;
- OptionalBool allowDiskUse;
- OptionalBool returnKey;
- OptionalBool showRecordId;
- OptionalBool tailable;
- OptionalBool awaitData;
- OptionalBool oplogReplay;
-
- // This anonymous struct represents the presence of the member variables as C++ bit fields.
- // In doing so, each of these boolean values takes up 1 bit instead of 1 byte.
- struct HasField {
- HasField() : projection(false), sort(false), limit(false), skip(false) {}
- bool projection : 1;
- bool sort : 1;
- bool limit : 1;
- bool skip : 1;
- } hasField;
-
- // We save a copy of the options used when constructed so we know how to properly append things
- // like limit and skip - either a 1 or "?number". We could have the caller pass the options
- // again during 'appendTo()', but this introduces a risk that the options provided are different
- // than the ones we used to compute 'filter' and the other components.
- SerializationOptions serializationOpts;
-
- void HashValue(absl::HashState state) const final;
-};
-
-class FindCmdShape : public CmdWithLetShape {
-public:
- FindCmdShape(const ParsedFindCommand& findRequest,
- const boost::intrusive_ptr<ExpressionContext>& expCtx);
-
- /**
- * Assembles a parseable FindCommandRequest representing this shape - some of the pieces are
- * stored right here in the shape, others are in parent classes.
- */
- std::unique_ptr<FindCommandRequest> toFindCommandRequest() const;
-
- FindCmdShapeComponents components;
-
-protected:
- void appendLetCmdSpecificShapeComponents(BSONObjBuilder& bob,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const SerializationOptions& opts) const final;
-};
-
-template <typename H>
-H AbslHashValue(H h, const FindCmdShapeComponents::HasField& hasField) {
- return H::combine(
- std::move(h), hasField.projection, hasField.sort, hasField.limit, hasField.skip);
-}
-
-// This assertion is still active on the maintained master branch. On the v6.0 branch, we disable it
-// since it is not passing on all toolchains/platforms - notably x86 macOS. The intent of the
-// assertion is to prevent accidental additions of data members, which should not happen on this
-// branch without first happening on the master branch and passing that assertion.
-// static_assert(sizeof(FindCmdShape) == sizeof(CmdWithLetShape) + sizeof(FindCmdShapeComponents),
-// "If the class' members have changed, this assert and the extraSize() calculation
-// may " "need to be updated with a new value.");
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/find_cmd_shape_test.cpp b/src/mongo/db/query/query_shape/find_cmd_shape_test.cpp
deleted file mode 100644
index 0d839a5d3d2..00000000000
--- a/src/mongo/db/query/query_shape/find_cmd_shape_test.cpp
+++ /dev/null
@@ -1,238 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/query_shape/find_cmd_shape.h"
-#include "mongo/db/service_context_test_fixture.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo::query_shape {
-
-namespace {
-/**
- * TODO this was stolen from another test. Time for a library?
- * Simplistic redaction strategy for testing which appends the field name to the prefix "REDACT_".
- */
-std::string applyHmacForTest(StringData sd) {
- return "REDACT_" + sd.toString();
-}
-
-static const NamespaceStringOrUUID kDefaultTestNss =
- NamespaceStringOrUUID{NamespaceString("testDB.testColl")};
-
-struct RequestOptions {
- OptionalBool singleBatch = {};
- OptionalBool allowDiskUse = {};
- OptionalBool returnKey = {};
- OptionalBool showRecordId = {};
- OptionalBool tailable = {};
- OptionalBool awaitData = {};
- OptionalBool limit = {};
- OptionalBool skip = {};
-};
-class FindCmdShapeTest : public ServiceContextTest {
-public:
- void setUp() final {
- _expCtx = make_intrusive<ExpressionContextForTest>();
- }
-
- std::unique_ptr<FindCmdShape> makeShapeFromSort(StringData sortJson) {
- auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcr->setSort(fromjson(sortJson.rawData()));
- auto&& parsedRequest =
- uassertStatusOK(::mongo::parsed_find_command::parse(_expCtx, std::move(fcr)));
- return std::make_unique<FindCmdShape>(*parsedRequest, _expCtx);
- }
-
- BSONObj sortShape(StringData sortJson) {
- auto shape = makeShapeFromSort(sortJson);
- return shape->components.sort;
- }
-
- /**
- * Returns the shape of the input sort, or boost::none if the input shape was a natural sort
- * which got converted into a hint.
- */
- boost::optional<BSONObj> maybeRedactedSortShape(StringData sortJson) {
- auto shape = makeShapeFromSort(sortJson);
- SerializationOptions opts = SerializationOptions::kDebugQueryShapeSerializeOptions;
- opts.transformIdentifiers = true;
- opts.transformIdentifiersCallback = applyHmacForTest;
- auto shapeBson = shape->toBson(_expCtx->opCtx, opts);
- if (auto sortElem = shapeBson["sort"]; !sortElem.eoo()) {
- return sortElem.Obj().getOwned();
- }
- return boost::none;
- }
-
- BSONObj redactedSortShape(StringData sortJson) {
- return *maybeRedactedSortShape(sortJson);
- }
-
- boost::intrusive_ptr<ExpressionContext> _expCtx;
-
- std::unique_ptr<FindCmdShapeComponents> makeShapeComponentsFromFilter(
- BSONObj filter, const RequestOptions& requestOptions = {}) {
- auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcr->setFilter(filter.getOwned());
- fcr->setSingleBatch(requestOptions.singleBatch);
- fcr->setAllowDiskUse(requestOptions.allowDiskUse);
- fcr->setReturnKey(requestOptions.returnKey);
- fcr->setAllowDiskUse(requestOptions.showRecordId);
- fcr->setTailable(requestOptions.tailable);
- fcr->setAwaitData(requestOptions.awaitData);
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)}));
- return std::make_unique<FindCmdShapeComponents>(*parsedFind, _expCtx);
- }
-
- std::unique_ptr<FindCmdShape> makeShapeFromFilter(const BSONObj& filter) {
- auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcr->setFilter(filter.getOwned());
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)}));
- return std::make_unique<FindCmdShape>(*parsedFind, _expCtx);
- }
-};
-
-TEST_F(FindCmdShapeTest, NormalSortPattern) {
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({"a.b.c":1,"foo":-1})",
- sortShape(R"({"a.b.c": 1, "foo": -1})"));
-}
-
-TEST_F(FindCmdShapeTest, NaturalSortPattern) {
- // $natural sorts are interpreted as a hint. Hints are not part of the shape (but should show up
- // in the query stats key).
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({})",
- sortShape(R"({$natural: 1})"));
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({})",
- sortShape(R"({$natural: -1})"));
-}
-
-TEST_F(FindCmdShapeTest, NaturalSortPatternWithMeta) {
- ASSERT_THROWS_CODE(
- sortShape(R"({$natural: 1, x: {$meta: "textScore"}})"), DBException, ErrorCodes::BadValue);
-}
-
-TEST_F(FindCmdShapeTest, MetaPatternWithoutNatural) {
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({"normal":1,"$computed1":{"$meta":"textScore"}})",
- sortShape(R"({normal: 1, x: {$meta: "textScore"}})"));
-}
-
-// Here we have one test to ensure that the redaction policy is accepted and applied in the
-// query_shape utility, but there are more extensive redaction tests in sort_pattern_test.cpp
-TEST_F(FindCmdShapeTest, RespectsRedactionPolicy) {
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({"REDACT_normal":1,"REDACT_y":1})",
- redactedSortShape(R"({normal: 1, y: 1})"));
-
- // No need to redact $natural. Again, this will be interpreted as a hint, but this test is
- // interesting to ensure the $-prefix of $natural doesn't confuse us.
- ASSERT(!maybeRedactedSortShape(R"({$natural: 1})"));
-}
-
-TEST_F(FindCmdShapeTest, SizeOfShapeComponents) {
- auto query = BSON("query" << 1 << "xEquals" << 42);
- auto findCmdComponent = makeShapeComponentsFromFilter(query.getOwned());
- const auto querySize = findCmdComponent->filter.objsize();
-
- const auto minimumSize = sizeof(FindCmdShapeComponents) + querySize;
- ASSERT_GT(findCmdComponent->size(), minimumSize);
- ASSERT_LTE(findCmdComponent->size(),
- minimumSize + static_cast<size_t>(4 * BSONObj().objsize()));
-}
-
-TEST_F(FindCmdShapeTest, EquivalentShapeComponentsSizes) {
- auto query = BSON("query" << 1 << "xEquals" << 42);
- // Tailable can not be set together with 'singleBatch' option.
- auto mostlyTrueComponent = makeShapeComponentsFromFilter(query.getOwned(),
- {/* singleBatch = */ false,
- /* allowDiskUse = */ true,
- /* returnKey = */ true,
- /* showRecordId = */ true,
- /* tailable = */ true,
- /* awaitData = */ true,
- /* limit = */ true,
- /* skip = */ true});
-
- auto mostlyFalseComponent = makeShapeComponentsFromFilter(query.getOwned(),
- {/* singleBatch = */ false,
- /* allowDiskUse = */ false,
- /* returnKey = */ false,
- /* showRecordId = */ false,
- /* tailable = */ true,
- /* awaitData = */ false,
- /* limit = */ false,
- /* skip = */ false});
-
- ASSERT_EQ(mostlyTrueComponent->size(), mostlyFalseComponent->size());
-}
-
-TEST_F(FindCmdShapeTest, DifferentShapeComponentsSizes) {
- auto smallQuery = BSON("query" << BSONObj());
- auto smallFindCmdComponent = makeShapeComponentsFromFilter(smallQuery.getOwned());
-
- auto largeQuery = BSON("query" << 1 << "xEquals" << 42);
- auto largeFindCmdComponent = makeShapeComponentsFromFilter(largeQuery.getOwned());
-
- ASSERT_LT(smallQuery.objsize(), largeQuery.objsize());
- ASSERT_LT(smallFindCmdComponent->size(), largeFindCmdComponent->size());
-}
-
-TEST_F(FindCmdShapeTest, SizeOfShapeWithAndWithoutLet) {
- auto filter = BSON("query" << 1 << "xEquals" << 42);
- auto shapeWithoutLet = makeShapeFromFilter(filter.getOwned());
-
- auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcr->setFilter(filter.getOwned());
- fcr->setLet(fromjson(R"({x: 4})"));
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)}));
- auto shapeWithLet = std::make_unique<FindCmdShape>(*parsedFind, _expCtx);
-
- ASSERT_LT(shapeWithoutLet->size(), shapeWithLet->size());
-}
-
-TEST_F(FindCmdShapeTest, SizeOfShapeWithAndWithoutCollation) {
- auto filter = BSON("query" << 1 << "xEquals" << 42);
- auto shapeWithoutCollation = makeShapeFromFilter(filter.getOwned());
-
- auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcr->setFilter(filter.getOwned());
- fcr->setCollation(fromjson(R"({locale: "en_US"})"));
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(_expCtx, {std::move(fcr)}));
- auto shapeWithCollation = std::make_unique<FindCmdShape>(*parsedFind, _expCtx);
-
- ASSERT_LT(shapeWithoutCollation->size(), shapeWithCollation->size());
-}
-
-} // namespace
-
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/query_shape.cpp b/src/mongo/db/query/query_shape/query_shape.cpp
deleted file mode 100644
index 2fa0520120e..00000000000
--- a/src/mongo/db/query/query_shape/query_shape.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_shape/query_shape.h"
-
-#include "mongo/base/status.h"
-#include "mongo/crypto/sha256_block.h"
-#include "mongo/db/query/find_command_gen.h"
-#include "mongo/db/query/projection_ast_util.h"
-#include "mongo/db/query/query_request_helper.h"
-#include "mongo/db/query/query_shape/agg_cmd_shape.h"
-#include "mongo/db/query/query_shape/find_cmd_shape.h"
-#include "mongo/db/query/query_shape/query_shape_gen.h"
-#include "mongo/db/query/query_shape/shape_helpers.h"
-#include "mongo/db/query/sort_pattern.h"
-
-namespace mongo::query_shape {
-
-namespace {
-void appendCmdNs(BSONObjBuilder& bob,
- const NamespaceString& nss,
- const SerializationOptions& opts) {
- BSONObjBuilder nsObj = bob.subobjStart("cmdNs");
- shape_helpers::appendNamespaceShape(nsObj, nss, opts);
- nsObj.doneFast();
-}
-} // namespace
-
-Shape::Shape(NamespaceStringOrUUID nssOrUUID_, BSONObj collation_)
- : nssOrUUID(nssOrUUID_), collation(std::move(collation_)) {}
-
-
-BSONObj Shape::toBson(OperationContext* opCtx, const SerializationOptions& opts) const {
- BSONObjBuilder bob;
- appendCmdNsOrUUID(bob, opts);
- if (!collation.isEmpty()) {
- // Collation is never shapified. We use find command's collation name definition, but it
- // should be the same for all requests.
- bob.append(FindCommandRequest::kCollationFieldName, collation);
- }
- appendCmdSpecificShapeComponents(bob, opCtx, opts);
- return bob.obj();
-}
-
-size_t Shape::size() const {
- return sizeof(Shape) + shape_helpers::optionalObjSize(collation) + specificComponents().size() +
- extraSize();
-}
-
-QueryShapeHash Shape::sha256Hash(OperationContext* opCtx) const {
- // The Query Shape Hash should use the representative query shape.
- auto serialized =
- toBson(opCtx, SerializationOptions::kRepresentativeQueryShapeSerializeOptions);
- return SHA256Block::computeHash((const uint8_t*)serialized.sharedBuffer().get(),
- serialized.objsize());
-}
-
-void Shape::appendCmdNsOrUUID(BSONObjBuilder& bob, const SerializationOptions& opts) const {
- if (nssOrUUID.nss()) {
- appendCmdNs(bob, *nssOrUUID.nss(), opts);
- } else {
- BSONObjBuilder cmdNs = bob.subobjStart("cmdNs");
- cmdNs.append("uuid", opts.serializeIdentifier(nssOrUUID.uuid()->toString()));
- cmdNs.append("db", opts.serializeIdentifier(nssOrUUID.db()));
- cmdNs.doneFast();
- }
-}
-
-void Shape::appendCmdNs(BSONObjBuilder& bob,
- const NamespaceString& nss,
- const SerializationOptions& opts) const {
- BSONObjBuilder nsObj = bob.subobjStart("cmdNs");
- shape_helpers::appendNamespaceShape(nsObj, nss, opts);
- nsObj.doneFast();
-}
-
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/query_shape.h b/src/mongo/db/query/query_shape/query_shape.h
deleted file mode 100644
index dc83cbab127..00000000000
--- a/src/mongo/db/query/query_shape/query_shape.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/db/matcher/expression.h"
-#include "mongo/db/pipeline/aggregate_command_gen.h"
-#include "mongo/db/query/find_command_gen.h"
-#include "mongo/db/query/query_request_helper.h"
-#include "mongo/db/query/query_shape/shape_helpers.h"
-
-namespace mongo::query_shape {
-
-/**
- * Each type of "query" command likely has different fields/options that are considered important
- * for the shape. For example, a find command has a skip and a limit, and an aggregate command has a
- * pipeline. This interface is used to allow different sub-commands to diverge in this way but still
- * ensure we can appropriately hash them to compare their shapes, and properly account for their
- * size.
- *
- * This struct is split out as a separate inheritence hierarchy from 'Shape' to make it easier to
- * ensure each piece is hashed without sub-classes needing to enumerate the parent class's member
- * variables.
- */
-struct CmdSpecificShapeComponents {
- virtual ~CmdSpecificShapeComponents() {}
-
- /**
- * Sub-classes should implement this in a way which includes all shape-relevant state. If two
- * shapes should compare equal, they should result in the same hash value. For example for the
- * find command - we would include the _shapified_ filter and projection here, but we will not
- * include the comment - which is not part of the shape.
- */
- virtual void HashValue(absl::HashState state) const = 0;
-
- /**
- * It is important for shape components to accurately report their size, and to make a
- * reasonable effort to maintain a minimal size. We use the query shape in memory-constrained
- * data structures, so a bigger shape means we can have fewer different shapes stored (for
- * example in the query stats store).
- *
- * We cannot just use sizeof() because there are some variable size data members (like BSON
- * objects) which depend on the particular instance.
- */
- virtual size_t size() const = 0;
-
- // Some template boilerplate to allow sub-classes to overload the hash implementation.
- template <typename H>
- friend H AbslHashValue(H state, const CmdSpecificShapeComponents& value) {
- value.HashValue(absl::HashState::Create(&state));
- return std::move(state);
- }
-};
-
-using QueryShapeHash = SHA256Block;
-
-/**
- * A query "shape" is a version of a command with literal values abstracted so that two instances of
- * the command may compare/hash equal even if they use slightly different literal values. This
- * concept exists not just the find command, but planned for many of the CRUD commands + aggregate.
- * It also includes most (but not all) components of these commands, not just the query predicate
- * (MatchExpresssion). In these ways, "query" is meant more generally.
- *
- * A "Query Shape" can vary depending on the command (e.g. find, aggregate, or distinct). This
- * abstract struct is the API we must implement for each command which we want to have a "shape"
- * concept.
- *
- * In order to properly account for the size of a query shape, the CmdSpecificShapeComponents should
- * include all meaningful memory consumption, and be sure to report it in 'size()'. Subclasses of
- * 'Shape' are not expected to have any meaningful memory usage outside of that struct.
- */
-class Shape {
-public:
- virtual ~Shape() {}
-
- /**
- * Sub-classes are expected to implement this as a mechanism for plugging in their command
- * specific shape components.
- */
- virtual const CmdSpecificShapeComponents& specificComponents() const = 0;
-
- /**
- * Note this may involve re-parsing command BSON and so is not necessarily cheap.
- */
- BSONObj toBson(OperationContext*, const SerializationOptions&) const;
-
- /**
- * The Query Shape Hash is defined to be the SHA256 Hash of the representatice query shape. This
- * helper computes that.
- */
- QueryShapeHash sha256Hash(OperationContext*) const;
-
- /**
- * The size of a query shape is important, since we store these in space-constrained
- * environments like the query stats store.
- */
- size_t size() const;
-
- /**
- * This should be overriden by a child class if it has members whose sizes are not included in
- * specificComponents().size().
- */
- virtual size_t extraSize() const {
- return 0;
- }
- template <typename H>
- friend H AbslHashValue(H h, const Shape& shape) {
- h = H::combine(std::move(h), shape.nssOrUUID, shape.specificComponents());
- if (!shape.collation.isEmpty())
- h = H::combine(std::move(h), simpleHash(shape.collation));
- return h;
- }
-
-
- // Not shapified but it is an identifier so it may be transformed.
- NamespaceStringOrUUID nssOrUUID;
-
- // Never shapified. If it's empty, leave it off.
- BSONObj collation;
-
-protected:
- Shape(NamespaceStringOrUUID, BSONObj collation_);
-
- /**
- * Along with the hash implementation, this is the main way that shapes are 'shapified' -
- * sub-classes should implement this to add the shapified versions of their literals to an
- * object. Depending on 'opts', this may be eligible to be used for output in $queryStats or as
- * the object to compute the QueryShapeHash.
- */
- virtual void appendCmdSpecificShapeComponents(BSONObjBuilder&,
- OperationContext*,
- const SerializationOptions& opts) const = 0;
-
-private:
- void appendCmdNsOrUUID(BSONObjBuilder&, const SerializationOptions&) const;
- void appendCmdNs(BSONObjBuilder&, const NamespaceString&, const SerializationOptions&) const;
-};
-
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/query_shape.idl b/src/mongo/db/query/query_shape/query_shape.idl
deleted file mode 100644
index 77e71756467..00000000000
--- a/src/mongo/db/query/query_shape/query_shape.idl
+++ /dev/null
@@ -1,50 +0,0 @@
-# Copyright (C) 2023-present MongoDB, Inc.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the Server Side Public License, version 1,
-# as published by MongoDB, Inc.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Server Side Public License for more details.
-#
-# You should have received a copy of the Server Side Public License
-# along with this program. If not, see
-# <http://www.mongodb.com/licensing/server-side-public-license>.
-#
-# As a special exception, the copyright holders give permission to link the
-# code of portions of this program with the OpenSSL library under certain
-# conditions as described in each individual source file and distribute
-# linked combinations including the program with the OpenSSL library. You
-# must comply with the Server Side Public License in all respects for
-# all of the code used other than as permitted herein. If you modify file(s)
-# with this exception, you may extend this exception to your version of the
-# file(s), but you are not obligated to do so. If you do not wish to do so,
-# delete this exception statement from your version. If you delete this
-# exception statement from all source files in the program, then also delete
-# it in the license file.
-
-global:
- cpp_namespace: "mongo::query_shape"
-
-imports:
- - "mongo/idl/basic_types.idl"
-
-
-structs:
- CommandNamespace:
- description: "Representation of the cmdNs sub-object of the query shape."
- fields:
- db:
- type: string
- coll:
- type: string
- optional: true
- uuid:
- type: string
- optional: true
- tenantId:
- type: string
- optional: true
- \ No newline at end of file
diff --git a/src/mongo/db/query/query_shape/query_shape_test.cpp b/src/mongo/db/query/query_shape/query_shape_test.cpp
deleted file mode 100644
index d6185b5c5cb..00000000000
--- a/src/mongo/db/query/query_shape/query_shape_test.cpp
+++ /dev/null
@@ -1,767 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/bson/bsonmisc.h"
-#include "mongo/bson/bsonobj.h"
-#include "mongo/db/matcher/expression_geo.h"
-#include "mongo/db/matcher/extensions_callback_real.h"
-#include "mongo/db/matcher/parsed_match_expression_for_test.h"
-#include "mongo/db/namespace_string.h"
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_shape/query_shape_test_gen.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
-#include "mongo/db/query/query_shape/shape_helpers.h"
-#include "mongo/db/service_context_test_fixture.h"
-#include "mongo/unittest/bson_test_util.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo::query_shape {
-
-namespace {
-BSONObj predicateShape(const MatchExpression* expr) {
- return expr->serialize(SerializationOptions::kDebugQueryShapeSerializeOptions);
-}
-BSONObj predicateShape(std::string filterJson) {
- return predicateShape(ParsedMatchExpressionForTest(filterJson).get());
-}
-
-BSONObj predicateShapeRedacted(const MatchExpression* expr) {
- return expr->serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST);
-}
-BSONObj predicateShapeRedacted(std::string filterJson) {
- return predicateShapeRedacted(ParsedMatchExpressionForTest(filterJson).get());
-}
-
-// TODO SERVER-87736 There is no 'auto' here, make that more clear.
-#define ASSERT_SHAPE_EQ_AUTO(expected, actual) \
- ASSERT_BSONOBJ_EQ_AUTO(expected, predicateShape(actual))
-
-#define ASSERT_REDACTED_SHAPE_EQ_AUTO(expected, actual) \
- ASSERT_BSONOBJ_EQ_AUTO(expected, predicateShapeRedacted(actual))
-
-
-TEST(QueryPredicateShape, Equals) {
- ASSERT_SHAPE_EQ_AUTO( // Implicit equals
- R"({"a":{"$eq":"?number"}})",
- "{a: 5}");
- ASSERT_SHAPE_EQ_AUTO( // Explicit equals
- R"({"a":{"$eq":"?number"}})",
- "{a: {$eq: 5}}");
- ASSERT_SHAPE_EQ_AUTO( // implicit $and
- R"({"$and":[{"a":{"$eq":"?number"}},{"b":{"$eq":"?number"}}]})",
- "{a: 5, b: 6}");
- ASSERT_REDACTED_SHAPE_EQ_AUTO( // Implicit equals
- R"({"HASH<a>":{"$eq":"?number"}})",
- "{a: 5}");
- ASSERT_REDACTED_SHAPE_EQ_AUTO( // Explicit equals
- R"({"HASH<a>":{"$eq":"?number"}})",
- "{a: {$eq: 5}}");
- ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT
- R"({"$and":[{"HASH<a>":{"$eq":"?number"}},{"HASH<b>":{"$eq":"?number"}}]})",
- "{a: 5, b: 6}");
- ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT
- R"({"HASH<foo>.HASH<$bar>":{"$eq":"?number"}})",
- R"({"foo.$bar":0})");
-}
-
-TEST(QueryPredicateShape, ArraySubTypes) {
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- "{a: {$eq: '[]'}}",
- "{a: []}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- "{a: {$eq: '?array<?number>'}}",
- "{a: [2]}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$eq":"?array<?number>"}})",
- "{a: [2, 3]}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$eq":"?array<?object>"}})",
- "{a: [{}]}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$eq":"?array<?object>"}})",
- "{a: [{}, {}]}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$eq":"?array<?array>"}})",
- "{a: [[], [], []]}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$eq":"?array<?array>"}})",
- "{a: [[2, 3], ['string'], []]}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$eq":"?array<>"}})",
- "{a: [{}, 2]}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$eq":"?array<>"}})",
- "{a: [[], 2]}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$eq":"?array<>"}})",
- "{a: [[{}, 'string'], 2]}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$eq":"?array<>"}})",
- "{a: [[{}, 'string'], 2]}");
-}
-
-TEST(QueryPredicateShape, Comparisons) {
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({
- "$and": [
- {
- "a": {
- "$lt": "?number"
- }
- },
- {
- "b": {
- "$gt": "?number"
- }
- },
- {
- "c": {
- "$gte": "?number"
- }
- },
- {
- "c": {
- "$lte": "?number"
- }
- }
- ]
- })",
- "{a: {$lt: 5}, b: {$gt: 6}, c: {$gte: 3, $lte: 10}}");
-}
-
-namespace {
-void assertShapeIs(std::string filterJson, BSONObj expectedShape) {
- ASSERT_BSONOBJ_EQ(expectedShape, predicateShape(filterJson));
-}
-
-void assertRedactedShapeIs(std::string filterJson, BSONObj expectedShape) {
- ASSERT_BSONOBJ_EQ(expectedShape, predicateShapeRedacted(filterJson));
-}
-} // namespace
-
-TEST(QueryPredicateShape, Regex) {
- // Note/warning: 'fromjson' will parse $regex into a /regex/, so these tests can't use
- // auto-updating BSON assertions.
- assertShapeIs("{a: /a+/}",
- BSON("a" << BSON("$regex"
- << "?string")));
- assertShapeIs("{a: /a+/i}",
- BSON("a" << BSON("$regex"
- << "?string"
- << "$options"
- << "?string")));
- assertRedactedShapeIs("{a: /a+/}",
- BSON("HASH<a>" << BSON("$regex"
- << "?string")));
- assertRedactedShapeIs("{a: /a+/}",
- BSON("HASH<a>" << BSON("$regex"
- << "?string")));
-}
-
-TEST(QueryPredicateShape, Mod) {
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$mod":["?number","?number"]}})",
- "{a: {$mod: [2, 0]}}");
-}
-
-TEST(QueryPredicateShape, Exists) {
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$exists":"?bool"}})",
- "{a: {$exists: true}}");
-}
-
-TEST(QueryPredicateShape, In) {
- // Any number of children in any order is always the same shape
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$in":"?array<?number>"}})",
- "{a: {$in: [1]}}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$in":"?array<>"}})",
- "{a: {$in: [1, 4, 'str', /regex/]}}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$in":"?array<>"}})",
- "{a: {$in: ['str', /regex/, 1, 4]}}");
-}
-
-TEST(QueryPredicateShape, BitTestOperators) {
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$bitsAllSet":"?array<?number>"}})",
- "{a: {$bitsAllSet: [1, 5]}}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$bitsAllSet":"?array<?number>"}})",
- "{a: {$bitsAllSet: 50}}");
-
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$bitsAnySet":"?array<?number>"}})",
- "{a: {$bitsAnySet: [1, 5]}}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$bitsAnySet":"?array<?number>"}})",
- "{a: {$bitsAnySet: 50}}");
-
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$bitsAllClear":"?array<?number>"}})",
- "{a: {$bitsAllClear: [1, 5]}}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$bitsAllClear":"?array<?number>"}})",
- "{a: {$bitsAllClear: 50}}");
-
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$bitsAnyClear":"?array<?number>"}})",
- "{a: {$bitsAnyClear: [1, 5]}}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$bitsAnyClear":"?array<?number>"}})",
- "{a: {$bitsAnyClear: 50}}");
-}
-
-TEST(QueryPredicateShape, AlwaysBoolean) {
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"$alwaysTrue":"?number"})",
- "{$alwaysTrue: 1}");
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"$alwaysFalse":"?number"})",
- "{$alwaysFalse: 1}");
-}
-
-TEST(QueryPredicateShape, And) {
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({
- "$and": [
- {
- "a": {
- "$lt": "?number"
- }
- },
- {
- "b": {
- "$gte": "?number"
- }
- },
- {
- "c": {
- "$lte": "?number"
- }
- }
- ]
- })",
- "{$and: [{a: {$lt: 5}}, {b: {$gte: 3}}, {c: {$lte: 10}}]}");
-}
-
-TEST(QueryPredicateShape, Or) {
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({
- "$or": [
- {
- "a": {
- "$eq": "?number"
- }
- },
- {
- "b": {
- "$in": "?array<?number>"
- }
- },
- {
- "c": {
- "$gt": "?number"
- }
- }
- ]
- })",
- "{$or: [{a: 5}, {b: {$in: [1,2,3]}}, {c: {$gt: 10}}]}");
-}
-
-TEST(QueryPredicateShape, ElemMatch) {
- // ElemMatchObjectMatchExpression
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({
- "a": {
- "$elemMatch": {
- "$and": [
- {
- "b": {
- "$eq": "?number"
- }
- },
- {
- "c": {
- "$exists": "?bool"
- }
- }
- ]
- }
- }
- })",
- "{a: {$elemMatch: {b: 5, c: {$exists: true}}}}");
-
- // ElemMatchValueMatchExpression
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"a":{"$elemMatch":{"$gt":"?number","$lt":"?number"}}})",
- "{a: {$elemMatch: {$gt: 5, $lt: 10}}}");
-
- // Nested
- ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT
- R"({
- "HASH<a>": {
- "$elemMatch": {
- "$elemMatch": {
- "$gt": "?number",
- "$lt": "?number"
- }
- }
- }
- })",
- "{a: {$elemMatch: {$elemMatch: {$gt: 5, $lt: 10}}}}");
-}
-
-TEST(QueryPredicateShape, InternalBucketGeoWithinMatchExpression) {
- auto query =
- "{ $_internalBucketGeoWithin: {withinRegion: {$centerSphere: [[0, 0], 10]}, field: "
- "\"a\"} "
- "}";
- ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT
- R"({
- "$_internalBucketGeoWithin": {
- "withinRegion": {
- "$centerSphere": "?array<>"
- },
- "field": "HASH<a>"
- }
- })",
- query);
-}
-
-TEST(QueryPredicateShape, NorMatchExpression) {
- ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT
- R"({"$nor":[{"HASH<a>":{"$lt":"?number"}},{"HASH<b>":{"$gt":"?number"}}]})",
- "{ $nor: [ { a: {$lt: 5} }, { b: {$gt: 4} } ] }");
-}
-
-TEST(QueryPredicateShape, NotMatchExpression) {
- ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT
- R"({"HASH<price>":{"$not":{"$gt":"?number"}}})",
- "{ price: { $not: { $gt: 1.99 } } }");
- // Test the special case where NotMatchExpression::serialize() reduces to $alwaysFalse.
- auto emptyAnd = std::make_unique<AndMatchExpression>();
- const MatchExpression& notExpr = NotMatchExpression(std::move(emptyAnd));
- auto serialized =
- notExpr.serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({"$alwaysFalse":"?number"})",
- serialized);
-}
-
-TEST(QueryPredicateShape, SizeMatchExpression) {
- ASSERT_REDACTED_SHAPE_EQ_AUTO( // NOLINT
- R"({"HASH<price>":{"$size":"?number"}})",
- "{ price: { $size: 2 } }");
-}
-
-TEST(QueryPredicateShape, TextMatchExpression) {
- TextMatchExpressionBase::TextParams params = {"coffee"};
- auto expr = ExtensionsCallbackNoop().createText(params);
- auto literalAndFieldRedactOpts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST;
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "$text": {
- "$search": "?string",
- "$language": "?string",
- "$caseSensitive": "?bool",
- "$diacriticSensitive": "?bool"
- }
- })",
- expr->serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST));
-}
-
-TEST(QueryPredicateShape, TwoDPtInAnnulusExpression) {
- const MatchExpression& expr = TwoDPtInAnnulusExpression({}, {});
- auto literalAndFieldRedactOpts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST;
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({"$TwoDPtInAnnulusExpression":true})",
- expr.serialize(SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST));
-}
-
-TEST(QueryPredicateShape, WhereMatchExpression) {
- ASSERT_SHAPE_EQ_AUTO( // NOLINT
- R"({"$where":"?javascript"})",
- "{$where: \"some_code()\"}");
-}
-
-BSONObj queryShapeForOptimizedExprExpression(std::string exprPredicateJson) {
- ParsedMatchExpressionForTest expr(exprPredicateJson);
- // We need to optimize an $expr expression in order to generate an $_internalExprEq. It's
- // not clear we'd want to do optimization before computing the query shape, but we should
- // support the computation on any MatchExpression, and this is the easiest way we can create
- // this type of MatchExpression node.
- auto optimized = MatchExpression::optimize(expr.release());
- return predicateShape(optimized.get());
-}
-
-TEST(QueryPredicateShape, OptimizedExprPredicates) {
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "$and": [
- {
- "a": {
- "$_internalExprEq": "?number"
- }
- },
- {
- "$expr": {
- "$eq": [
- "$a",
- "?number"
- ]
- }
- }
- ]
- })",
- queryShapeForOptimizedExprExpression("{$expr: {$eq: ['$a', 2]}}"));
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "$and": [
- {
- "a": {
- "$_internalExprLt": "?number"
- }
- },
- {
- "$expr": {
- "$lt": [
- "$a",
- "?number"
- ]
- }
- }
- ]
- })",
- queryShapeForOptimizedExprExpression("{$expr: {$lt: ['$a', 2]}}"));
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "$and": [
- {
- "a": {
- "$_internalExprLte": "?number"
- }
- },
- {
- "$expr": {
- "$lte": [
- "$a",
- "?number"
- ]
- }
- }
- ]
- })",
- queryShapeForOptimizedExprExpression("{$expr: {$lte: ['$a', 2]}}"));
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "$and": [
- {
- "a": {
- "$_internalExprGt": "?number"
- }
- },
- {
- "$expr": {
- "$gt": [
- "$a",
- "?number"
- ]
- }
- }
- ]
- })",
- queryShapeForOptimizedExprExpression("{$expr: {$gt: ['$a', 2]}}"));
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "$and": [
- {
- "a": {
- "$_internalExprGte": "?number"
- }
- },
- {
- "$expr": {
- "$gte": [
- "$a",
- "?number"
- ]
- }
- }
- ]
- })",
- queryShapeForOptimizedExprExpression("{$expr: {$gte: ['$a', 2]}}"));
-}
-
-TEST(QueryShapeIDL, ShapifyIDLStruct) {
- SerializationOptions options;
- options.transformIdentifiers = true;
- options.transformIdentifiersCallback = [](StringData s) -> std::string {
- return str::stream() << "HASH<" << s << ">";
- };
- options.literalPolicy = LiteralSerializationPolicy::kToDebugTypeString;
-
- auto nested = NestedStruct("value",
- ExampleEnumEnum::Value1,
- "hello",
- {1, 2, 3, 4},
- "field.path",
- {"field.path.1", "fieldpath2"},
- NamespaceString{"db", "coll"},
- NamespaceString{"db", "coll"},
- 177,
- true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "stringField": "value",
- "enumField": "EnumValue1",
- "stringIntVariantEnum": "hello",
- "arrayOfInts": [
- 1,
- 2,
- 3,
- 4
- ],
- "fieldpath": "field.path",
- "fieldpathList": [
- "field.path.1",
- "fieldpath2"
- ],
- "nss": "db.coll",
- "plainNss": "db.coll",
- "safeInt64Field": 177,
- "boolField": true
- })",
- nested.toBSON());
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "stringField": "?string",
- "enumField": "EnumValue1",
- "stringIntVariantEnum": "hello",
- "arrayOfInts": "?array<?number>",
- "fieldpath": "HASH<field>.HASH<path>",
- "fieldpathList": [
- "HASH<field>.HASH<path>.HASH<1>",
- "HASH<fieldpath2>"
- ],
- "nss": "HASH<db.coll>",
- "plainNss": "db.coll",
- "safeInt64Field": "?number",
- "boolField": "?bool"
- })",
- nested.toBSON(options));
-
-
- auto parent = ParentStruct(nested, nested);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "nested_shape": {
- "stringField": "value",
- "enumField": "EnumValue1",
- "stringIntVariantEnum": "hello",
- "arrayOfInts": [
- 1,
- 2,
- 3,
- 4
- ],
- "fieldpath": "field.path",
- "fieldpathList": [
- "field.path.1",
- "fieldpath2"
- ],
- "nss": "db.coll",
- "plainNss": "db.coll",
- "safeInt64Field": 177,
- "boolField": true
- },
- "nested_no_shape": {
- "stringField": "value",
- "enumField": "EnumValue1",
- "stringIntVariantEnum": "hello",
- "arrayOfInts": [
- 1,
- 2,
- 3,
- 4
- ],
- "fieldpath": "field.path",
- "fieldpathList": [
- "field.path.1",
- "fieldpath2"
- ],
- "nss": "db.coll",
- "plainNss": "db.coll",
- "safeInt64Field": 177,
- "boolField": true
- }
- })",
- parent.toBSON());
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "nested_shape": {
- "stringField": "?string",
- "enumField": "EnumValue1",
- "stringIntVariantEnum": "hello",
- "arrayOfInts": "?array<?number>",
- "fieldpath": "HASH<field>.HASH<path>",
- "fieldpathList": [
- "HASH<field>.HASH<path>.HASH<1>",
- "HASH<fieldpath2>"
- ],
- "nss": "HASH<db.coll>",
- "plainNss": "db.coll",
- "safeInt64Field": "?number",
- "boolField": "?bool"
- },
- "nested_no_shape": {
- "stringField": "value",
- "enumField": "EnumValue1",
- "stringIntVariantEnum": "hello",
- "arrayOfInts": [
- 1,
- 2,
- 3,
- 4
- ],
- "fieldpath": "field.path",
- "fieldpathList": [
- "field.path.1",
- "fieldpath2"
- ],
- "nss": "db.coll",
- "plainNss": "db.coll",
- "safeInt64Field": 177,
- "boolField": true
- }
- })",
- parent.toBSON(options));
-}
-
-} // namespace
-
-namespace {
-
-static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl");
-
-struct DummyShapeSpecificComponents : public query_shape::CmdSpecificShapeComponents {
- DummyShapeSpecificComponents(){};
- void HashValue(absl::HashState state) const {}
- size_t size() const final {
- return sizeof(DummyShapeSpecificComponents);
- }
-};
-
-class DummyShape : public Shape {
-public:
- DummyShape(NamespaceStringOrUUID nssOrUUID,
- BSONObj collation,
- DummyShapeSpecificComponents dummyComponents)
- : Shape(nssOrUUID, collation) {
- components = dummyComponents;
- }
-
- const CmdSpecificShapeComponents& specificComponents() const final {
- return components;
- }
-
- void appendCmdSpecificShapeComponents(BSONObjBuilder&,
- OperationContext*,
- const SerializationOptions& opts) const final {}
- DummyShapeSpecificComponents components;
-};
-
-class DummyShapeWithExtraSize : public Shape {
-public:
- DummyShapeWithExtraSize(NamespaceStringOrUUID nssOrUUID,
- BSONObj collation,
- DummyShapeSpecificComponents dummyComponents)
- : Shape(nssOrUUID, collation) {
- components = dummyComponents;
- }
-
- const CmdSpecificShapeComponents& specificComponents() const final {
- return components;
- }
-
- // Random number for testing purposes.
- size_t extraSize() const final override {
- return 125;
- }
- void appendCmdSpecificShapeComponents(BSONObjBuilder&,
- OperationContext*,
- const SerializationOptions& opts) const final {}
-
- DummyShapeSpecificComponents components;
-};
-
-class UniversalShapeTest : public ServiceContextTest {};
-
-TEST_F(UniversalShapeTest, SizeOfSpecificComponents) {
- auto innerComponents = std::make_unique<DummyShapeSpecificComponents>();
- ASSERT_EQ(innerComponents->size(), sizeof(CmdSpecificShapeComponents));
- ASSERT_EQ(innerComponents->size(), sizeof(void*) /*vtable ptr*/);
-}
-
-TEST_F(UniversalShapeTest, SizeOfShape) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
-
- // Make shape for testing.
- auto collation = BSONObj{};
- auto innerComponents = std::make_unique<DummyShapeSpecificComponents>();
- auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents);
-
- ASSERT_EQ(innerComponents->size(), shape->specificComponents().size());
- ASSERT_EQ(shape->size(),
- sizeof(NamespaceStringOrUUID) + sizeof(BSONObj) + sizeof(void*) /*vtable ptr*/ +
- shape->specificComponents().size() + static_cast<size_t>(collation.objsize()));
-}
-
-TEST_F(UniversalShapeTest, SizeOfShapeWithExtraSize) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
-
- // Make shape for testing.
- auto collation = BSONObj{};
- auto innerComponents = std::make_unique<DummyShapeSpecificComponents>();
- auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents);
- auto shapeWithExtraSize =
- std::make_unique<DummyShapeWithExtraSize>(kDefaultTestNss, collation, *innerComponents);
-
- ASSERT_EQ(shapeWithExtraSize->size(), shape->size() + shapeWithExtraSize->extraSize());
-}
-} // namespace
-} // namespace mongo::query_shape
diff --git a/src/mongo/db/query/query_shape/query_shape_test.idl b/src/mongo/db/query/query_shape/query_shape_test.idl
deleted file mode 100644
index 06efb7ed1ef..00000000000
--- a/src/mongo/db/query/query_shape/query_shape_test.idl
+++ /dev/null
@@ -1,91 +0,0 @@
-# Copyright (C) 2023-present MongoDB, Inc.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the Server Side Public License, version 1,
-# as published by MongoDB, Inc.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Server Side Public License for more details.
-#
-# You should have received a copy of the Server Side Public License
-# along with this program. If not, see
-# <http://www.mongodb.com/licensing/server-side-public-license>.
-#
-# As a special exception, the copyright holders give permission to link the
-# code of portions of this program with the OpenSSL library under certain
-# conditions as described in each individual source file and distribute
-# linked combinations including the program with the OpenSSL library. You
-# must comply with the Server Side Public License in all respects for
-# all of the code used other than as permitted herein. If you modify file(s)
-# with this exception, you may extend this exception to your version of the
-# file(s), but you are not obligated to do so. If you do not wish to do so,
-# delete this exception statement from your version. If you delete this
-# exception statement from all source files in the program, then also delete
-# it in the license file.
-#
-
-global:
- cpp_namespace: "mongo"
-
-imports:
- - "mongo/idl/basic_types.idl"
-
-enums:
- ExampleEnum:
- description: ""
- type: string
- values:
- Value1: "EnumValue1"
- Value2: "EnumValue2"
-
-structs:
- NestedStruct:
- query_shape_component: true
- strict: true
- description: ""
- fields:
- stringField:
- query_shape: literal
- type: string
- enumField:
- query_shape: parameter
- type: ExampleEnum
- stringIntVariantEnum:
- query_shape: parameter
- type:
- variant: [string, int]
- arrayOfInts:
- query_shape: literal
- type: array<int>
- fieldpath:
- query_shape: anonymize
- type: string
- fieldpathList:
- query_shape: anonymize
- type: array<string>
- nss:
- query_shape: custom
- type: namespacestring
- plainNss:
- query_shape: parameter
- type: namespacestring
- safeInt64Field:
- query_shape: literal
- type: safeInt64
- boolField:
- query_shape: literal
- type: bool
-
- ParentStruct:
- query_shape_component: true
- strict: true
- description: ""
- fields:
- nested_shape:
- query_shape: literal
- type: NestedStruct
- nested_no_shape:
- query_shape: parameter
- type: NestedStruct
diff --git a/src/mongo/db/query/query_shape/serialization_options.cpp b/src/mongo/db/query/query_shape/serialization_options.cpp
deleted file mode 100644
index e6008f8579b..00000000000
--- a/src/mongo/db/query/query_shape/serialization_options.cpp
+++ /dev/null
@@ -1,515 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "serialization_options.h"
-#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
-
-#include "mongo/db/query/query_shape/serialization_options.h"
-
-#include <boost/optional.hpp>
-#include <string>
-
-#include "mongo/base/string_data.h"
-#include "mongo/bson/timestamp.h"
-#include "mongo/db/exec/document_value/document.h"
-#include "mongo/db/exec/document_value/value.h"
-#include "mongo/db/query/explain_options.h"
-#include "mongo/logv2/log.h"
-#include "mongo/util/assert_util.h"
-
-namespace mongo {
-
-namespace {
-
-// We'll pre-declare all of these strings so that we can avoid the allocations when we reference
-// them later.
-static constexpr StringData kUndefinedTypeString = "?undefined"_sd;
-static constexpr StringData kStringTypeString = "?string"_sd;
-static constexpr StringData kNumberTypeString = "?number"_sd;
-static constexpr StringData kMinKeyTypeString = "?minKey"_sd;
-static constexpr StringData kObjectTypeString = "?object"_sd;
-static constexpr StringData kArrayTypeString = "?array"_sd;
-static constexpr StringData kBinDataTypeString = "?binData"_sd;
-static constexpr StringData kObjectIdTypeString = "?objectId"_sd;
-static constexpr StringData kBoolTypeString = "?bool"_sd;
-static constexpr StringData kDateTypeString = "?date"_sd;
-static constexpr StringData kNullTypeString = "?null"_sd;
-static constexpr StringData kRegexTypeString = "?regex"_sd;
-static constexpr StringData kDbPointerTypeString = "?dbPointer"_sd;
-static constexpr StringData kJavascriptTypeString = "?javascript"_sd;
-static constexpr StringData kJavascriptWithScopeTypeString = "?javascriptWithScope"_sd;
-static constexpr StringData kTimestampTypeString = "?timestamp"_sd;
-static constexpr StringData kMaxKeyTypeString = "?maxKey"_sd;
-
-static const StringMap<StringData> kArrayTypeStringConstants{
- {kUndefinedTypeString.rawData(), "?array<?undefined>"_sd},
- {kStringTypeString.rawData(), "?array<?string>"_sd},
- {kNumberTypeString.rawData(), "?array<?number>"_sd},
- {kMinKeyTypeString.rawData(), "?array<?minKey>"_sd},
- {kObjectTypeString.rawData(), "?array<?object>"_sd},
- {kArrayTypeString.rawData(), "?array<?array>"_sd},
- {kBinDataTypeString.rawData(), "?array<?binData>"_sd},
- {kObjectIdTypeString.rawData(), "?array<?objectId>"_sd},
- {kBoolTypeString.rawData(), "?array<?bool>"_sd},
- {kDateTypeString.rawData(), "?array<?date>"_sd},
- {kNullTypeString.rawData(), "?array<?null>"_sd},
- {kRegexTypeString.rawData(), "?array<?regex>"_sd},
- {kDbPointerTypeString.rawData(), "?array<?dbPointer>"_sd},
- {kJavascriptTypeString.rawData(), "?array<?javascript>"_sd},
- {kJavascriptWithScopeTypeString.rawData(), "?array<?javascriptWithScope>"_sd},
- {kTimestampTypeString.rawData(), "?array<?timestamp>"_sd},
- {kMaxKeyTypeString.rawData(), "?array<?maxKey>"_sd},
-};
-
-static constexpr auto kRepresentativeString = "?"_sd;
-static constexpr auto kRepresentativeNumber = 1;
-static const auto kRepresentativeObject = BSON("?"
- << "?");
-static const auto kRepresentativeArray = BSONArray();
-static constexpr auto kRepresentativeBinData = BSONBinData();
-static const auto kRepresentativeObjectId = OID::max();
-static constexpr auto kRepresentativeBool = true;
-static const auto kRepresentativeDate = Date_t::fromMillisSinceEpoch(0);
-static const auto kRepresentativeRegex = BSONRegEx("/\?/");
-static const auto kRepresentativeDbPointer = BSONDBRef("?.?", OID::max());
-static const auto kRepresentativeJavascript = BSONCode("return ?;");
-static const auto kRepresentativeJavascriptWithScope = BSONCodeWScope("return ?;", BSONObj());
-static const auto kRepresentativeTimestamp = Timestamp::min();
-
-/**
- * A default redaction strategy that generates easy to check results for testing purposes.
- */
-std::string applyHmacForTest(StringData s) {
- // Avoid ending in a parenthesis since the results will occur in a raw string where the )"
- // sequence will accidentally terminate the string.
- return str::stream() << "HASH<" << s << ">";
-}
-
-/**
- * Computes a debug string meant to represent "any value of type t", where "t" is the type of the
- * provided argument. For example "?number" for any number (int, double, etc.).
- */
-StringData debugTypeString(BSONType t) {
- // This is tightly coupled with 'canonicalizeBSONType' and therefore also with
- // sorting/comparison semantics.
- switch (t) {
- case EOO:
- case Undefined:
- return kUndefinedTypeString;
- case Symbol:
- case String:
- return kStringTypeString;
- case NumberInt:
- case NumberLong:
- case NumberDouble:
- case NumberDecimal:
- return kNumberTypeString;
- case MinKey:
- return kMinKeyTypeString;
- case Object:
- return kObjectTypeString;
- case Array:
- // This case should only happen if we have an array within an array.
- return kArrayTypeString;
- case BinData:
- return kBinDataTypeString;
- case jstOID:
- return kObjectIdTypeString;
- case Bool:
- return kBoolTypeString;
- case Date:
- return kDateTypeString;
- case jstNULL:
- return kNullTypeString;
- case RegEx:
- return kRegexTypeString;
- case DBRef:
- return kDbPointerTypeString;
- case Code:
- return kJavascriptTypeString;
- case CodeWScope:
- return kJavascriptWithScopeTypeString;
- case bsonTimestamp:
- return kTimestampTypeString;
- case MaxKey:
- return kMaxKeyTypeString;
- default:
- MONGO_UNREACHABLE_TASSERT(7539806);
- }
-}
-
-/**
- * Returns an arbitrary value of the same type as the one given. For any number, this will be the
- * number 1. For any boolean this will be true.
- * TODO if you need a different value to make sure it will parse, you should not use this API.
- */
-ImplicitValue defaultLiteralOfType(BSONType t) {
- // This is tightly coupled with 'canonicalizeBSONType' and therefore also with
- // sorting/comparison semantics.
- switch (t) {
- case EOO:
- case Undefined:
- return BSONUndefined;
- case Symbol:
- case String:
- return kRepresentativeString;
- case NumberInt:
- case NumberLong:
- case NumberDouble:
- case NumberDecimal:
- return kRepresentativeNumber;
- case MinKey:
- return MINKEY;
- case Object:
- return kRepresentativeObject;
- case Array:
- // This case should only happen if we have an array within an array.
- return kRepresentativeArray;
- case BinData:
- return kRepresentativeBinData;
- case jstOID:
- return kRepresentativeObjectId;
- case Bool:
- return kRepresentativeBool;
- case Date:
- return kRepresentativeDate;
- case jstNULL:
- return BSONNULL;
- case RegEx:
- return kRepresentativeRegex;
- case DBRef:
- return kRepresentativeDbPointer;
- case Code:
- return kRepresentativeJavascript;
- case CodeWScope:
- return kRepresentativeJavascriptWithScope;
- case bsonTimestamp:
- return kRepresentativeTimestamp;
- case MaxKey:
- return MAXKEY;
- default:
- MONGO_UNREACHABLE_TASSERT(7539803);
- }
-}
-
-/**
- * A struct representing the sub-type information for an array.
- */
-struct ArraySubtypeInfo {
- /**
- * Whether the values of an array are all the same BSON type or not (mixed).
- */
- enum class NTypes { kEmpty, kOneType, kMixed };
- ArraySubtypeInfo(NTypes nTypes_) : nTypes(nTypes_) {}
- ArraySubtypeInfo(BSONType oneType) : nTypes(NTypes::kOneType), singleType(oneType) {}
-
- NTypes nTypes;
- boost::optional<BSONType> singleType = boost::none;
-};
-
-template <typename ValueType>
-using GetTypeFn = std::function<BSONType(ValueType)>;
-
-static GetTypeFn<BSONElement> getBSONElementType = [](const BSONElement& e) { return e.type(); };
-static GetTypeFn<Value> getValueType = [](const Value& v) { return v.getType(); };
-
-/**
- * Scans 'arrayOfValues' to see if all values are of the same type or not. Returns this info in a
- * struct - see the struct definition for how it is represented.
- *
- * Templated algorithm to handle both iterators of BSONElements or iterators of Values.
- * 'getTypeCallback' is provided to abstract away the different '.type()' vs '.getType()' APIs.
- */
-template <typename ArrayType, typename ValueType>
-ArraySubtypeInfo determineArraySubType(const ArrayType& arrayOfValues,
- GetTypeFn<ValueType> getTypeCallback) {
- boost::optional<BSONType> firstType = boost::none;
- for (auto&& v : arrayOfValues) {
- if (!firstType) {
- firstType.emplace(getTypeCallback(v));
- } else if (*firstType != getTypeCallback(v)) {
- return {ArraySubtypeInfo::NTypes::kMixed};
- }
- }
- return firstType ? ArraySubtypeInfo{*firstType}
- : ArraySubtypeInfo{ArraySubtypeInfo::NTypes::kEmpty};
-}
-
-ArraySubtypeInfo determineArraySubType(const BSONObj& arrayAsObj) {
- return determineArraySubType<BSONObj, BSONElement>(arrayAsObj, getBSONElementType);
-}
-ArraySubtypeInfo determineArraySubType(const std::vector<Value>& values) {
- return determineArraySubType<std::vector<Value>, Value>(values, getValueType);
-}
-
-template <typename ValueType>
-StringData debugTypeString(
- const ValueType& v,
- GetTypeFn<ValueType> getTypeCallback,
- std::function<ArraySubtypeInfo(ValueType)> determineArraySubTypeCallback) {
- if (getTypeCallback(v) == BSONType::Array) {
- // Iterating the array as .Obj(), as if it were a BSONObj (with field names '0', '1', etc.)
- // is faster than converting the whole thing to an array which would force a copy.
- auto typeInfo = determineArraySubTypeCallback(v);
- switch (typeInfo.nTypes) {
- case ArraySubtypeInfo::NTypes::kEmpty:
- return "[]"_sd;
- case ArraySubtypeInfo::NTypes::kOneType:
- return kArrayTypeStringConstants.at(debugTypeString(*typeInfo.singleType));
- case ArraySubtypeInfo::NTypes::kMixed:
- return "?array<>";
- default:
- MONGO_UNREACHABLE_TASSERT(7539801);
- }
- }
- return debugTypeString(getTypeCallback(v));
-}
-
-template <typename ValueType>
-ImplicitValue defaultLiteralOfType(
- const ValueType& v,
- GetTypeFn<ValueType> getTypeCallback,
- std::function<ArraySubtypeInfo(ValueType)> determineArraySubTypeCallback) {
- if (getTypeCallback(v) == BSONType::Array) {
- auto typeInfo = determineArraySubTypeCallback(v);
- switch (typeInfo.nTypes) {
- case ArraySubtypeInfo::NTypes::kEmpty:
- return BSONArray();
- case ArraySubtypeInfo::NTypes::kOneType:
- return std::vector<Value>{defaultLiteralOfType(*typeInfo.singleType)};
- case ArraySubtypeInfo::NTypes::kMixed:
- // We don't care which types, we'll use a number and a string as the canonical
- // mixed type array regardless. This is to ensure we don't get 2^N possibilities
- // for mixed type scenarios - we wish to collapse all "mixed type" arrays to one
- // canonical mix. The choice of int and string is mostly arbitrary - hopefully
- // somewhat comprehensible at a glance.
- return std::vector<Value>{Value(2), Value("or more types"_sd)};
- default:
- MONGO_UNREACHABLE_TASSERT(7539805);
- }
- }
- return defaultLiteralOfType(getTypeCallback(v));
-}
-
-ArraySubtypeInfo getSubTypeFromBSONElemArray(BSONElement arrayElem) {
- // Iterating the array as .Obj(), as if it were a BSONObj (with field names '0', '1', etc.)
- // is faster than converting the whole thing to an array which would force a copy.
- return determineArraySubType(arrayElem.Obj());
-}
-ArraySubtypeInfo getSubTypeFromValueArray(const Value& arrayVal) {
- return determineArraySubType(arrayVal.getArray());
-}
-
-void appendDefaultOfNonArrayType(BSONObjBuilder* bob, StringData name, const BSONElement& e) {
- switch (e.type()) {
- case EOO:
- case Undefined:
- bob->appendUndefined(name);
- return;
- case Symbol:
- case String:
- bob->append(name, kRepresentativeString);
- return;
- case NumberInt:
- case NumberLong:
- case NumberDouble:
- case NumberDecimal:
- bob->append(name, kRepresentativeNumber);
- return;
- case MinKey:
- bob->appendMinKey(name);
- return;
- case Object:
- bob->append(name, kRepresentativeObject);
- return;
- case Array:
- // This case is more complicated and callers should use a more generic helper.
- MONGO_UNREACHABLE_TASSERT(8094100);
- case BinData:
- bob->append(name, kRepresentativeBinData);
- return;
- case jstOID:
- bob->append(name, kRepresentativeObjectId);
- return;
- case Bool:
- bob->append(name, kRepresentativeBool);
- return;
- case Date:
- bob->append(name, kRepresentativeDate);
- return;
- case jstNULL:
- bob->appendNull(name);
- return;
- case RegEx:
- bob->append(name, kRepresentativeRegex);
- return;
- case DBRef:
- bob->append(name, kRepresentativeDbPointer);
- return;
- case Code:
- bob->append(name, kRepresentativeJavascript);
- return;
- case CodeWScope:
- bob->append(name, kRepresentativeJavascriptWithScope);
- return;
- case bsonTimestamp:
- bob->append(name, kRepresentativeTimestamp);
- return;
- case MaxKey:
- bob->appendMaxKey(name);
- return;
- default:
- MONGO_UNREACHABLE_TASSERT(8094101);
- };
-}
-} // namespace
-
-const SerializationOptions SerializationOptions::kRepresentativeQueryShapeSerializeOptions =
- SerializationOptions{LiteralSerializationPolicy::kToRepresentativeParseableValue};
-
-const SerializationOptions SerializationOptions::kDebugQueryShapeSerializeOptions =
- SerializationOptions{LiteralSerializationPolicy::kToDebugTypeString};
-
-SerializationOptions::SerializationOptions(LiteralSerializationPolicy policy)
- : literalPolicy(policy) {}
-SerializationOptions::SerializationOptions(
- boost::optional<ExplainOptions::Verbosity> explainVerbosity)
- : verbosity(explainVerbosity) {}
-
-SerializationOptions::SerializationOptions(LiteralSerializationPolicy policy,
- bool transformIdentifiers,
- TokenizeIdentifierFunc transformIdentifiersCallbackFn)
- : literalPolicy(policy),
- transformIdentifiers(transformIdentifiers),
- transformIdentifiersCallback(transformIdentifiersCallbackFn) {}
-
-const SerializationOptions SerializationOptions::kMarkIdentifiers_FOR_TEST{
- LiteralSerializationPolicy::kUnchanged, true, applyHmacForTest};
-
-const SerializationOptions SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST{
- LiteralSerializationPolicy::kToDebugTypeString, true, applyHmacForTest};
-
-// Overloads for BSONElem and Value.
-StringData debugTypeString(BSONElement e) {
- return debugTypeString<BSONElement>(e, getBSONElementType, getSubTypeFromBSONElemArray);
-}
-StringData debugTypeString(const Value& v) {
- return debugTypeString<Value>(v, getValueType, getSubTypeFromValueArray);
-}
-
-// Overloads for BSONElem and Value.
-ImplicitValue defaultLiteralOfType(const Value& v) {
- return defaultLiteralOfType<Value>(v, getValueType, getSubTypeFromValueArray);
-}
-ImplicitValue defaultLiteralOfType(BSONElement e) {
- return defaultLiteralOfType<BSONElement>(e, getBSONElementType, getSubTypeFromBSONElemArray);
-}
-
-void SerializationOptions::appendLiteral(BSONObjBuilder* bob, const BSONElement& e) const {
- appendLiteral(bob, e.fieldNameStringData(), e);
-}
-void SerializationOptions::appendLiteral(BSONObjBuilder* bob,
- StringData name,
- const BSONElement& e) const {
- // The first two cases are particularly performance sensitive. We could answer everything here
- // with the code inside the 'kToDebugTypeString' branch, but there are some relatively easy ways
- // to accomplish the first two policy cases (in the common cases), so we'll special case those
- // in order to avoid constructing a temporary Value.
- switch (literalPolicy) {
- case LiteralSerializationPolicy::kUnchanged:
- bob->appendAs(e, name);
- return;
- case LiteralSerializationPolicy::kToRepresentativeParseableValue: {
- if (e.type() != BSONType::Array) {
- appendDefaultOfNonArrayType(bob, name, e);
- return;
- }
- // If it's an array we'll default to the slow but general codepath below.
- [[fallthrough]];
- }
- case LiteralSerializationPolicy::kToDebugTypeString: {
- // Performance isn't as sensitive here.
- return serializeLiteral(e).addToBsonObj(bob, name);
- }
- default:
- MONGO_UNREACHABLE_TASSERT(8094102);
- }
-}
-
-void SerializationOptions::appendLiteral(BSONObjBuilder* bob,
- StringData fieldName,
- const ImplicitValue& v,
- const boost::optional<Value>& representativeValue) const {
- serializeLiteral(v, representativeValue).addToBsonObj(bob, fieldName);
-}
-
-Value SerializationOptions::serializeLiteral(
- const BSONElement& e, const boost::optional<Value>& representativeValue) const {
- switch (literalPolicy) {
- case LiteralSerializationPolicy::kUnchanged:
- return Value(e);
- case LiteralSerializationPolicy::kToDebugTypeString:
- return Value(debugTypeString(e));
- case LiteralSerializationPolicy::kToRepresentativeParseableValue:
- return representativeValue.value_or(defaultLiteralOfType(e));
- default:
- MONGO_UNREACHABLE_TASSERT(7539802);
- }
-}
-
-Value SerializationOptions::serializeLiteral(
- const ImplicitValue& v, const boost::optional<Value>& representativeValue) const {
- switch (literalPolicy) {
- case LiteralSerializationPolicy::kUnchanged:
- return v;
- case LiteralSerializationPolicy::kToDebugTypeString:
- return Value(debugTypeString(v));
- case LiteralSerializationPolicy::kToRepresentativeParseableValue:
- return representativeValue.value_or(defaultLiteralOfType(v));
- default:
- MONGO_UNREACHABLE_TASSERT(7539804);
- }
-}
-
-std::string SerializationOptions::serializeFieldPathFromString(StringData path) const {
- if (transformIdentifiers) {
- try {
- return serializeFieldPath(FieldPath(path, false));
- } catch (DBException& ex) {
- LOGV2_DEBUG(7549808,
- 1,
- "Failed to convert a path string to a FieldPath",
- "pathString"_attr = path,
- "failure"_attr = ex.toStatus());
- return serializeFieldPath("invalidFieldPathPlaceholder");
- }
- }
- return path.toString();
-}
-} // namespace mongo
diff --git a/src/mongo/db/query/query_shape/serialization_options.h b/src/mongo/db/query/query_shape/serialization_options.h
deleted file mode 100644
index 226da7689d3..00000000000
--- a/src/mongo/db/query/query_shape/serialization_options.h
+++ /dev/null
@@ -1,236 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-#include "mongo/base/string_data.h"
-#include "mongo/bson/bsonobj.h"
-#include "mongo/bson/bsonobjbuilder.h"
-#include "mongo/db/exec/document_value/document.h"
-#include "mongo/db/exec/document_value/value.h"
-#include "mongo/db/pipeline/field_path.h"
-#include "mongo/db/query/explain_options.h"
-#include "mongo/util/assert_util.h"
-#include <boost/optional.hpp>
-#include <string>
-
-namespace mongo {
-namespace {
-// Should never be called, throw to ensure we catch this in tests.
-std::string defaultHmacStrategy(StringData s) {
- MONGO_UNREACHABLE_TASSERT(7332410);
-}
-} // namespace
-
-/**
- * A policy enum for how to serialize literal values.
- */
-enum class LiteralSerializationPolicy {
- // The default way to serialize. Just serialize whatever literals were given if they are still
- // available, or whatever you parsed them to. This is expected to be able to parse again, since
- // it worked the first time.
- kUnchanged,
- // Serialize any literal value as "?number" or similar. For example "?bool" for any boolean. Use
- // 'debugTypeString()' helper.
- kToDebugTypeString,
- // Serialize any literal value to one canonical value of the given type, with the constraint
- // that the chosen representative value should be parseable in this context. There are some
- // default implementations that will usually work (e.g. using the number 1 almost always works
- // for numbers), but serializers should be careful to think about and test this if their parsers
- // reject certain values.
- kToRepresentativeParseableValue,
-};
-
-/**
- * A struct with options for how you want to serialize a match or aggregation expression.
- */
-struct SerializationOptions {
- using TokenizeIdentifierFunc = std::function<std::string(StringData)>;
-
- // The default serialization options for a query shape. No need to redact identifiers for the
- // this purpose. We may do that on the $queryStats read path.
- static const SerializationOptions kRepresentativeQueryShapeSerializeOptions;
- static const SerializationOptions kDebugQueryShapeSerializeOptions;
- static const SerializationOptions kMarkIdentifiers_FOR_TEST;
- static const SerializationOptions kDebugShapeAndMarkIdentifiers_FOR_TEST;
-
- SerializationOptions() = default;
- SerializationOptions(LiteralSerializationPolicy policy);
- SerializationOptions(boost::optional<ExplainOptions::Verbosity> explain);
- SerializationOptions(LiteralSerializationPolicy policy,
- bool transformIdentifiers,
- TokenizeIdentifierFunc transformIdentifiersCallbackFn);
-
- /**
- * Checks if this SerializationOptions represents the same options as another
- * SerializationOptions. Note it cannot compare whether the two 'transformIdentifiersCallback's
- * are the same - the language purposefully leaves the comparison operator undefined.
- */
- bool operator==(const SerializationOptions& other) const {
- return this->transformIdentifiers == other.transformIdentifiers &&
- // You cannot well determine std::function equivalence in C++, so this is the best we'll
- // do.
- (this->transformIdentifiersCallback == nullptr) ==
- (other.transformIdentifiersCallback == nullptr) &&
- this->literalPolicy == other.literalPolicy && this->verbosity == other.verbosity;
- }
- bool operator!=(const SerializationOptions& other) const {
- return !(*this == other);
- }
-
- // Helper function for removing identifiable information (like collection/db names).
- // Note: serializeFieldPath/serializeFieldPathFromString should be used for field
- // names.
- std::string serializeIdentifier(StringData str) const {
- if (transformIdentifiers) {
- return transformIdentifiersCallback(str);
- }
- return str.toString();
- }
-
- std::string serializeFieldPath(FieldPath path) const {
- if (transformIdentifiers) {
- std::stringstream hmaced;
- for (size_t i = 0; i < path.getPathLength(); ++i) {
- if (i > 0) {
- hmaced << ".";
- }
- hmaced << transformIdentifiersCallback(path.getFieldName(i));
- }
- return hmaced.str();
- }
- return path.fullPath();
- }
-
- std::string serializeFieldPathWithPrefix(FieldPath path) const {
- return "$" + serializeFieldPath(path);
- }
-
- std::string serializeFieldPathFromString(StringData path) const;
-
- std::vector<std::string> serializeFieldPathFromString(
- const std::vector<std::string>& paths) const {
- std::vector<std::string> result;
- result.reserve(paths.size());
- for (auto& p : paths) {
- result.push_back(serializeFieldPathFromString(p));
- }
- return result;
- }
-
- // Helper functions for applying hmac to BSONObj. Does not take into account anything to do with
- // MQL semantics, removes all field names and literals in the passed in obj.
- void addHmacedArrayToBuilder(BSONArrayBuilder* bab, std::vector<BSONElement> array) const {
- for (const auto& elem : array) {
- if (elem.type() == BSONType::Object) {
- BSONObjBuilder subObj(bab->subobjStart());
- addHmacedObjToBuilder(&subObj, elem.Obj());
- subObj.done();
- } else if (elem.type() == BSONType::Array) {
- BSONArrayBuilder subArr(bab->subarrayStart());
- addHmacedArrayToBuilder(&subArr, elem.Array());
- subArr.done();
- } else {
- *bab << serializeLiteral(elem);
- }
- }
- }
-
- void addHmacedObjToBuilder(BSONObjBuilder* bob, BSONObj objToHmac) const {
- for (const auto& elem : objToHmac) {
- auto fieldName = serializeFieldPath(elem.fieldName());
- if (elem.type() == BSONType::Object) {
- BSONObjBuilder subObj(bob->subobjStart(fieldName));
- addHmacedObjToBuilder(&subObj, elem.Obj());
- subObj.done();
- } else if (elem.type() == BSONType::Array) {
- BSONArrayBuilder subArr(bob->subarrayStart(fieldName));
- addHmacedArrayToBuilder(&subArr, elem.Array());
- subArr.done();
- } else {
- appendLiteral(bob, fieldName, elem);
- }
- }
- }
-
- /**
- * Helper method to call 'serializeLiteral()' on 'e' and append the resulting value to 'bob'
- * using the same name as 'e'.
- */
- void appendLiteral(BSONObjBuilder* bob, const BSONElement& e) const;
- void appendLiteral(BSONObjBuilder* bob, StringData name, const BSONElement& e) const;
- /**
- * Helper method to call 'serializeLiteral()' on 'v' and append the result to 'bob' using field
- * name 'fieldName'.
- */
- void appendLiteral(BSONObjBuilder* bob,
- StringData fieldName,
- const ImplicitValue& v,
- const boost::optional<Value>& representativeValue = boost::none) const;
-
- /**
- * Depending on the configured 'literalPolicy', serializeLiteral will return the appropriate
- * value for adding literals to serialization output:
- * - If 'literalPolicy' is 'kUnchanged', returns the input value unmodified.
- * - If it is 'kToDebugTypeString', computes and returns the type string as a string Value.
- * - If it is 'kToRepresentativeValue', it returns an arbitrary value of the same type as the
- * one given. For any number, this will be the number 1. For any boolean this will be true.
- * If the 'representativeValue' parameter if it is not none, returns it (regardless of type).
- *
- * Example usage: BSON("myArg" << options.serializeLiteral(_myArg));
- */
- Value serializeLiteral(const BSONElement& e,
- const boost::optional<Value>& representativeValue = boost::none) const;
- Value serializeLiteral(const ImplicitValue& v,
- const boost::optional<Value>& representativeValue = boost::none) const;
-
- // 'literalPolicy' is an independent option to serialize in a general format with the aim of
- // similar "shaped" queries serializing to the same object. For example, if set to
- // 'kToDebugTypeString', then the serialization of {a: {$gt: 2}} should result in {a: {$gt:
- // '?number'}}, as will the serialization of {a: {$gt: 3}}.
- //
- // "Literal" here is meant to stand in contrast to expression arguments, as in the $gt
- // expressions in {$and: [{a: {$gt: 3}}, {b: {$gt: 4}}]}. There the only literals are 3 and 4,
- // so the serialization expected for 'kToDebugTypeString' would be {$and: [{a: {$gt:
- // '?number'}}, {b: {$lt: '?number'}}]}.
- LiteralSerializationPolicy literalPolicy = LiteralSerializationPolicy::kUnchanged;
-
- // If true the caller must set transformIdentifiersCallback. 'transformIdentifiers' if set along
- // with a strategy the redaction strategy will be called on any personal identifiable
- // information (e.g., field paths/names, collection names) encountered before serializing them.
- bool transformIdentifiers = false;
- std::function<std::string(StringData)> transformIdentifiersCallback = defaultHmacStrategy;
-
- // For aggregation indicate whether we should use the more verbose serialization format.
- boost::optional<ExplainOptions::Verbosity> verbosity = boost::none;
-
- // If set to true, serializes each stage and expression as needed for query analysis.
- bool serializeForQueryAnalysis = false;
-};
-
-} // namespace mongo
diff --git a/src/mongo/db/query/query_shape/shape_helpers.cpp b/src/mongo/db/query/query_shape/shape_helpers.cpp
deleted file mode 100644
index 8eea475ab78..00000000000
--- a/src/mongo/db/query/query_shape/shape_helpers.cpp
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_shape/shape_helpers.h"
-
-#include "mongo/db/query/query_shape/query_shape_gen.h"
-
-namespace mongo::shape_helpers {
-
-static constexpr StringData hintSpecialField = "$hint"_sd;
-// A "Flat" object is one with only top-level fields. We won't descend recursively to shapify any
-// sub-objects.
-BSONObj shapifyFlatObj(BSONObj obj, const SerializationOptions& opts, bool valuesAreLiterals) {
- if (obj.isEmpty()) {
- // fast-path for the common case.
- return obj;
- }
-
- BSONObjBuilder bob;
- for (BSONElement elem : obj) {
- if (hintSpecialField.compare(elem.fieldNameStringData()) == 0) {
- if (elem.type() == BSONType::String) {
- bob.append(hintSpecialField, opts.serializeFieldPathFromString(elem.String()));
- } else if (elem.type() == BSONType::Object) {
- opts.appendLiteral(&bob, hintSpecialField, elem.Obj());
- } else {
- // SERVER-85500: $hint syntax will not be validated if the collection does not
- // exist, so we should accept a value that is neither string nor object here.
- opts.appendLiteral(&bob, hintSpecialField, elem);
- }
- continue;
- }
-
- // $natural doesn't need to be redacted.
- if (elem.fieldNameStringData().compare(query_request_helper::kNaturalSortField) == 0) {
- bob.append(elem);
- continue;
- }
-
- if (valuesAreLiterals) {
- opts.appendLiteral(&bob, opts.serializeFieldPathFromString(elem.fieldName()), elem);
- } else {
- bob.appendAs(elem, opts.serializeFieldPathFromString(elem.fieldName()));
- }
- }
- return bob.obj();
-}
-
-BSONObj extractHintShape(BSONObj hintObj, const SerializationOptions& opts) {
- return shapifyFlatObj(hintObj, opts, /* valuesAreLiterals = */ false);
-}
-
-BSONObj extractMinOrMaxShape(BSONObj obj, const SerializationOptions& opts) {
- return shapifyFlatObj(obj, opts, /* valuesAreLiterals = */ true);
-}
-
-void appendNamespaceShape(BSONObjBuilder& bob,
- const NamespaceString& nss,
- const SerializationOptions& opts) {
- bob.append("db", opts.serializeIdentifier(nss.db()));
- bob.append("coll", opts.serializeIdentifier(nss.coll()));
-}
-
-NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt) {
- tassert(7632900, "cmdNs must be an object.", cmdNsElt.type() == BSONType::Object);
- auto cmdNs = query_shape::CommandNamespace::parse("cmdNs"_sd, cmdNsElt.embeddedObject());
-
- if (cmdNs.getColl().has_value()) {
- tassert(7632903,
- "Exactly one of 'uuid' and 'coll' can be defined.",
- !cmdNs.getUuid().has_value());
- return NamespaceString(cmdNs.getDb(), cmdNs.getColl().value());
- } else {
- tassert(7632904,
- "Exactly one of 'uuid' and 'coll' can be defined.",
- !cmdNs.getColl().has_value());
- UUID uuid = uassertStatusOK(UUID::parse(cmdNs.getUuid().value().toString()));
- return NamespaceStringOrUUID(cmdNs.getDb().toString(), uuid);
- }
-}
-
-} // namespace mongo::shape_helpers
diff --git a/src/mongo/db/query/query_shape/shape_helpers.h b/src/mongo/db/query/query_shape/shape_helpers.h
deleted file mode 100644
index 4d0fadb4a47..00000000000
--- a/src/mongo/db/query/query_shape/shape_helpers.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/bson/simple_bsonobj_comparator.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
-
-namespace mongo::shape_helpers {
-
-int64_t inline optionalObjSize(boost::optional<BSONObj> optionalObj) {
- if (!optionalObj)
- return 0;
- return optionalObj->objsize();
-}
-
-template <typename T>
-int64_t optionalSize(boost::optional<T> optionalVal) {
- if (!optionalVal)
- return 0;
- return optionalVal->size();
-}
-
-template <typename T>
-std::function<size_t(size_t, const T&)> sizeAccumulatorFunc() {
- MONGO_UNREACHABLE; // Don't know how to compute the size of this template type.
-};
-
-template <>
-inline std::function<size_t(size_t, const BSONObj&)> sizeAccumulatorFunc<BSONObj>() {
- return [](size_t total, const BSONObj& obj) {
- return total + sizeof(BSONObj) + static_cast<size_t>(obj.objsize());
- };
-}
-
-template <>
-inline std::function<size_t(size_t, const NamespaceString&)>
-sizeAccumulatorFunc<NamespaceString>() {
- return [](size_t total, const NamespaceString& nss) {
- // For each element, we have to track the size of the
- // nss as well as the size allocated by the nss. It would be
- // ideal to be able to ask the underlying namespace string for
- // its capacity, but it's not something we have access to.
- // Further, namespace strings appear to shrink to fit (i.e
- // resize to correct size), so it may not be necessary. Should
- // we also try to consider short string optimization? At the
- // very least, the current approach gives us a good upper bound
- // memory usage (assuming shrink to fit).
- return total + sizeof(nss) + nss.size();
- };
-}
-
-template <typename Container>
-size_t containerSize(const Container& container) {
- return std::accumulate(container.begin(),
- container.end(),
- 0,
- sizeAccumulatorFunc<typename Container::value_type>());
-}
-
-/**
- * Serializes the given 'hintObj' in accordance with the options. Assumes the hint is correct and
- * contains field names. It is possible that this hint doesn't actually represent an index, but we
- * can't detect that here.
- */
-BSONObj extractHintShape(BSONObj hintObj, const SerializationOptions& opts);
-BSONObj extractMinOrMaxShape(BSONObj obj, const SerializationOptions& opts);
-
-NamespaceStringOrUUID parseNamespaceShape(BSONElement cmdNsElt);
-void appendNamespaceShape(BSONObjBuilder& bob,
- const NamespaceString& nss,
- const SerializationOptions& opts);
-
-} // namespace mongo::shape_helpers
diff --git a/src/mongo/db/query/query_solution.cpp b/src/mongo/db/query/query_solution.cpp
index 191218b2f4d..dc24bc3653a 100644
--- a/src/mongo/db/query/query_solution.cpp
+++ b/src/mongo/db/query/query_solution.cpp
@@ -316,11 +316,7 @@ void CollectionScanNode::computeProperties() {
void CollectionScanNode::appendToString(str::stream* ss, int indent) const {
addIndent(ss, indent);
- if (doClusteredCollectionScan()) {
- *ss << "CLUSTERED_IDXSCAN\n";
- } else {
- *ss << "COLLSCAN\n";
- }
+ *ss << "COLLSCAN\n";
addIndent(ss, indent + 1);
*ss << "ns = " << name << '\n';
if (nullptr != filter) {
@@ -337,9 +333,6 @@ QuerySolutionNode* CollectionScanNode::clone() const {
copy->name = this->name;
copy->tailable = this->tailable;
copy->direction = this->direction;
- copy->minRecord = this->minRecord;
- copy->maxRecord = this->maxRecord;
- copy->clusteredIndex = this->clusteredIndex;
copy->shouldTrackLatestOplogTimestamp = this->shouldTrackLatestOplogTimestamp;
copy->assertTsHasNotFallenOffOplog = this->assertTsHasNotFallenOffOplog;
copy->shouldWaitForOplogVisibility = this->shouldWaitForOplogVisibility;
@@ -1090,8 +1083,8 @@ bool IndexScanNode::operator==(const IndexScanNode& other) const {
// ColumnIndexScanNode
//
ColumnIndexScanNode::ColumnIndexScanNode(ColumnIndexEntry indexEntry,
- OrderedPathSet outputFieldsIn,
- OrderedPathSet matchFieldsIn,
+ std::set<std::string> outputFieldsIn,
+ std::set<std::string> matchFieldsIn,
StringMap<std::unique_ptr<MatchExpression>> filtersByPath,
std::unique_ptr<MatchExpression> postAssemblyFilter)
: indexEntry(std::move(indexEntry)),
@@ -1566,11 +1559,11 @@ void GroupNode::appendToString(str::stream* ss, int indent) const {
if (idx > 0) {
*ss << ", ";
}
- *ss << "{" << groupName << ": " << exprObj->serialize().toString() << "}";
+ *ss << "{" << groupName << ": " << exprObj->serialize(false).toString() << "}";
++idx;
}
} else {
- *ss << "{_id: " << groupByExpression->serialize().toString() << "}";
+ *ss << "{_id: " << groupByExpression->serialize(false).toString() << "}";
}
*ss << '\n';
addIndent(ss, indent + 1);
@@ -1581,11 +1574,7 @@ void GroupNode::appendToString(str::stream* ss, int indent) const {
}
auto& acc = accumulators[idx];
*ss << "{" << acc.fieldName << ": {" << acc.expr.name << ": "
- << acc.expr.argument
- ->serialize(SerializationOptions{
- boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner)})
- .toString()
- << "}}";
+ << acc.expr.argument->serialize(true).toString() << "}}";
}
*ss << "]" << '\n';
addCommon(ss, indent);
diff --git a/src/mongo/db/query/query_solution.h b/src/mongo/db/query/query_solution.h
index 8b04e9f0bf3..f318884280b 100644
--- a/src/mongo/db/query/query_solution.h
+++ b/src/mongo/db/query/query_solution.h
@@ -463,10 +463,6 @@ struct CollectionScanNode : public QuerySolutionNodeWithSortSet {
QuerySolutionNode* clone() const;
- bool doClusteredCollectionScan() const {
- return (!isOplog && (minRecord || maxRecord));
- }
-
// Name of the namespace.
std::string name;
@@ -506,9 +502,6 @@ struct CollectionScanNode : public QuerySolutionNodeWithSortSet {
int direction{1};
- // Tells whether the collection is an oplog.
- bool isOplog = false;
-
// By default, includes the minRecord and maxRecord when present.
CollectionScanParams::ScanBoundInclusion boundInclusion =
CollectionScanParams::ScanBoundInclusion::kIncludeBothStartAndEndRecords;
@@ -522,8 +515,8 @@ struct CollectionScanNode : public QuerySolutionNodeWithSortSet {
struct ColumnIndexScanNode : public QuerySolutionNode {
ColumnIndexScanNode(ColumnIndexEntry,
- OrderedPathSet outputFields,
- OrderedPathSet matchFields,
+ std::set<std::string> outputFields,
+ std::set<std::string> matchFields,
StringMap<std::unique_ptr<MatchExpression>> filtersByPath,
std::unique_ptr<MatchExpression> postAssemblyFilter);
@@ -563,11 +556,11 @@ struct ColumnIndexScanNode : public QuerySolutionNode {
ColumnIndexEntry indexEntry;
// The fields we need to output. Dot separated path names.
- OrderedPathSet outputFields;
+ std::set<std::string> outputFields;
// The fields which are referenced by any and all filters - either in 'filtersByPath' or
// 'postAssemblyFilter'.
- OrderedPathSet matchFields;
+ std::set<std::string> matchFields;
// A column scan can apply a filter to the columns directly while scanning, or to a document
// assembled from the scanned columns.
@@ -582,7 +575,7 @@ struct ColumnIndexScanNode : public QuerySolutionNode {
// A cached copy of the union of the above two field sets which we expect to be frequently asked
// for.
- OrderedPathSet allFields;
+ std::set<std::string> allFields;
};
/**
@@ -1401,13 +1394,11 @@ struct GroupNode : public QuerySolutionNode {
for (auto& groupByExprField : groupByExpression->getDependencies().fields) {
requiredFields.insert(groupByExprField);
}
- needWholeDocument = groupByExpression->getDependencies().needWholeDocument;
for (auto&& acc : accumulators) {
auto argExpr = acc.expr.argument;
for (auto& argExprField : argExpr->getDependencies().fields) {
requiredFields.insert(argExprField);
}
- needWholeDocument |= argExpr->getDependencies().needWholeDocument;
}
}
@@ -1443,7 +1434,6 @@ struct GroupNode : public QuerySolutionNode {
// the fields in the 'groupByExpressions' and the fields in the input Expressions of the
// 'accumulators'.
StringSet requiredFields;
- bool needWholeDocument = false;
// If set to true, generated SBE plan will produce result as BSON object. If false,
// 'sbe::Object' is produced instead.
diff --git a/src/mongo/db/query/query_stats/README.md b/src/mongo/db/query/query_stats/README.md
deleted file mode 100644
index 6f2667fbfd9..00000000000
--- a/src/mongo/db/query/query_stats/README.md
+++ /dev/null
@@ -1,200 +0,0 @@
-# Query Stats
-This directory is the home of the infrastructure related to recording runtime query statistics for
-the database. It is not to be confused with `src/mongo/db/query/stats/` which is the home of the
-logic for computing and maintaining statistics about a collection or index's data distribution - for
-use by the query planner.
-
-The system will collect metrics for each query execution, and the results will be aggregated in a
-structure called the [`QueryStatsStore`](#querystatsstore) upon completion of each successful
-execution. Metrics will be aggregated according to an abstracted version of the query known as the
-query stats key and will be collected on any mongod or mongos process for which they are configured,
-including primaries and secondaries.
-
-## QueryStatsStore
-At the center of everything here is the [`QueryStatsStore`](query_stats.h#93-97), which is a
-partitioned hash table that maps the hash of a [Query Stats Key](#glossary) (also known as the
-_Query Stats Store Key_) to some metrics about how often each one occurs.
-
-### Computing the Query Stats Store Key
-A query stats store key contains various dimensions that distinctify a specific query. One main
-attribute to the query stats store key, is the query shape (`query_shape::Shape`). For example, if
-the client does this:
-```js
-db.example.findOne({x: 24});
-db.example.findOne({x: 53});
-```
-then the `QueryStatsStore` should contain an entry for a single query shape which would record 2
-executions and some related statistics (see [`QueryStatsEntry`](query_stats_entry.h) for details).
-
-For more information on query shape, see the [query_shape](../query_shape/README.md) directory.
-
-The query stats store has _more_ dimensions (i.e. more granularity) to group incoming queries than
-just the query shape. For example, these queries would all three have the same shape but the first
-would have a different query stats store entry from the other two:
-```js
-db.example.find({x: 55});
-db.example.find({x: 55}).batchSize(2);
-db.example.find({x: 55}).batchSize(3);
-```
-There are two distinct query stats store entries here - both the examples which include the batch
-size will be treated separately from the example which does not specify a batch size.
-
-The dimensions considered will depend on the command, but can generally be found in the
-[`KeyGenerator`](key_generator.h) interface, which will generate the query stats store keys by which
-we accumulate statistics. As one example, you can find the
-[`FindKey`](find_key.h) which will include all the things tracked in the
-`FindCmdQueryStatsStoreKeyComponents` (including `batchSize` shown in this example).
-
-### Query Stats Store Cache Size
-The size of the`QueryStatsStore` can be set by the server parameter
-[`internalQueryStatsCacheSize`](#server-parameters), and the partitions will be created based off
-that. See [`queryStatsStoreManagerRegisterer`](query_stats.cpp#L138-L154) for more details about how
-the number of partitions and their size is determined; Each partition is an LRU cache, therefore, if
-adding a new entry to the partition makes it go over its size limit, the least recently used entries
-will be evicted to drop below the max size. Eviction will be tracked in the new [server status
-metrics](#server-status-metrics) for queryStats.
-
-## Metric Collection
-At a high level, when a query is run and collection of query stats is enabled, during planning we
-call [`registerRequest`]((query_stats.h#L195-L198)) in which the query stats store key will be
-generated based on the query's shape and the various other dimensions. The key will always be serialized
-and stored on the `opDebug`, and also on the cursor in the case that there are `getMore`s, so that we can
-continue to aggregate the operation's metrics. Once the query execution is fully complete,
-[`writeQueryStats`](query_stats.h#L200-216) will be called and will either retrieve the entry for
-the key from the store if it exists and update it, or create a new one and add it to the store. See
-more details in the [comments](query_stats.h#L158-L216).
-
-### Rate Limiting
-Whether or not query stats will be recorded for a specific query execution depends on a Rate
-Limiter, which limits the number of recordings per second based on the server parameter
-[internalQueryStatsRateLimit](#server-parameters). The goal of the rate limiter is to minimize
-impact to overall system performance through restricting excessive traffic. If a query is run but
-the rate limit has been reached, the query will still execute as expected but query stats will not
-be updated in the query stats store. Our rate limiter uses the sliding window algorithm; see details
-[here](rate_limiting.h#82-87).
-
-## Metric Retrieval
-To retrieve the stats gathered in the `QueryStatsStore`, there is a new aggregation stage,
-`$queryStats`. This stage must be the first in a pipeline and it must be run against the admin
-database. The structure of the command is as follows (note `aggregate: 1` reflecting there is no collection):
-```js
-db.adminCommand({
- aggregate: 1,
- pipeline: [{
- $queryStats: {
- tranformIdentifiers: {
- algorithm: "hmac-sha-256",
- hmacKey: BinData(8, "87c4082f169d3fef0eef34dc8e23458cbb457c3sf3n2") /* bindata
- subtype 8 - a new type for sensitive data */,
- }
- }
- }]
-})
-```
-`transformIdentifiers` is optional. If not present, we will generate the regular Query Stats Key. If
-present:
-- `algorithm` is required and the only currently supported option is "hmac-sha-256".
-- `hmacKey` is required
-- We will generate the [One-way Tokenized](#glossary) Query Stats Key by applying the "hmac-sha-256"
- to the names of any field, collection, or database. Application Name field is not transformed.
-
-The query stats store will output one document for each query stats key, which is structured in the
-following way:
-```js
-{
- key: {/* Query Stats Key */},
- asOf: ISODate(/* … */),
- metrics: {
- execCount: 0,
- firstSeenTimestamp: ISODate(/* … */),
- latestSeenTimestamp: ISODate(/* … */),
- docsReturned: {sum: 0, max: 0, min: 0, sumOfSquares: 0},
- firstResponseExecMicros: {sum: 0, max: 0, min: 0, sumOfSquares: 0},
- totalExecMicros: {sum: 0, max: 0, min: 0, sumOfSquares: 0},
- lastExecutionMicros: 0,
- }
-}
-```
-- `key`: Query Stats Key.
-- `asOf`: UTC time when $queryStats read this entry from the store. This will not return the same
- UTC time for each result. The data structure used for the store is partitioned, and each partition
- will be read at a snapshot individually. You may see up to the number of partitions in unique
- timestamps returned by one $queryStats cursor.
-- `metrics`: the metrics collected; these may be flawed due to:
- - Server restarts, which will reset metrics.
- - LRU eviction, which will reset metrics.
- - Rate limiting, which will skew metrics.
-- `metrics.execCount`: Number of recorded observations of this query.
-- `metrics.firstSeenTimestamp`: UTC time taken at query completion (including getMores) for the
- first recording of this query stats store entry.
-- `metrics.lastSeenTimestamp`: UTC time taken at query completion (including getMores) for the
- latest recording of this query stats store entry.
-- `metrics.docsReturned`: Various broken down metrics for the number of documents returned by
- observation of this query.
-- `metrics.firstResponseExecMicros`: Estimated time spent computing and returning the first batch.
-- `metrics.totalExecMicros`: Estimated time spent computing and returning all batches, which is the
- same as the above for single-batch queries.
-- `metrics.lastExecutionMicros`: Estimated time spent processing the latest query (akin to
- "totalExecMicros", not "firstResponseExecMicros").
-
-#### Permissions
-`$queryStats` is restricted by two privilege actions:
-- `queryStatsRead` privilege allows running `$queryStats` without passing the `transformIdentifiers`
- options.
-- `queryStatsReadTransformed` allows running `$queryStats` with `transformIdentifiers` set. These
-two privileges are included in the clusterMonitor role in Atlas.
-
-### Server Parameters
-- `internalQueryStatsCacheSize`:
- * Max query stats store size, specified as a string like "4MB" or "1%". Defaults to 1% of the
- machine's total memory.
- * Query stats store is a LRU cache structure with partitions, so we may be under the cap due to
- implementation.
-
-- `internalQueryStatsRateLimit`:
- * The rate limit is an integer which imposes a maximum number of recordings per second. Default is
- 0 which has the effect of disabling query stats collection. Setting the parameter to -1 means
- there will be no rate limit.
-
-- `logComponentVerbosity.queryStats`:
- * Controls the logging behavior for query stats. See [Logging](#logging) for details.
-
-### Logging
-Setting `logComponentVerbosity.queryStats` will do the following for each level:
-* Level 0 (default): Nothing will be logged.
-* Level 1 or higher: Invocations of $queryStats will be logged if and only if the algorithm is
- "hmac-sha-256". The specification of the $queryStats stage is logged, with any provided hmac key
- redacted.
-* Level 2 or higher: Nothing extra, reserved for future use.
-* Level 3 or higher: All results of any "hmac-sha-256" $queryStats invocation are logged. Each
- result will be its own entry and there will be one final entry that says "we finished".
-* Levels 4 and 5 do nothing extra.
-
-### Server Status Metrics
-The following will be added to the `serverStatus.metrics`:
-```js
-queryStats: {
- numEvicted: NumberLong(0),
- numHmacApplicationErrors: NumberLong(0),
- numQueryStatsStoreWriteErrors: NumberLong(0),
- numRateLimitedRequests: NumberLong(0),
- queryStatsStoreSizeEstimateBytes: NumberLong(0)
-}
-```
-
-# Glossary
-**Query Execution**: This term implies the overall execution of what a client would consider one
-query, but which may or may not involve one or more getMore commands to iterate a cursor. For
-example, a find command and two getMore commands on the returned cursor is one query execution. An
-aggregate command which returns everything in one batch is also one query execution.
-
-**One-way Tokenized Object**: A one-way tokenized object has an HMAC hashing function applied to
-particular sensitive elements/pieces of an object. It is "one-way" because it is never meant to be
-undone. This allows us to detect when two queries are using the same identifiers, but never to
-reveal what those identifiers were.
-
-**Query Shape**: [Query Shape](../query_shape/README.md)
-
-**Query Stats Key**: Also known as the _Query Stats Store Key_, this is the collection of attributes
-championed by the query shape which identifies one grouping of metrics. The $queryStats stage will
-output one document per query stats key - output in the "key" field.
diff --git a/src/mongo/db/query/query_stats/SConscript b/src/mongo/db/query/query_stats/SConscript
deleted file mode 100644
index f9f3a8b1c2e..00000000000
--- a/src/mongo/db/query/query_stats/SConscript
+++ /dev/null
@@ -1,121 +0,0 @@
-# -*- mode: python -*-
-
-Import([
- "env",
- "get_option",
-])
-
-env = env.Clone()
-
-env.Library(
- target='rate_limiting',
- source=[
- 'rate_limiting.cpp',
- ],
- LIBDEPS=[
- '$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/util/clock_sources',
- ],
-)
-
-env.Library(target='query_stats_parse', source=['transform_algorithm.idl'], LIBDEPS=[
- '$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/idl/idl_parser',
-])
-
-env.Library(
- target='query_stats',
- source=[
- '$BUILD_DIR/mongo/db/curop.cpp',
- 'key.cpp',
- 'query_stats.cpp',
- 'query_stats_entry.cpp'
- ],
- LIBDEPS=[
- '$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/bson/mutable/mutable_bson',
- '$BUILD_DIR/mongo/db/commands',
- '$BUILD_DIR/mongo/db/concurrency/lock_manager',
- '$BUILD_DIR/mongo/db/exec/document_value/document_value',
- '$BUILD_DIR/mongo/db/generic_cursor',
- '$BUILD_DIR/mongo/db/profile_filter',
- '$BUILD_DIR/mongo/db/query/command_request_response',
- '$BUILD_DIR/mongo/db/query/memory_util',
- '$BUILD_DIR/mongo/db/query/query_knobs',
- '$BUILD_DIR/mongo/db/query/query_shape/query_shape',
- '$BUILD_DIR/mongo/db/server_options',
- '$BUILD_DIR/mongo/db/service_context',
- '$BUILD_DIR/mongo/db/stats/counters',
- '$BUILD_DIR/mongo/db/stats/timer_stats',
- '$BUILD_DIR/mongo/db/storage/storage_engine_parameters',
- '$BUILD_DIR/mongo/rpc/client_metadata',
- '$BUILD_DIR/mongo/transport/service_executor',
- '$BUILD_DIR/mongo/util/diagnostic_info' if get_option('use-diagnostic-latches') == 'on' else [],
- '$BUILD_DIR/mongo/util/fail_point',
- '$BUILD_DIR/mongo/util/net/network',
- '$BUILD_DIR/mongo/util/processinfo',
- '$BUILD_DIR/mongo/util/progress_meter',
- 'query_stats_parse',
- 'rate_limiting',
- ],
- LIBDEPS_PRIVATE=[
- '$BUILD_DIR/mongo/db/auth/auth',
- '$BUILD_DIR/mongo/db/auth/user_acquisition_stats',
- '$BUILD_DIR/mongo/db/exec/projection_executor',
- '$BUILD_DIR/mongo/db/prepare_conflict_tracker',
- '$BUILD_DIR/mongo/db/stats/resource_consumption_metrics',
- ],
-)
-
-env.CppUnitTest(
- target="db_query_query_stats_test",
- source=[
- "agg_key_test.cpp",
- "find_key_test.cpp",
- "key_test.cpp",
- "query_stats_test.cpp",
- "query_stats_store_test.cpp",
- "rate_limiting_test.cpp",
- ],
- LIBDEPS=[
- "$BUILD_DIR/mongo/db/auth/authmocks",
- "$BUILD_DIR/mongo/db/query/query_shape/query_shape",
- "$BUILD_DIR/mongo/db/query/query_test_service_context",
- "$BUILD_DIR/mongo/db/service_context_d_test_fixture",
- "query_stats",
- "rate_limiting",
- ],
-)
-
-env.Benchmark(
- target='rate_limiting_bm',
- source=[
- 'rate_limiting_bm.cpp',
- ],
- LIBDEPS=[
- '$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/unittest/unittest',
- '$BUILD_DIR/mongo/util/processinfo',
- 'rate_limiting',
- ],
-)
-
-env.Benchmark(
- target='shapifying_bm',
- source=[
- 'shapifying_bm.cpp',
- ],
- LIBDEPS=[
- '$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/db/auth/auth',
- '$BUILD_DIR/mongo/db/pipeline/pipeline',
- '$BUILD_DIR/mongo/db/query/canonical_query',
- '$BUILD_DIR/mongo/db/query/query_shape/query_shape',
- '$BUILD_DIR/mongo/db/query/query_test_service_context',
- '$BUILD_DIR/mongo/db/service_context',
- '$BUILD_DIR/mongo/rpc/client_metadata',
- '$BUILD_DIR/mongo/unittest/unittest',
- '$BUILD_DIR/mongo/util/processinfo',
- 'query_stats',
- ],
-)
diff --git a/src/mongo/db/query/query_stats/agg_key.cpp b/src/mongo/db/query/query_stats/agg_key.cpp
deleted file mode 100644
index 1d53418d371..00000000000
--- a/src/mongo/db/query/query_stats/agg_key.cpp
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_stats/agg_key.h"
-
-#include "mongo/db/query/explain_options.h"
-#include <absl/container/node_hash_set.h>
-#include <boost/cstdint.hpp>
-#include <functional>
-#include <initializer_list>
-#include <memory>
-#include <numeric>
-#include <vector>
-
-#include <boost/move/utility_core.hpp>
-#include <boost/optional/optional.hpp>
-#include <boost/smart_ptr/intrusive_ptr.hpp>
-
-#include "mongo/crypto/fle_field_schema_gen.h"
-#include "mongo/db/pipeline/exchange_spec_gen.h"
-#include "mongo/db/pipeline/pipeline.h"
-#include "mongo/db/query/query_shape/agg_cmd_shape.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
-#include "mongo/db/query/query_shape/shape_helpers.h"
-#include "mongo/idl/basic_types_gen.h"
-#include "mongo/util/assert_util.h"
-
-namespace mongo::query_stats {
-
-AggCmdComponents::AggCmdComponents(const AggregateCommandRequest& request_,
- stdx::unordered_set<NamespaceString> involvedNamespaces_)
- : involvedNamespaces(std::move(involvedNamespaces_)),
- _bypassDocumentValidation(request_.getBypassDocumentValidation().value_or(false)),
- _verbosity(request_.getExplain()),
- _hasField() {
- _hasField.batchSize = request_.getCursor().getBatchSize().has_value();
- _hasField.bypassDocumentValidation = request_.getBypassDocumentValidation().has_value();
- _hasField.explain = request_.getExplain().has_value();
- _hasField.passthroughToShard = request_.getPassthroughToShard().has_value();
-}
-
-
-void AggCmdComponents::HashValue(absl::HashState state) const {
- // The hashing for verbosity in this branch needed to be different because the compiler was
- // complaining about the different wrappers. This is not important since this computation is
- // only used locally in memory on a single machine, and the query shape is still stable.
- auto verbosity =
- _hasField.explain ? std::string(ExplainOptions::verbosityString(_verbosity.value())) : "";
- state = absl::HashState::combine(std::move(state),
- _bypassDocumentValidation,
- _hasField.batchSize,
- _hasField.bypassDocumentValidation,
- verbosity,
- _hasField.explain,
- _hasField.passthroughToShard);
- // We don't need to add 'involvedNamespaces' here since they are already tracked/duplicated in
- // the Pipeline component of the query shape. We just expose them here for ease of
- // analysis/querying.
-}
-
-void AggCmdComponents::appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const {
-
- // otherNss
- if (!involvedNamespaces.empty()) {
- BSONArrayBuilder otherNss = bob.subarrayStart(kOtherNssFieldName);
- for (const auto& nss : involvedNamespaces) {
- BSONObjBuilder otherNsEntryBob = otherNss.subobjStart();
- shape_helpers::appendNamespaceShape(otherNsEntryBob, nss, opts);
- otherNsEntryBob.doneFast();
- }
- otherNss.doneFast();
- }
-
- // bypassDocumentValidation
- if (_hasField.bypassDocumentValidation) {
- bob.append(AggregateCommandRequest::kBypassDocumentValidationFieldName,
- _bypassDocumentValidation);
- }
-
- // We don't store the specified batch size values since they don't matter.
- // Provide an arbitrary literal long here.
-
- tassert(78429,
- "Serialization policy not supported - original values have been discarded",
- opts.literalPolicy != LiteralSerializationPolicy::kUnchanged);
-
- if (_hasField.batchSize) {
- // cursor
- BSONObjBuilder cursorInfo = bob.subobjStart(AggregateCommandRequest::kCursorFieldName);
- opts.appendLiteral(&cursorInfo, SimpleCursorOptions::kBatchSizeFieldName, 0ll);
- cursorInfo.doneFast();
- }
-
- if (_hasField.explain) {
- // The verbosity can be explicitly set by using the .explain() command, but when using the
- // flag {explain: true} it is set to 'queryPlanner'.
- bob.append(AggregateCommandRequest::kExplainFieldName,
- ExplainOptions::verbosityString(_verbosity.value()));
- }
-
- // The values here don't matter (assuming we're not using the 'kUnchanged' policy).
- tassert(8949601,
- "Serialization policy not supported - original values have been discarded",
- opts.literalPolicy != LiteralSerializationPolicy::kUnchanged);
- if (_hasField.passthroughToShard) {
- BSONObjBuilder passthroughToShardInfo =
- bob.subobjStart(AggregateCommandRequest::kPassthroughToShardFieldName);
- static const PassthroughToShardOptions representativePassthroughOptions = []() {
- PassthroughToShardOptions passthroughOpts;
- // The value doesn't matter since we will only use this for shapified output.
- passthroughOpts.setShard("?");
- return passthroughOpts;
- }();
- representativePassthroughOptions.serialize(&passthroughToShardInfo, opts);
- passthroughToShardInfo.doneFast();
- }
-}
-
-size_t AggCmdComponents::size() const {
- return sizeof(AggCmdComponents) +
- std::accumulate(involvedNamespaces.begin(),
- involvedNamespaces.end(),
- 0,
- [](int64_t total, const auto& nss) { return total + nss.size(); });
-}
-
-void AggKey::appendCommandSpecificComponents(BSONObjBuilder& bob,
- const SerializationOptions& opts) const {
- return _components.appendTo(bob, opts);
-}
-
-AggKey::AggKey(AggregateCommandRequest request,
- const Pipeline& pipeline,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- stdx::unordered_set<NamespaceString> involvedNamespaces,
- const NamespaceString& origNss,
- query_shape::CollectionType collectionType)
- : Key(expCtx->opCtx,
- std::make_unique<query_shape::AggCmdShape>(
- request, origNss, involvedNamespaces, pipeline, expCtx),
- request.getHint(),
- request.getReadConcern(),
- request.getMaxTimeMS().has_value(),
- collectionType),
- _components(request, std::move(involvedNamespaces)) {}
-
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/agg_key.h b/src/mongo/db/query/query_stats/agg_key.h
deleted file mode 100644
index 38b80e28006..00000000000
--- a/src/mongo/db/query/query_stats/agg_key.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include <cstdint>
-#include <utility>
-
-#include <absl/container/node_hash_map.h>
-#include <boost/move/utility_core.hpp>
-#include <boost/none.hpp>
-#include <boost/optional/optional.hpp>
-#include <boost/smart_ptr/intrusive_ptr.hpp>
-
-#include "mongo/base/string_data.h"
-#include "mongo/bson/bsonobj.h"
-#include "mongo/bson/bsonobjbuilder.h"
-#include "mongo/db/collection_type.h"
-#include "mongo/db/namespace_string.h"
-#include "mongo/db/operation_context.h"
-#include "mongo/db/pipeline/aggregate_command_gen.h"
-#include "mongo/db/pipeline/expression_context.h"
-#include "mongo/db/pipeline/pipeline.h"
-#include "mongo/db/pipeline/variables.h"
-#include "mongo/db/query/explain_options.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_stats/key.h"
-
-namespace mongo::query_stats {
-
-/**
- * Struct representing the aggregate command's unique arguments which should be included in the
- * query stats key.
- */
-struct AggCmdComponents : public SpecificKeyComponents {
- static constexpr StringData kOtherNssFieldName = "otherNss"_sd;
-
- AggCmdComponents(const AggregateCommandRequest&,
- stdx::unordered_set<NamespaceString> involvedNamespaces);
-
- void HashValue(absl::HashState state) const final;
-
- void appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const;
-
- size_t size() const;
-
- stdx::unordered_set<NamespaceString> involvedNamespaces;
- bool _bypassDocumentValidation;
- const boost::optional<mongo::ExplainOptions::Verbosity> _verbosity;
-
- // This anonymous struct represents the presence of the member variables as C++ bit fields.
- // In doing so, each of these boolean values takes up 1 bit instead of 1 byte.
- struct HasField {
- HasField() : batchSize(false), bypassDocumentValidation(false), explain(false) {}
- bool batchSize : 1;
- bool bypassDocumentValidation : 1;
- bool explain : 1;
- bool passthroughToShard : 1;
- } _hasField;
-};
-
-/**
- * Handles shapification for AggregateCommandRequests. Requires a pre-parsed pipeline in order to
- * avoid parsing the raw pipeline multiple times, but users should be sure to provide a
- * non-optimized pipeline.
- */
-class AggKey final : public Key {
-public:
- AggKey(AggregateCommandRequest request,
- const Pipeline& pipeline,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- stdx::unordered_set<NamespaceString> involvedNamespaces,
- const NamespaceString& origNss,
- query_shape::CollectionType collectionType = query_shape::CollectionType::kUnknown);
-
- const SpecificKeyComponents& specificComponents() const final {
- return _components;
- }
-
- // The default implementation of hashing for smart pointers is not a good one for our purposes.
- // Here we overload them to actually take the hash of the object, rather than hashing the
- // pointer itself.
- template <typename H>
- friend H AbslHashValue(H h, const std::unique_ptr<const AggKey>& key) {
- return H::combine(std::move(h), *key);
- }
- template <typename H>
- friend H AbslHashValue(H h, const std::shared_ptr<const AggKey>& key) {
- return H::combine(std::move(h), *key);
- }
-
-
-protected:
- void appendCommandSpecificComponents(BSONObjBuilder& bob,
- const SerializationOptions& opts) const final override;
-
-private:
- const AggCmdComponents _components;
-};
-static_assert(
- sizeof(AggKey) == sizeof(Key) + sizeof(AggCmdComponents),
- "If the class' members have changed, this assert may need to be updated with a new value.");
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/agg_key_test.cpp b/src/mongo/db/query/query_stats/agg_key_test.cpp
deleted file mode 100644
index 35d0ae20d86..00000000000
--- a/src/mongo/db/query/query_stats/agg_key_test.cpp
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Copyright (C) 2024-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include <boost/smart_ptr/intrusive_ptr.hpp>
-
-#include "mongo/db/pipeline/expression_context.h"
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/pipeline/pipeline.h"
-#include "mongo/db/query/query_shape/agg_cmd_shape.h"
-#include "mongo/db/query/query_stats/agg_key.h"
-#include "mongo/db/service_context_test_fixture.h"
-#include "mongo/idl/basic_types.h"
-#include "mongo/unittest/unittest.h"
-#include "mongo/util/intrusive_counter.h"
-
-namespace mongo::query_stats {
-
-namespace {
-
-static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl");
-
-static constexpr auto collectionType = query_shape::CollectionType::kCollection;
-
-class AggKeyTest : public ServiceContextTest {
-public:
- static std::unique_ptr<const Key> makeAggKeyFromRawPipeline(
- const std::vector<BSONObj>& rawPipeline) {
- auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss);
- AggregateCommandRequest acr(kDefaultTestNss);
- acr.setPipeline(rawPipeline);
- auto pipeline = Pipeline::parse(rawPipeline, expCtx);
- return std::make_unique<AggKey>(acr,
- *pipeline,
- expCtx,
- pipeline->getInvolvedCollections(),
- acr.getNamespace(),
- collectionType);
- }
- size_t namespaceSize(stdx::unordered_set<NamespaceString> involvedNamespaces) {
- return std::accumulate(involvedNamespaces.begin(),
- involvedNamespaces.end(),
- 0,
- [](int64_t total, const auto& nss) { return total + nss.size(); });
- }
-};
-
-TEST_F(AggKeyTest, SizeOfAggCmdComponents) {
- auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss);
- auto rawPipeline = {fromjson(R"({
- $match: {
- foo: { $in: ["a", "b"] },
- bar: { $gte: { $date: "2022-01-01T00:00:00Z" } }
- }
- })")};
-
- AggregateCommandRequest acr(kDefaultTestNss);
- acr.setPipeline(rawPipeline);
- auto pipeline = Pipeline::parse(rawPipeline, expCtx);
- auto namespaces = pipeline->getInvolvedCollections();
- auto aggComponents = std::make_unique<AggCmdComponents>(acr, namespaces);
-
- const auto minimumSize = sizeof(SpecificKeyComponents) +
- sizeof(stdx::unordered_set<NamespaceString>) + 2 /*size for bool and HasField*/ +
- sizeof(boost::optional<mongo::ExplainOptions::Verbosity>) + namespaceSize(namespaces);
- ASSERT_GTE(aggComponents->size(), minimumSize);
- ASSERT_LTE(aggComponents->size(), minimumSize + 8 /*padding*/);
-}
-
-TEST_F(AggKeyTest, EquivalentAggCmdComponentSizes) {
- auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss);
- auto rawPipeline = {fromjson(R"({
- $match: {
- foo: { $in: ["a", "b"] },
- bar: { $gte: { $date: "2022-01-01T00:00:00Z" } }
- }
- })")};
-
- // Set different values in the command request.
- AggregateCommandRequest acrBypassTrue(kDefaultTestNss);
- acrBypassTrue.setPipeline(rawPipeline);
- acrBypassTrue.setBypassDocumentValidation(true);
- auto pipeline = Pipeline::parse(rawPipeline, expCtx);
- auto namespaces = pipeline->getInvolvedCollections();
- auto aggComponentsBypassTrue = std::make_unique<AggCmdComponents>(acrBypassTrue, namespaces);
-
-
- AggregateCommandRequest acrBypassFalse(kDefaultTestNss);
- acrBypassFalse.setPipeline(rawPipeline);
- acrBypassFalse.setBypassDocumentValidation(false);
- auto aggComponentsBypassFalse = std::make_unique<AggCmdComponents>(acrBypassFalse, namespaces);
-
- ASSERT_EQ(aggComponentsBypassTrue->size(), aggComponentsBypassFalse->size());
-}
-
-TEST_F(AggKeyTest, DifferentAggCmdComponentSizes) {
- auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss);
- auto rawPipeline = {fromjson(R"({
- $match: {
- foo: { $in: ["a", "b"] },
- bar: { $gte: { $date: "2022-01-01T00:00:00Z" } }
- }
- })")};
-
- AggregateCommandRequest acr(kDefaultTestNss);
- acr.setPipeline(rawPipeline);
- // Manually creating different namespaces for testing purposes.
- const auto namespaceStringOne = NamespaceString("testDB.testColl1");
- const auto namespaceStringTwo = NamespaceString("testDB.testColl2");
-
- stdx::unordered_set<NamespaceString> smallNamespaces;
- smallNamespaces.insert(namespaceStringOne);
-
- stdx::unordered_set<NamespaceString> largeNamespaces;
- largeNamespaces.insert(namespaceStringOne);
- largeNamespaces.insert(namespaceStringTwo);
-
- auto smallAggComponents = std::make_unique<AggCmdComponents>(acr, smallNamespaces);
- auto largeAggComponents = std::make_unique<AggCmdComponents>(acr, largeNamespaces);
-
- ASSERT_LT(namespaceSize(smallNamespaces), namespaceSize(largeNamespaces));
- ASSERT_LT(smallAggComponents->size(), largeAggComponents->size());
-}
-
-// Testing item in opCtx that should impact key size.
-TEST_F(AggKeyTest, SizeOfAggKeyWithAndWithoutWriteConcern) {
- auto rawPipeline = {fromjson(R"({
- $match: {
- foo: { $in: ["a", "b"] },
- bar: { $gte: { $date: "2022-01-01T00:00:00Z" } }
- }
- })")};
-
- auto keyWithoutComment = makeAggKeyFromRawPipeline(rawPipeline);
-
- auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss);
- AggregateCommandRequest acrWithComment(kDefaultTestNss);
- acrWithComment.setPipeline(rawPipeline);
- expCtx->opCtx->setComment(BSON("comment"
- << " foo"));
- auto pipelineWithComment = Pipeline::parse(rawPipeline, expCtx);
- auto keyWithComment = std::make_unique<AggKey>(acrWithComment,
- *pipelineWithComment,
- expCtx,
- pipelineWithComment->getInvolvedCollections(),
- acrWithComment.getNamespace(),
- collectionType);
-
- ASSERT_LT(keyWithoutComment->size(), keyWithComment->size());
-}
-
-// Testing item in command request that should impact key size.
-TEST_F(AggKeyTest, SizeOfAggKeyWithAndWithoutReadConcern) {
- auto rawPipeline = {fromjson(R"({
- $match: {
- foo: { $in: ["a", "b"] },
- bar: { $gte: { $date: "2022-01-01T00:00:00Z" } }
- }
- })")};
-
- auto keyWithoutReadConcern = makeAggKeyFromRawPipeline(rawPipeline);
-
- auto expCtx = make_intrusive<ExpressionContextForTest>(kDefaultTestNss);
- AggregateCommandRequest acrWithReadConcern(kDefaultTestNss);
- acrWithReadConcern.setPipeline(rawPipeline);
- acrWithReadConcern.setReadConcern(fromjson(R"({level: "local"})"));
- auto pipelineWithReadConcern = Pipeline::parse(rawPipeline, expCtx);
- auto keyWithReadConcern =
- std::make_unique<AggKey>(acrWithReadConcern,
- *pipelineWithReadConcern,
- expCtx,
- pipelineWithReadConcern->getInvolvedCollections(),
- acrWithReadConcern.getNamespace(),
- collectionType);
-
- ASSERT_LT(keyWithoutReadConcern->size(), keyWithReadConcern->size());
-}
-} // namespace
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/aggregate_key_generator.cpp b/src/mongo/db/query/query_stats/aggregate_key_generator.cpp
deleted file mode 100644
index f175df296f2..00000000000
--- a/src/mongo/db/query/query_stats/aggregate_key_generator.cpp
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_stats/aggregate_key_generator.h"
-
-#include "mongo/db/pipeline/pipeline.h"
-#include "mongo/db/query/query_shape.h"
-#include "mongo/db/query/serialization_options.h"
-#include "mongo/db/query/shape_helpers.h"
-
-namespace mongo::query_stats {
-
-BSONObj AggregateKeyGenerator::generate(
- OperationContext* opCtx,
- boost::optional<SerializationOptions::TokenizeIdentifierFunc> hmacPolicy) const {
- // TODO SERVER-76087 We will likely want to set a flag here to stop $search from calling out
- // to mongot.
- auto expCtx = makeDummyExpCtx(opCtx);
- SerializationOptions opts{LiteralSerializationPolicy::kToDebugTypeString};
- if (hmacPolicy) {
- opts.transformIdentifiers = true;
- opts.transformIdentifiersCallback = *hmacPolicy;
- opts.includePath = true;
- opts.verbosity = boost::none;
- }
-
- return makeQueryStatsKey(opts, expCtx);
-}
-
-void AggregateKeyGenerator::appendCommandSpecificComponents(
- BSONObjBuilder& bob, const SerializationOptions& opts) const {
- // cursor
- if (auto param = _request.getCursor().getBatchSize()) {
- BSONObjBuilder cursorInfo = bob.subobjStart(AggregateCommandRequest::kCursorFieldName);
- opts.appendLiteral(&cursorInfo,
- SimpleCursorOptions::kBatchSizeFieldName,
- static_cast<long long>(param.get()));
- cursorInfo.doneFast();
- }
-
- // maxTimeMS
- if (auto param = _request.getMaxTimeMS()) {
- opts.appendLiteral(&bob,
- AggregateCommandRequest::kMaxTimeMSFieldName,
- static_cast<long long>(param.get()));
- }
-
- // bypassDocumentValidation
- if (auto param = _request.getBypassDocumentValidation()) {
- opts.appendLiteral(
- &bob, AggregateCommandRequest::kBypassDocumentValidationFieldName, bool(param.get()));
- }
-
- // otherNss
- if (!_involvedNamespaces.empty()) {
- BSONArrayBuilder otherNss = bob.subarrayStart(kOtherNssFieldName);
- for (const auto& nss : _involvedNamespaces) {
- BSONObjBuilder otherNsEntryBob = otherNss.subobjStart();
- shape_helpers::appendNamespaceShape(otherNsEntryBob, nss, opts);
- otherNsEntryBob.doneFast();
- }
- otherNss.doneFast();
- }
-}
-
-BSONObj AggregateKeyGenerator::makeQueryStatsKey(
- const SerializationOptions& opts, const boost::intrusive_ptr<ExpressionContext>& expCtx) const {
- auto pipeline = Pipeline::parse(_request.getPipeline(), expCtx);
- return _makeQueryStatsKeyHelper(opts, expCtx, *pipeline);
-}
-
-BSONObj AggregateKeyGenerator::_makeQueryStatsKeyHelper(
- const SerializationOptions& opts,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const Pipeline& pipeline) const {
- return generateWithQueryShape(
- query_shape::extractQueryShape(_request, pipeline, opts, expCtx, _origNss), opts);
-}
-
-namespace {
-
-int64_t sum(const std::initializer_list<int64_t>& sizes) {
- return std::accumulate(sizes.begin(), sizes.end(), 0, std::plus{});
-}
-
-int64_t size(const std::vector<BSONObj>& objects) {
- return std::accumulate(objects.begin(), objects.end(), 0, [](int64_t total, const auto& obj) {
- // Include the 'sizeof' to account for the variable number in the vector.
- return total + sizeof(BSONObj) + obj.objsize();
- });
-}
-
-int64_t size(const boost::optional<PassthroughToShardOptions>& passthroughToShardOpts) {
- if (!passthroughToShardOpts) {
- return 0;
- }
- return passthroughToShardOpts->getShard().size();
-}
-
-int64_t size(const boost::optional<ExchangeSpec>& exchange) {
- if (!exchange) {
- return 0;
- }
- return sum(
- {exchange->getKey().objsize(),
- (exchange->getBoundaries() ? size(exchange->getBoundaries().get()) : 0),
- (exchange->getConsumerIds() ? 4 * static_cast<int64_t>(exchange->getConsumerIds()->size())
- : 0)});
-}
-
-int64_t size(const boost::optional<EncryptionInformation>& encryptInfo) {
- if (!encryptInfo) {
- return 0;
- }
- tasserted(7659700,
- "Unexpected encryption information - not expecting to collect query shape stats on "
- "encrypted querys");
-}
-
-int64_t size(const StringData& str) {
- return str.size();
-}
-
-int64_t size(const boost::optional<BSONObj>& obj) {
- return optionalObjSize(obj);
-}
-
-// variadic base case.
-template <typename T>
-int64_t sumOfSizes(const T& t) {
- return size(t);
-}
-
-// variadic recursive case. Making the compiler expand the pluses everywhere to give us good
-// formatting at the call site. sumOfSizes(x, y, z) rather than size(x) + size(y) + size(z).
-template <typename T, typename... Args>
-int64_t sumOfSizes(const T& t, const Args&... args) {
- return size(t) + sumOfSizes(args...);
-}
-
-int64_t aggRequestSize(const AggregateCommandRequest& request) {
- return sumOfSizes(request.getPipeline(),
- request.getLet(),
- request.getUnwrappedReadPref(),
- request.getExchange(),
- request.getPassthroughToShard(),
- request.getEncryptionInformation(),
- request.getDbName());
-}
-
-} // namespace
-
-int64_t AggregateKeyGenerator::doGetSize() const {
- return sum({sizeof(*this),
- static_cast<int64_t>(_origNss.size()),
- optionalObjSize(_initialQueryStatsKey),
- aggRequestSize(_request)});
-}
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/find_key.cpp b/src/mongo/db/query/query_stats/find_key.cpp
deleted file mode 100644
index 437c75aecaa..00000000000
--- a/src/mongo/db/query/query_stats/find_key.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_stats/find_key.h"
-
-namespace mongo::query_stats {
-
-void FindCmdComponents::appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const {
-
- if (_hasField.allowPartialResults) {
- bob.append(FindCommandRequest::kAllowPartialResultsFieldName, _allowPartialResults);
- }
-
- // Fields for literal redaction. Adds batchSize, and noCursorTimeOut.
-
- if (_hasField.noCursorTimeout) {
- bob.append(FindCommandRequest::kNoCursorTimeoutFieldName, _noCursorTimeout);
- }
-
- // We don't store the specified batch size value since it doesn't matter.
- // Provide an arbitrary literal long here.
- tassert(7973602,
- "Serialization policy not supported - original values have been discarded",
- opts.literalPolicy != LiteralSerializationPolicy::kUnchanged);
-
- if (_hasField.batchSize) {
- opts.appendLiteral(&bob, FindCommandRequest::kBatchSizeFieldName, 0ll);
- }
-}
-
-std::unique_ptr<FindCommandRequest> FindKey::reparse(OperationContext* opCtx) const {
- auto fcr =
- static_cast<const query_shape::FindCmdShape*>(universalComponents()._queryShape.get())
- ->toFindCommandRequest();
- if (_components._hasField.allowPartialResults)
- fcr->setAllowPartialResults(_components._allowPartialResults);
- if (_components._hasField.noCursorTimeout)
- fcr->setNoCursorTimeout(_components._noCursorTimeout);
- if (_components._hasField.batchSize)
- fcr->setBatchSize(1ll);
- return fcr;
-}
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/find_key.h b/src/mongo/db/query/query_stats/find_key.h
deleted file mode 100644
index 8578a77e573..00000000000
--- a/src/mongo/db/query/query_stats/find_key.h
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include <memory>
-
-#include "mongo/db/namespace_string.h"
-#include "mongo/db/operation_context.h"
-#include "mongo/db/query/query_shape/find_cmd_shape.h"
-#include "mongo/db/query/query_stats/key.h"
-
-namespace mongo::query_stats {
-
-struct FindCmdComponents : public SpecificKeyComponents {
- FindCmdComponents(const FindCommandRequest* findCmd)
- : _allowPartialResults(findCmd->getAllowPartialResults().value_or(false)),
- _noCursorTimeout(findCmd->getNoCursorTimeout().value_or(false)),
- _hasField() {
- _hasField.batchSize = findCmd->getBatchSize().has_value();
- _hasField.allowPartialResults = findCmd->getAllowPartialResults().has_value();
- _hasField.noCursorTimeout = findCmd->getNoCursorTimeout().has_value();
- }
-
- std::size_t size() const {
- return sizeof(FindCmdComponents);
- }
-
- void HashValue(absl::HashState state) const final {
- absl::HashState::combine(
- std::move(state), _hasField, _allowPartialResults, _noCursorTimeout);
- }
-
- void appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const;
-
- // Avoid using boost::optional here because it creates extra padding at the beginning of the
- // struct. Since each QueryStatsEntry can have its own FindKey, it's better to
- // minimize the struct's size as much as possible.
-
- // Preserved literal.
- bool _allowPartialResults;
- bool _noCursorTimeout;
-
- // This anonymous struct represents the presence of the member variables as C++ bit fields.
- // In doing so, each of these boolean values takes up 1 bit instead of 1 byte.
- struct HasField {
- HasField() : batchSize(false), allowPartialResults(false), noCursorTimeout(false) {}
- bool batchSize : 1;
- bool allowPartialResults : 1;
- bool noCursorTimeout : 1;
- bool operator==(const HasField& other) const {
- return batchSize == other.batchSize &&
- allowPartialResults == other.allowPartialResults &&
- noCursorTimeout == other.noCursorTimeout;
- }
-
- } _hasField;
-
- template <typename H>
- friend H AbslHashValue(H h, const HasField& hasField) {
- return H::combine(std::move(h),
- hasField.batchSize,
- hasField.noCursorTimeout,
- hasField.allowPartialResults);
- }
-};
-
-// This static assert checks to ensure that the struct's size is changed thoughtfully. If adding
-// or otherwise changing the members, this assert may be updated with care.
-static_assert(
- // Expecting two bytes for allowPartialResults and noCursorTimeout, and another
- // byte for _hasField. For alignment reasons (alignment is 8 bytes here), this means the trailer
- // will bring up the total bytecount to a multiple of 8.
- sizeof(FindCmdComponents) <= sizeof(SpecificKeyComponents) + 8,
- "Size of FindCmdComponents is too large! "
- "Make sure that the struct has been align- and padding-optimized. "
- "If the struct's members have changed, this assert may need to be updated with a new "
- "value.");
-
-class FindKey final : public Key {
-public:
- FindKey(const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const ParsedFindCommand& request,
- query_shape::CollectionType collectionType = query_shape::CollectionType::kUnknown)
- : Key(expCtx->opCtx,
- std::make_unique<query_shape::FindCmdShape>(request, expCtx),
- request.findCommandRequest->getHint(),
- request.findCommandRequest->getReadConcern(),
- request.findCommandRequest->getMaxTimeMS().has_value(),
- collectionType),
- _components(request.findCommandRequest.get()) {}
-
- // The default implementation of hashing for smart pointers is not a good one for our purposes.
- // Here we overload them to actually take the hash of the object, rather than hashing the
- // pointer itself.
- template <typename H>
- friend H AbslHashValue(H h, const std::unique_ptr<const FindKey>& key) {
- return H::combine(std::move(h), *key);
- }
- template <typename H>
- friend H AbslHashValue(H h, const std::shared_ptr<const FindKey>& key) {
- return H::combine(std::move(h), *key);
- }
-
- const SpecificKeyComponents& specificComponents() const {
- return _components;
- }
-
-private:
- void appendCommandSpecificComponents(BSONObjBuilder& bob,
- const SerializationOptions& opts) const final {
- _components.appendTo(bob, opts);
- }
-
- std::unique_ptr<FindCommandRequest> reparse(OperationContext* opCtx) const;
-
- FindCmdComponents _components;
-};
-
-// This static assert checks to ensure that the struct's size is changed thoughtfully. If adding
-// or otherwise changing the members, this assert may be updated with care.
-static_assert(sizeof(FindKey) == sizeof(Key) + sizeof(FindCmdComponents),
- "If the class' members have changed, this assert may need to be updated with a new "
- "value and the size calcuation will need to be changed.");
-
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/find_key_test.cpp b/src/mongo/db/query/query_stats/find_key_test.cpp
deleted file mode 100644
index 6c34ba5a606..00000000000
--- a/src/mongo/db/query/query_stats/find_key_test.cpp
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/parsed_find_command.h"
-#include "mongo/db/query/query_stats/find_key.h"
-#include "mongo/db/service_context_test_fixture.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo::query_stats {
-
-namespace {
-static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl");
-
-static constexpr auto collectionType = query_shape::CollectionType::kCollection;
-
-class FindKeyTest : public ServiceContextTest {
-public:
- static std::unique_ptr<const Key> makeFindKeyFromQuery(const BSONObj& filter) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
- auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcr->setFilter(filter.getOwned());
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcr)}));
- return std::make_unique<FindKey>(expCtx, *parsedFind, collectionType);
- }
-};
-
-TEST_F(FindKeyTest, SizeOfFindCmdComponents) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
-
- auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- auto query = BSON("query" << 1 << "xEquals" << 42);
- fcr->setFilter(query.getOwned());
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcr)}));
- auto findComponents = std::make_unique<FindCmdComponents>(parsedFind->findCommandRequest.get());
-
- ASSERT_GTE(findComponents->size(), sizeof(SpecificKeyComponents) + 3 /*bools and HasField*/);
- ASSERT_LTE(findComponents->size(),
- sizeof(SpecificKeyComponents) + 8 /*bools, HasField, and padding*/);
-}
-
-TEST_F(FindKeyTest, EquivalentFindCmdComponentsSizes) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
- auto query = BSON("query" << 1 << "xEquals" << 42);
-
- // Set different fields in the find commands.
- auto fcrCursorTimeout = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcrCursorTimeout->setFilter(query.getOwned());
- fcrCursorTimeout->setNoCursorTimeout(true);
- auto parsedFindCursorTimeout =
- uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrCursorTimeout)}));
- auto findComponentsCursorTimeout =
- std::make_unique<FindCmdComponents>(parsedFindCursorTimeout->findCommandRequest.get());
-
- auto fcrAllowPartial = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcrAllowPartial->setFilter(query.getOwned());
- fcrAllowPartial->setAllowPartialResults(true);
- auto parsedFindAllowPartial =
- uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrAllowPartial)}));
- auto findComponentsAllowPartial =
- std::make_unique<FindCmdComponents>(parsedFindAllowPartial->findCommandRequest.get());
-
- ASSERT_EQ(findComponentsCursorTimeout->size(), findComponentsAllowPartial->size());
-}
-
-// Testing item from opCtx that should impact key size.
-TEST_F(FindKeyTest, SizeOfFindKeyWithAndWithoutComment) {
- auto query = BSON("query" << 1 << "xEquals" << 42);
-
- auto keyWithoutComment = makeFindKeyFromQuery(query);
-
- auto opCtx = makeOperationContext();
- auto fcrWithComment = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcrWithComment->setFilter(query.getOwned());
- opCtx->setComment(BSON("comment"
- << " foo"));
- auto expCtxWithComment = make_intrusive<ExpressionContext>(
- opCtx.get(), *fcrWithComment, nullptr, true /* mayDbProfile*/);
- auto parsedFindWithComment =
- uassertStatusOK(parsed_find_command::parse(expCtxWithComment, {std::move(fcrWithComment)}));
- auto keyWithComment = std::make_unique<query_stats::FindKey>(
- expCtxWithComment, *parsedFindWithComment, collectionType);
-
- ASSERT_LT(keyWithoutComment->size(), keyWithComment->size());
-}
-
-// Testing item from command request that should impact key size.
-TEST_F(FindKeyTest, SizeOfFindKeyWithAndWithoutReadConcern) {
- auto query = BSON("query" << 1 << "xEquals" << 42);
-
- auto keyWithoutReadConcern = makeFindKeyFromQuery(query);
-
- auto expCtxWithReadConcern = make_intrusive<ExpressionContextForTest>();
- auto fcrWithReadConcern = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcrWithReadConcern->setFilter(query.getOwned());
- fcrWithReadConcern->setReadConcern(fromjson(R"({level: "local"})"));
- auto parsedFindWithReadConcern = uassertStatusOK(
- parsed_find_command::parse(expCtxWithReadConcern, {std::move(fcrWithReadConcern)}));
- auto keyWithReadConcern = std::make_unique<query_stats::FindKey>(
- expCtxWithReadConcern, *parsedFindWithReadConcern, collectionType);
-
- ASSERT_LT(keyWithoutReadConcern->size(), keyWithReadConcern->size());
-}
-
-
-} // namespace
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/key.cpp b/src/mongo/db/query/query_stats/key.cpp
deleted file mode 100644
index f282ef21a2a..00000000000
--- a/src/mongo/db/query/query_stats/key.cpp
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_stats/key.h"
-
-#include "mongo/db/query/query_stats/query_stats_helpers.h"
-#include "mongo/rpc/metadata/client_metadata.h"
-
-namespace mongo::query_stats {
-
-namespace {
-
-BSONObj scrubHighCardinalityFields(const ClientMetadata* clientMetadata) {
- if (!clientMetadata) {
- return BSONObj();
- }
- return clientMetadata->documentWithoutMongosInfo();
-}
-
-BSONObj shapifyReadPreference(boost::optional<BSONObj> readPreference) {
- if (!readPreference) {
- return BSONObj();
- }
-
- BSONObjBuilder builder;
- for (const auto& elem : *readPreference) {
- if (elem.fieldNameStringData() != "tags"_sd) {
- builder.append(elem);
- continue;
- }
-
- // Sort the $readPreference tags so that different orderings still map to one query stats
- // store key.
- BSONObjSet sortedTags = SimpleBSONObjComparator::kInstance.makeBSONObjSet();
- for (const auto& tag : elem.Array()) {
- sortedTags.insert(tag.Obj());
- }
-
- BSONArrayBuilder arrBuilder(builder.subarrayStart("tags"_sd));
- for (const auto& tag : sortedTags) {
- arrBuilder.append(tag);
- }
- }
- return builder.obj();
-}
-
-} // namespace
-
-UniversalKeyComponents::UniversalKeyComponents(std::unique_ptr<query_shape::Shape> queryShape,
- const ClientMetadata* clientMetadata,
- boost::optional<BSONObj> commentObj,
- boost::optional<BSONObj> hint,
- boost::optional<BSONObj> readPreference,
- boost::optional<BSONObj> writeConcern,
- boost::optional<BSONObj> readConcern,
- std::unique_ptr<APIParameters> apiParams,
- query_shape::CollectionType collectionType,
- bool maxTimeMS)
- : _clientMetaData(scrubHighCardinalityFields(clientMetadata)),
- _commentObj(commentObj.value_or(BSONObj()).getOwned()),
- _hintObj(hint.value_or(BSONObj()).getOwned()),
- _writeConcern(writeConcern.value_or(BSONObj()).getOwned()),
- _shapifiedReadPreference(shapifyReadPreference(readPreference)),
- _shapifiedReadConcern(shapifyReadConcern(readConcern.value_or(BSONObj()))),
- _comment(commentObj ? _commentObj.firstElement() : BSONElement()),
- _queryShape(std::move(queryShape)),
- _apiParams(std::move(apiParams)),
- _clientMetaDataHash(clientMetadata ? clientMetadata->hashWithoutMongosInfo()
- : simpleHash(BSONObj())),
- _collectionType(collectionType),
- _hasField() {
- _hasField.clientMetaData = bool(clientMetadata);
- _hasField.comment = bool(commentObj);
- _hasField.hint = bool(hint);
- _hasField.readPreference = bool(readPreference);
- _hasField.writeConcern = bool(writeConcern);
- _hasField.readConcern = bool(readConcern);
- _hasField.maxTimeMS = maxTimeMS;
- tassert(7973600, "shape must not be null", _queryShape);
-}
-
-BSONObj UniversalKeyComponents::shapifyReadConcern(const BSONObj& readConcern,
- const SerializationOptions& opts) {
- // Read concern should not be considered a literal.
- // afterClusterTime is distinct for every operation with causal consistency enabled. We
- // normalize it in order not to blow out the queryStats store cache.
- if (readConcern["afterClusterTime"].eoo() && readConcern["atClusterTime"].eoo()) {
- return readConcern.copy();
- } else {
- BSONObjBuilder bob;
-
- if (auto levelElem = readConcern["level"]) {
- bob.append(levelElem);
- }
- if (auto afterClusterTime = readConcern["afterClusterTime"]) {
- opts.appendLiteral(&bob, "afterClusterTime", afterClusterTime);
- }
- if (auto atClusterTime = readConcern["atClusterTime"]) {
- opts.appendLiteral(&bob, "atClusterTime", atClusterTime);
- }
- return bob.obj();
- }
-}
-
-size_t UniversalKeyComponents::size() const {
- return sizeof(*this) + _queryShape->size() +
- (_apiParams ? sizeof(*_apiParams) + shape_helpers::optionalSize(_apiParams->getAPIVersion())
- : 0) +
- _hintObj.objsize() + (_hasField.clientMetaData ? _clientMetaData.objsize() : 0) +
- _commentObj.objsize() +
- (_hasField.readPreference ? _shapifiedReadPreference.objsize() : 0) +
- (_hasField.readConcern ? _shapifiedReadConcern.objsize() : 0) +
- (_hasField.writeConcern ? _writeConcern.objsize() : 0);
-}
-
-void UniversalKeyComponents::appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const {
- if (_hasField.comment) {
- opts.appendLiteral(&bob, "comment", _comment);
- }
-
- if (_hasField.readConcern) {
- auto readConcernToAppend = _shapifiedReadConcern;
- if (opts != SerializationOptions::kRepresentativeQueryShapeSerializeOptions) {
- // The options aren't the same as the first time we shapified, so re-computation is
- // necessary (e.g. use "?timestamp" instead of the representative Timestamp(0, 0)).
- readConcernToAppend = shapifyReadConcern(_shapifiedReadConcern, opts);
- }
- bob.append("readConcern", readConcernToAppend);
- }
-
- if (const auto& apiVersion = _apiParams->getAPIVersion()) {
- bob.append("apiVersion", apiVersion.value());
- }
-
- if (const auto& apiStrict = _apiParams->getAPIStrict()) {
- bob.append("apiStrict", apiStrict.value());
- }
-
- if (const auto& apiDeprecationErrors = _apiParams->getAPIDeprecationErrors()) {
- bob.append("apiDeprecationErrors", apiDeprecationErrors.value());
- }
-
- if (_hasField.readPreference) {
- bob.append("$readPreference", _shapifiedReadPreference);
- }
-
- if (_hasField.writeConcern) {
- bob.append("writeConcern", _writeConcern);
- }
-
- if (_hasField.clientMetaData) {
- bob.append("client", _clientMetaData);
- }
- if (_collectionType > query_shape::CollectionType::kUnknown) {
- bob.append("collectionType", toStringData(_collectionType));
- }
- if (!_hintObj.isEmpty()) {
- bob.append("hint", shape_helpers::extractHintShape(_hintObj, opts));
- }
- if (_hasField.maxTimeMS) {
- opts.appendLiteral(&bob, "maxTimeMS", 0ll);
- }
-}
-Key::Key(OperationContext* opCtx,
- std::unique_ptr<query_shape::Shape> queryShape,
- boost::optional<BSONObj> hint,
- boost::optional<BSONObj> readConcern,
- bool maxTimeMS,
- query_shape::CollectionType collectionType)
- : _universalComponents(
- std::move(queryShape),
- ClientMetadata::get(opCtx->getClient()),
- opCtx->getCommentOwnedCopy(),
- hint,
- ReadPreferenceSetting::get(opCtx).usedDefaultReadPrefValue()
- ? boost::none
- : boost::make_optional(ReadPreferenceSetting::get(opCtx).toInnerBSON()),
- opCtx->getWriteConcern().isImplicitDefaultWriteConcern()
- ? boost::none
- : boost::make_optional(opCtx->getWriteConcern().toBSON()),
- readConcern,
- std::make_unique<APIParameters>(APIParameters::get(opCtx)),
- collectionType,
- maxTimeMS) {}
-
-BSONObj Key::toBson(OperationContext* opCtx, const SerializationOptions& opts) const {
- BSONObjBuilder bob;
-
- // We'll take care of appending this one outside of the appendTo() call below since it needs
- // an OperationContext in some re-parsing cases. The rest is simpler.
- bob.append("queryShape", _universalComponents._queryShape->toBson(opCtx, opts));
-
- _universalComponents.appendTo(bob, opts);
- appendCommandSpecificComponents(bob, opts);
- return bob.obj();
-}
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/key.h b/src/mongo/db/query/query_stats/key.h
deleted file mode 100644
index bb83fca82ed..00000000000
--- a/src/mongo/db/query/query_stats/key.h
+++ /dev/null
@@ -1,304 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include <memory>
-
-#include "mongo/bson/bsonobj.h"
-#include "mongo/db/api_parameters.h"
-#include "mongo/db/collection_type.h"
-#include "mongo/db/pipeline/expression_context.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
-#include "mongo/db/query/query_shape/shape_helpers.h"
-#include "mongo/db/query/query_stats/transform_algorithm_gen.h"
-#include "mongo/rpc/metadata/client_metadata.h"
-
-namespace mongo::query_stats {
-
-/**
- * A struct holding pieces of the command request that are a component of the query stats store key
- * and are options/arguments to all supported query stats commands.
- *
- * This struct (and the SpecificKeyComponents) are split out as a separate inheritence hierarchy to
- * make it easier to ensure each piece is hashed without sub-classes needing to enumerate the parent
- * class's member variables.
- */
-struct UniversalKeyComponents {
- UniversalKeyComponents(std::unique_ptr<query_shape::Shape> queryShape,
- const ClientMetadata* clientMetadata,
- boost::optional<BSONObj> commentObj,
- boost::optional<BSONObj> hint,
- boost::optional<BSONObj> readPreference,
- boost::optional<BSONObj> writeConcern,
- boost::optional<BSONObj> readConcern,
- std::unique_ptr<APIParameters> apiParams,
- query_shape::CollectionType collectionType,
- bool maxTimeMS);
- /**
- * Returns a copy of the read concern object. If there is an "afterClusterTime" or
- * "atClusterTime" component, the timestamp is shapified according to 'opts'.
- */
- static BSONObj shapifyReadConcern(
- const BSONObj& readConcern,
- const SerializationOptions& opts =
- SerializationOptions::kRepresentativeQueryShapeSerializeOptions);
-
- size_t size() const;
-
- void appendTo(BSONObjBuilder& bob, const SerializationOptions& opts) const;
-
- // Avoid using boost::optional here because it creates extra padding at the beginning of the
- // struct. Since each QueryStatsEntry has its own Key subclass, it's better to minimize
- // the struct's size as much as possible.
-
- BSONObj _clientMetaData; // Preserve this value.
- BSONObj _commentObj; // Shapify this value.
- BSONObj _hintObj; // Preserve this value.
- BSONObj _writeConcern; // Preserve this value.
-
- // Preserved literal except value of 'tags' field is sorted.
- BSONObj _shapifiedReadPreference;
- // Preserved literal except 'afterClusterTime' and 'atClusterTime' are shapified.
- BSONObj _shapifiedReadConcern;
-
- // Separate the possibly-enormous BSONObj from the remaining members
-
- BSONElement _comment;
-
- std::unique_ptr<query_shape::Shape> _queryShape;
- std::unique_ptr<APIParameters> _apiParams; // Preserve this value in the query shape.
-
- // Simple hash of the client metadata object. This value is stored separately because it is
- // cached on the client to avoid re-computing on every operation. If no client metadata is
- // present, this will be the hash of an empty BSON object (otherwise known as 0).
- const unsigned long _clientMetaDataHash;
-
- // This value is not known when run a query is run on mongos over an unsharded collection, so it
- // is not set through that code path.
- query_shape::CollectionType _collectionType;
-
- // This anonymous struct represents the presence of the member variables as C++ bit fields.
- // In doing so, each of these boolean values takes up 1 bit instead of 1 byte.
- struct HasField {
- HasField()
- : clientMetaData(false),
- comment(false),
- hint(false),
- readPreference(false),
- writeConcern(false),
- readConcern(false),
- maxTimeMS(false) {}
-
- bool clientMetaData : 1;
- bool comment : 1;
- bool hint : 1;
- bool readPreference : 1;
- bool writeConcern : 1;
- bool readConcern : 1;
- bool maxTimeMS : 1;
- } _hasField;
-};
-
-/**
- * A base class for sub-classes to derive from to expose the hashing ability for all of their
- * sub-components.
- *
- * This struct (and the UniversalKeyComponents) are split out as a separate inheritence hierarchy to
- * make it easier to ensure each piece is hashed without sub-classes needing to enumerate the parent
- * class's member variables.
- */
-struct SpecificKeyComponents {
- virtual ~SpecificKeyComponents() {}
-
- virtual void HashValue(absl::HashState state) const = 0;
-
- /**
- * Sub-classes should implement this to report how much memory is used. This is important to do
- * carefully since we are under a budget in the query stats store and use this to do the
- * accounting. Implementers should include sizeof(*derivedThis) and be sure to also include the
- * size of any owned pointer-like objects such as BSONObj or NamespaceString which are
- * indirectly using memory elsehwhere.
- *
- * We cannot just use sizeof() because there are some variable size data members (like BSON
- * objects) which depend on the particular instance.
- */
- virtual size_t size() const = 0;
-};
-
-template <typename H>
-H AbslHashValue(H state, const SpecificKeyComponents& value) {
- value.HashValue(absl::HashState::Create(&state));
- return std::move(state);
-}
-
-template <typename H>
-H AbslHashValue(H h, const UniversalKeyComponents& components) {
- return H::combine(std::move(h),
- *components._queryShape,
- components._clientMetaDataHash,
- // Note we use the comment's type in the hash function.
- components._comment.type(),
- simpleHash(components._hintObj),
- simpleHash(components._shapifiedReadPreference),
- simpleHash(components._writeConcern),
- simpleHash(components._shapifiedReadConcern),
- components._apiParams ? APIParameters::Hash{}(*components._apiParams) : 0,
- components._collectionType,
- components._hasField);
-}
-
-template <typename H>
-H AbslHashValue(H h, const UniversalKeyComponents::HasField& hasField) {
- return H::combine(std::move(h),
- hasField.clientMetaData,
- hasField.comment,
- hasField.hint,
- hasField.readPreference,
- hasField.writeConcern,
- hasField.readConcern,
- hasField.maxTimeMS);
-}
-
-
-// This static assert checks to ensure that the struct's size is changed thoughtfully. If adding
-// or otherwise changing the members, this assert may be updated with care.
-static_assert(
- sizeof(UniversalKeyComponents) <= sizeof(query_shape::Shape) + 6 * sizeof(BSONObj) +
- sizeof(BSONElement) + sizeof(std::unique_ptr<APIParameters>) +
- sizeof(query_shape::CollectionType) + sizeof(query_shape::QueryShapeHash) +
- sizeof(int64_t),
- "Size of Key is too large! "
- "Make sure that the struct has been align- and padding-optimized. "
- "If the struct's members have changed, this assert may need to be updated with a new value.");
-
-/**
- * An abstract base class representing a query stats store key for a given request. All query stats
- * store entries should include some common elements, tracked in `_universalComponents`. For
- * example, everything tracked must have a `query_shape::Shape`.
- *
- * Subclasses can add more components to include as discriminating factors in which entries should
- * be tracked separately. For example, two find commands which are identical except in their read
- * concern should be tracked differently. Maybe they will have quite different performance
- * characteristics or help us determine when the read concern was changed by the client.
- *
- * The interface to do this is to split out the state/memory for these components as a separate
- * struct which can indpendently hash itself and compute its size (both of which are important for
- * the query stats store). Subclasses of Key itself should not have any meaningfully sized
- * state other than the 'specificComponents().'
- */
-class Key {
-public:
- virtual ~Key() = default;
-
- /**
- * All Keys will share these characteristics as part of their query stats store key.
- * Returns an unowned reference so the caller must ensure the result does not outlive this
- * Key instance.
- */
- const auto& universalComponents() const {
- return _universalComponents;
- }
-
- /**
- * Different commands will have different components they want to be included in the query stats
- * store key. This interface allows them to do so and easily have those components incorporated
- * into this key generation and hashing.
- */
- virtual const SpecificKeyComponents& specificComponents() const = 0;
-
- /**
- * Materializes the query stats store key. Not expected to be used on ingestion, since we should
- * store this object and its components directly in their native C++ data structures - we can
- * use the absl::Hash<query_stats::Key>{}() API to look them up. Instead, this may be useful to
- * display the key (as it is used for $queryStats) or perhaps one day persist it to storage.
- */
- BSONObj toBson(OperationContext* opCtx, const SerializationOptions& opts) const;
-
- /**
- * Convenience function.
- */
- query_shape::QueryShapeHash getQueryShapeHash(OperationContext* opCtx) const {
- // TODO (future ticket?) should we cache this somewhere else?
- return _universalComponents._queryShape->sha256Hash(opCtx);
- }
-
- size_t size() const {
- return sizeof(Key) + specificComponents().size() + _universalComponents.size();
- }
-
- template <typename H>
- friend H AbslHashValue(H h, const Key& key) {
- return H::combine(std::move(h), key._universalComponents, key.specificComponents());
- }
-
- // The default implementation of hashing for smart pointers is not a good one for our purposes.
- // Here we overload them to actually take the hash of the object, rather than hashing the
- // pointer itself.
- template <typename H>
- friend H AbslHashValue(H h, const std::unique_ptr<const Key>& key) {
- return H::combine(std::move(h), *key);
- }
- template <typename H>
- friend H AbslHashValue(H h, const std::shared_ptr<const Key>& key) {
- return H::combine(std::move(h), *key);
- }
-
-protected:
- /**
- * Sub-classes can use this to instantiate a 'real' Key. 'queryShape' must not be null,
- * but is tracked as a pointer since it is a virtual class and we want to own it here.
- */
- Key(OperationContext* opCtx,
- std::unique_ptr<query_shape::Shape> queryShape,
- boost::optional<BSONObj> hint,
- boost::optional<BSONObj> readConcern,
- bool maxTimeMS,
- query_shape::CollectionType collectionType = query_shape::CollectionType::kUnknown);
-
- /**
- * With a given BSONObjBuilder, append the command-specific components of the query stats key.
- *
- * You may be wondering why this API is here rather than as a virtual method on
- * CmdSpecificComponents - and that would be because many implementations can involve a re-parse
- * of the request if it needs to serialize with different serialization options. This re-parsing
- * process often needs the context of things tracked in _universalComponents, which is hard to
- * access from the specific components.
- */
- virtual void appendCommandSpecificComponents(BSONObjBuilder& bob,
- const SerializationOptions& opts) const = 0;
-
-private:
- UniversalKeyComponents _universalComponents;
-};
-static_assert(
- sizeof(Key) == sizeof(void*) /*vtable ptr*/ + sizeof(UniversalKeyComponents),
- "If the class' members have changed, this assert may need to be updated with a new value.");
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/key_test.cpp b/src/mongo/db/query/query_stats/key_test.cpp
deleted file mode 100644
index 69359d08bda..00000000000
--- a/src/mongo/db/query/query_stats/key_test.cpp
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Copyright (C) 2024-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/bson/bsonelement.h"
-#include "mongo/db/collection_type.h"
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/parsed_find_command.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_stats/find_key.h"
-#include "mongo/db/query/query_stats/key.h"
-#include "mongo/db/service_context_test_fixture.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo::query_stats {
-
-namespace {
-static const NamespaceString kDefaultTestNss = NamespaceString("testDB.testColl");
-
-
-struct DummyShapeSpecificComponents : public query_shape::CmdSpecificShapeComponents {
- DummyShapeSpecificComponents(){};
- void HashValue(absl::HashState state) const {}
- size_t size() const final {
- return sizeof(DummyShapeSpecificComponents);
- }
-};
-
-class DummyShape : public query_shape::Shape {
-public:
- DummyShape(NamespaceStringOrUUID nssOrUUID,
- BSONObj collation,
- DummyShapeSpecificComponents dummyComponents)
- : Shape(nssOrUUID, collation) {
- components = dummyComponents;
- }
-
- const query_shape::CmdSpecificShapeComponents& specificComponents() const final {
- return components;
- }
-
- void appendCmdSpecificShapeComponents(BSONObjBuilder&,
- OperationContext*,
- const SerializationOptions& opts) const final {}
- DummyShapeSpecificComponents components;
-};
-
-struct DummyKeyComponents : public SpecificKeyComponents {
- DummyKeyComponents(){};
-
- void HashValue(absl::HashState state) const {}
- size_t size() const {
- return sizeof(DummyKeyComponents);
- }
-};
-
-class DummyKey : public Key {
-public:
- DummyKey(OperationContext* opCtx,
- std::unique_ptr<query_shape::Shape> queryShape,
- boost::optional<BSONObj> hint,
- boost::optional<BSONObj> readConcern,
- bool maxTimeMS,
- query_shape::CollectionType collectionType,
- DummyKeyComponents dummyComponents)
- : Key(opCtx, std::move(queryShape), hint, readConcern, maxTimeMS, collectionType) {
- components = dummyComponents;
- }
- const SpecificKeyComponents& specificComponents() const {
- return components;
- };
- void appendCommandSpecificComponents(BSONObjBuilder& bob,
- const SerializationOptions& opts) const {};
- DummyKeyComponents components;
-};
-class UniversalKeyTest : public ServiceContextTest {};
-
-TEST_F(UniversalKeyTest, SizeOfUniversalComponents) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
-
- // Make shape for testing.
- auto collation = BSONObj{};
- auto innerComponents = std::make_unique<DummyShapeSpecificComponents>();
- auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents);
-
- // Gather sizes and create universalComponents.
- const auto shapeSize = shape->size();
- auto clientMetadata = ClientMetadata::get(expCtx->opCtx->getClient());
-
- auto clientMetadataSize = clientMetadata ? clientMetadata->documentWithoutMongosInfo().objsize()
- : BSONObj().objsize();
-
- auto apiParams = std::make_unique<APIParameters>(APIParameters::get(expCtx->opCtx));
- const auto apiParamsSize = static_cast<size_t>(
- apiParams ? sizeof(*apiParams) + shape_helpers::optionalSize(apiParams->getAPIVersion())
- : 0);
- auto universalComponents =
- std::make_unique<UniversalKeyComponents>(std::move(shape),
- clientMetadata,
- BSONObj(),
- BSONObj(),
- BSONObj(),
- BSONObj(),
- BSONObj(),
- std::move(apiParams),
- query_shape::CollectionType::kUnknown,
- true);
-
- const auto minimumUniversalKeyComponentSize = sizeof(std::unique_ptr<query_shape::Shape>) +
- (6 * sizeof(BSONObj)) + sizeof(std::unique_ptr<APIParameters>) + sizeof(BSONElement) +
- sizeof(query_shape::CollectionType) + sizeof(unsigned long) + 1 /*HasField*/;
- ASSERT_GTE(sizeof(UniversalKeyComponents), minimumUniversalKeyComponentSize);
- ASSERT_LTE(sizeof(UniversalKeyComponents), minimumUniversalKeyComponentSize + 8 /*padding*/);
-
- ASSERT_GT(universalComponents->size(),
- sizeof(UniversalKeyComponents) + shapeSize + clientMetadataSize + apiParamsSize);
- ASSERT_LTE(universalComponents->size(),
- sizeof(UniversalKeyComponents) + shapeSize + clientMetadataSize +
- (5 * static_cast<size_t>(BSONObj().objsize())) + apiParamsSize);
-}
-
-TEST_F(UniversalKeyTest, SizeOfSpecificComponents) {
- auto innerComponents = std::make_unique<DummyShapeSpecificComponents>();
- auto keyComponents = std::make_unique<DummyKeyComponents>();
-
- ASSERT_EQ(keyComponents->size(), sizeof(SpecificKeyComponents));
- ASSERT_EQ(sizeof(SpecificKeyComponents), sizeof(void*) /*vtable ptr*/);
-}
-
-TEST_F(UniversalKeyTest, SizeOfKey) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
-
- auto collation = BSONObj{};
- auto innerComponents = std::make_unique<DummyShapeSpecificComponents>();
- auto shape = std::make_unique<DummyShape>(kDefaultTestNss, collation, *innerComponents);
-
- auto keyComponents = std::make_unique<DummyKeyComponents>();
-
- auto key = std::make_unique<DummyKey>(expCtx->opCtx,
- std::move(shape),
- BSONObj(),
- BSONObj(),
- false,
- query_shape::CollectionType::kUnknown,
- *keyComponents);
- ASSERT_EQ(innerComponents->size(), key->specificComponents().size());
- ASSERT_EQ(sizeof(Key), sizeof(UniversalKeyComponents) + sizeof(void*));
- ASSERT_EQ(key->size(),
- sizeof(Key) + key->universalComponents().size() + key->specificComponents().size());
-}
-} // namespace
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/query_stats.cpp b/src/mongo/db/query/query_stats/query_stats.cpp
deleted file mode 100644
index 2e6393aab34..00000000000
--- a/src/mongo/db/query/query_stats/query_stats.cpp
+++ /dev/null
@@ -1,441 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQueryStats
-
-#include "mongo/db/query/query_stats/query_stats.h"
-
-#include "mongo/crypto/hash_block.h"
-#include "mongo/db/concurrency/d_concurrency.h"
-#include "mongo/db/concurrency/locker.h"
-#include "mongo/db/curop.h"
-#include "mongo/db/exec/projection_executor_builder.h"
-#include "mongo/db/namespace_string.h"
-#include "mongo/db/pipeline/aggregate_command_gen.h"
-#include "mongo/db/pipeline/process_interface/stub_mongo_process_interface.h"
-#include "mongo/db/query/find_command_gen.h"
-#include "mongo/db/query/plan_explainer.h"
-#include "mongo/db/query/projection_ast_util.h"
-#include "mongo/db/query/projection_parser.h"
-#include "mongo/db/query/query_feature_flags_gen.h"
-#include "mongo/db/query/query_planner_params.h"
-#include "mongo/db/query/query_request_helper.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
-#include "mongo/db/query/query_stats/query_stats_on_parameter_change.h"
-#include "mongo/db/query/sort_pattern.h"
-#include "mongo/logv2/log.h"
-#include "mongo/rpc/metadata/client_metadata.h"
-#include "mongo/util/assert_util.h"
-#include "mongo/util/debug_util.h"
-#include "mongo/util/processinfo.h"
-#include "mongo/util/system_clock_source.h"
-#include <optional>
-
-namespace mongo::query_stats {
-
-Counter64 queryStatsStoreSizeEstimateBytesMetric;
-ServerStatusMetricField<Counter64> displaySizeEstimateMetric(
- "queryStats.queryStatsStoreSizeEstimateBytes", &queryStatsStoreSizeEstimateBytesMetric);
-
-
-const Decorable<ServiceContext>::Decoration<std::unique_ptr<QueryStatsStoreManager>>
- QueryStatsStoreManager::get =
- ServiceContext::declareDecoration<std::unique_ptr<QueryStatsStoreManager>>();
-
-const Decorable<ServiceContext>::Decoration<std::unique_ptr<RateLimiting>>
- QueryStatsStoreManager::getRateLimiter =
- ServiceContext::declareDecoration<std::unique_ptr<RateLimiting>>();
-
-
-namespace {
-
-Counter64 queryStatsEvictedMetric;
-ServerStatusMetricField<Counter64> displayEvictedMetric("queryStats.numEvicted",
- &queryStatsEvictedMetric);
-Counter64 queryStatsRateLimitedRequestsMetric;
-ServerStatusMetricField<Counter64> displayRateLimitMetric("queryStats.numRateLimitedRequests",
- &queryStatsRateLimitedRequestsMetric);
-Counter64 queryStatsStoreWriteErrorsMetric;
-ServerStatusMetricField<Counter64> displayWriteErrorsMetric(
- "queryStats.numQueryStatsStoreWriteErrors", &queryStatsStoreWriteErrorsMetric);
-
-/**
- * Indicates whether or not query stats is enabled via the feature flag.
- */
-bool isQueryStatsFeatureEnabled() {
- // We need to call isVersionInitialized() first because this could run during startup while the
- // FCV is still uninitialized.
- if (serverGlobalParams.featureCompatibility.isVersionInitialized()) {
- return feature_flags::gFeatureFlagQueryStats.isEnabled(
- serverGlobalParams.featureCompatibility);
- }
- // (Generic FCV reference): This reference is needed to ensure we correctly initialize query
- // stats during startup.
- return feature_flags::gFeatureFlagQueryStats.isEnabledOnVersion(
- multiversion::GenericFCV::kLatest);
-}
-
-/**
- * Cap the queryStats store size.
- */
-size_t capQueryStatsStoreSize(size_t requestedSize) {
- size_t cappedStoreSize = memory_util::capMemorySize(
- requestedSize /*requestedSizeBytes*/, 1 /*maximumSizeGB*/, 25 /*percentTotalSystemMemory*/);
- // If capped size is less than requested size, the queryStats store has been capped at its
- // upper limit.
- if (cappedStoreSize < requestedSize) {
- LOGV2_DEBUG(7106502,
- 1,
- "The queryStats store size has been capped",
- "cappedSize"_attr = cappedStoreSize);
- }
- return cappedStoreSize;
-}
-
-/**
- * Get the queryStats store size based on the query job's value.
- */
-size_t getQueryStatsStoreSize() {
- auto status = memory_util::MemorySize::parse(internalQueryStatsCacheSize.get());
- uassertStatusOK(status);
- size_t requestedSize = memory_util::convertToSizeInBytes(status.getValue());
- return capQueryStatsStoreSize(requestedSize);
-}
-
-void assertConfigurationAllowed() {
- uassert(ErrorCodes::QueryFeatureNotAllowed,
- "Cannot configure queryStats store. The feature flag is not enabled. Please restart "
- "and specify the feature flag, or upgrade the feature compatibility version to one "
- "where it is enabled by default.",
- isQueryStatsFeatureEnabled());
-}
-
-class QueryStatsOnParamChangeUpdaterImpl final : public query_stats_util::OnParamChangeUpdater {
-public:
- void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) final {
- assertConfigurationAllowed();
- auto requestedSize = memory_util::convertToSizeInBytes(memSize);
- auto cappedSize = capQueryStatsStoreSize(requestedSize);
- auto& queryStatsStoreManager = QueryStatsStoreManager::get(serviceCtx);
- size_t numEvicted = queryStatsStoreManager->resetSize(cappedSize);
- queryStatsEvictedMetric.increment(numEvicted);
- }
-
- void updateSamplingRate(ServiceContext* serviceCtx, int samplingRate) {
- assertConfigurationAllowed();
- QueryStatsStoreManager::getRateLimiter(serviceCtx).get()->setSamplingRate(samplingRate);
- }
-};
-
-ServiceContext::ConstructorActionRegisterer queryStatsStoreManagerRegisterer{
- "QueryStatsStoreManagerRegisterer", [](ServiceContext* serviceCtx) {
- // Note: it is possible that this is called before FCV is properly set up. The feature flags
- // can only be specified at startup, but the feature compatibility version may change at
- // runtime. If the feature compatibility version upgrades at runtime, the feature may now be
- // enabled by default, even if the flag was not specified. To allow for this possibility, we
- // will always configure a query stats store of the size currently specified by
- // 'internalQueryStatsCacheSize', but we will prevent changing its shape or rate limit at
- // runtime unless the feature flag is enabled (at whatever current FCV when the
- // configuration setParameter command is run).
-
- query_stats_util::queryStatsStoreOnParamChangeUpdater(serviceCtx) =
- std::make_unique<QueryStatsOnParamChangeUpdaterImpl>();
- size_t size = getQueryStatsStoreSize();
- auto&& globalQueryStatsStoreManager = QueryStatsStoreManager::get(serviceCtx);
- // Initially the queryStats store used the same number of partitions as the plan cache, that
- // is the number of cpu cores. However, with performance investigation we found that when
- // the size of the partitions was too large, it took too long to copy out and read one
- // partition. We are now capping each partition at 16MB (the largest size a query shape can
- // be. If that gives us fewer partitions than we have cores, we set it to match the
- // number of cores. The size needs to be cast to a double since we want to round up the
- // number of partitions, and therefore need to avoid int division.
- size_t numPartitions = std::ceil(double(size) / (16 * 1024 * 1024));
- auto numLogicalCores = ProcessInfo::getNumCores();
- if (numPartitions < numLogicalCores) {
- numPartitions = numLogicalCores;
- }
-
- globalQueryStatsStoreManager =
- std::make_unique<QueryStatsStoreManager>(size, numPartitions);
- auto configuredSamplingRate = internalQueryStatsRateLimit.load();
- QueryStatsStoreManager::getRateLimiter(serviceCtx) = std::make_unique<RateLimiting>(
- configuredSamplingRate < 0 ? INT_MAX : configuredSamplingRate, Seconds{1});
- }};
-
-/**
- * Top-level checks for whether queryStats collection is enabled. If this returns false, we must
- * go no further.
- */
-bool isQueryStatsEnabled(const ServiceContext* serviceCtx) {
- // During initialization, FCV may not yet be setup but queries could be run. We can't
- // check whether queryStats should be enabled without FCV, so default to not recording
- // those queries.
- return isQueryStatsFeatureEnabled() &&
- QueryStatsStoreManager::get(serviceCtx)->getMaxSize() > 0;
-}
-
-/**
- * Internal check for whether we should collect metrics. This checks the rate limiting
- * configuration for a global on/off decision and, if enabled, delegates to the rate limiter.
- */
-bool shouldCollect(const ServiceContext* serviceCtx) {
- // Cannot collect queryStats if sampling rate is not greater than 0. Note that we do not
- // increment queryStatsRateLimitedRequestsMetric here since queryStats is entirely disabled.
- auto samplingRate = QueryStatsStoreManager::getRateLimiter(serviceCtx)->getSamplingRate();
- if (samplingRate <= 0) {
- LOGV2_DEBUG(8473001,
- 5,
- "sampling rate is <= 0, skipping this request",
- "samplingRate"_attr = samplingRate);
- return false;
- }
- // Check if rate limiting allows us to collect queryStats for this request.
- if (samplingRate < INT_MAX &&
- !QueryStatsStoreManager::getRateLimiter(serviceCtx)->handleRequestSlidingWindow()) {
- queryStatsRateLimitedRequestsMetric.increment();
- LOGV2_DEBUG(8473002,
- 5,
- "rate limited this request",
- "samplingRate"_attr = samplingRate,
- "totalLimited"_attr = queryStatsRateLimitedRequestsMetric.get());
- return false;
- }
- return true;
-}
-
-void updateStatistics(const QueryStatsStore::Partition& proofOfLock,
- QueryStatsEntry& toUpdate,
- const uint64_t queryExecMicros,
- const uint64_t firstResponseExecMicros,
- const uint64_t docsReturned) {
- toUpdate.latestSeenTimestamp = Date_t::now();
- toUpdate.lastExecutionMicros = queryExecMicros;
- toUpdate.execCount++;
- toUpdate.totalExecMicros.aggregate(queryExecMicros);
- toUpdate.firstResponseExecMicros.aggregate(firstResponseExecMicros);
- toUpdate.docsReturned.aggregate(docsReturned);
-}
-
-} // namespace
-
-void registerRequest(OperationContext* opCtx,
- const NamespaceString& collection,
- std::function<std::unique_ptr<Key>(void)> makeKey,
- bool willNeverExhaust) {
- if (!isQueryStatsEnabled(opCtx->getServiceContext())) {
- LOGV2_DEBUG(8473000,
- 5,
- "not collecting query stats for this request since it is disabled",
- "featureEnabled"_attr = isQueryStatsFeatureEnabled());
- return;
- }
-
- // Queries against metadata collections should never appear in queryStats data.
- if (collection.isFLE2StateCollection()) {
- return;
- }
-
- // Don't record queries from internal clients.
- if (opCtx->getClient()->session() &&
- (opCtx->getClient()->session()->getTags() & transport::Session::kInternalClient)) {
- return;
- }
-
- auto& opDebug = CurOp::get(opCtx)->debug();
-
- if (opDebug.queryStatsInfo.wasRateLimited) {
- LOGV2_DEBUG(
- 8288900,
- 4,
- "Query stats request was previously rate limited. We expect this is a query on a view");
- return;
- }
-
- if (!shouldCollect(opCtx->getServiceContext())) {
- opDebug.queryStatsInfo.wasRateLimited = true;
- return;
- }
-
- if (opDebug.queryStatsInfo.key) {
- // A find() request may have already registered the shapifier. Ie, it's a find command over
- // a non-physical collection, eg view, which is implemented by generating an agg pipeline.
- LOGV2_DEBUG(7198700,
- 2,
- "Query stats request shapifier already registered",
- "collection"_attr = collection);
- return;
- }
-
- opDebug.queryStatsInfo.willNeverExhaust = willNeverExhaust;
- // There are a few cases where a query shape can be larger than the original query. For example,
- // {$exists: false} in the input query serializes to {$not: {$exists: true}. In rare cases where
- // an input query has thousands of clauses, the cumulative bloat that shapification adds results
- // in a BSON object that exceeds the 16 MB memory limit. In these cases, we want to exclude the
- // original query from queryStats metrics collection and let it execute normally.
- try {
- opDebug.queryStatsInfo.key = makeKey();
- } catch (ExceptionFor<ErrorCodes::BSONObjectTooLarge>&) {
- LOGV2_DEBUG(7979400,
- 1,
- "Query Stats shapification has exceeded the 16 MB memory limit. Metrics will "
- "not be collected ");
- queryStatsStoreWriteErrorsMetric.increment();
- return;
- }
- opDebug.queryStatsInfo.keyHash = absl::Hash<query_stats::Key>{}(*opDebug.queryStatsInfo.key);
- // TODO look up this query shape (sub-component of query stats store key) in some new shared
- // data structure that the query settings component could share. See if the query SHAPE hash has
- // been computed before. If so, record the query shape hash on the opDebug. If not, compute the
- // hash and store it there so we can avoid re-doing this for each request.
-}
-
-QueryStatsStore& getQueryStatsStore(OperationContext* opCtx) {
- uassert(ErrorCodes::QueryFeatureNotAllowed,
- "Query stats is not enabled without the feature flag on and a cache size greater than "
- "0 bytes",
- isQueryStatsEnabled(opCtx->getServiceContext()));
- return QueryStatsStoreManager::get(opCtx->getServiceContext())->getQueryStatsStore();
-}
-
-void writeQueryStats(OperationContext* opCtx,
- boost::optional<size_t> queryStatsKeyHash,
- std::unique_ptr<Key> key,
- const uint64_t queryExecMicros,
- const uint64_t firstResponseExecMicros,
- const uint64_t docsReturned,
- bool willNeverExhaust) {
- // Generally we expect a 'key' to write query stats. However, for a change stream query, we
- // expect it has no 'key' after its first writeQueryStats(), but it must have a
- // 'queryStatsKeyHash' for its entry to be updated.
- // TODO SERVER-89058 Modify comment to include tailable cursors.
- if (!key && !(willNeverExhaust && queryStatsKeyHash)) {
- return;
- }
-
- // It's possible that query stats was enabled in registerRequest but has been disabled since
- // (e.g., by FCV downgrade or setting the store size to 0). Rather than calling
- // getQueryStatsStore (which would trigger a uassert if queryStats is disabled), we return and
- // log a message if query stats is disabled, and otherwise grab the query stats store directly.
- if (!isQueryStatsEnabled(opCtx->getServiceContext())) {
- LOGV2_DEBUG(8456700,
- 2,
- "Query stats was enabled when the command started but is now disabled. "
- "Metrics will not be collected.",
- "queryStatsKeyHash"_attr = queryStatsKeyHash);
- return;
- }
- auto&& queryStatsStore =
- QueryStatsStoreManager::get(opCtx->getServiceContext())->getQueryStatsStore();
- if (key) {
- dassert(absl::Hash<query_stats::Key>{}(*key) == queryStatsKeyHash,
- "Expecting query stats key to hash to the given hash. Is the OpCtx state being "
- "incorrectly re-used?");
- }
- auto&& [statusWithMetrics, partitionLock] =
- queryStatsStore.getWithPartitionLock(*queryStatsKeyHash);
- if (statusWithMetrics.isOK()) {
- // Found an existing entry! Just update the metrics and we're done.
- return updateStatistics(partitionLock,
- *statusWithMetrics.getValue(),
- queryExecMicros,
- firstResponseExecMicros,
- docsReturned);
- }
-
- // It is possible a cursor that lives forever has no key associated with it and its entry may
- // have been evicted.
- if (willNeverExhaust && !key) {
- return;
- }
-
- // Otherwise we didn't find an existing entry. Try to create one.
- tassert(7315200,
- "key cannot be null when writing a new entry to the queryStats store",
- key != nullptr);
- size_t numEvicted =
- queryStatsStore.put(*queryStatsKeyHash, QueryStatsEntry(std::move(key)), partitionLock);
- queryStatsEvictedMetric.increment(numEvicted);
- auto newMetrics = partitionLock->get(*queryStatsKeyHash);
- if (!newMetrics.isOK()) {
- // This can happen if the budget is immediately exceeded. Specifically if the there is
- // not enough room for a single new entry if the number of partitions is too high
- // relative to the size.
- queryStatsStoreWriteErrorsMetric.increment();
- LOGV2_DEBUG(7560900,
- 0,
- "Failed to store queryStats entry.",
- "status"_attr = newMetrics.getStatus(),
- "queryStatsKeyHash"_attr = queryStatsKeyHash);
- return;
- }
-
- return updateStatistics(partitionLock,
- newMetrics.getValue()->second,
- queryExecMicros,
- firstResponseExecMicros,
- docsReturned);
-}
-
-void writeQueryStatsOnCursorDisposeOrKill(OperationContext* opCtx,
- boost::optional<size_t> queryStatsKeyHash,
- std::unique_ptr<Key> key,
- bool willNeverExhaust,
- const uint64_t queryExecMicros,
- const uint64_t firstResponseExecMicros,
- const uint64_t docsReturned) {
- // It is discouraged but technically possible for a user to enable queryStats on the mongods of
- // a replica set. In this case, a cursor will be created for each mongod. However, the
- // queryStatsKey is behind a unique_ptr on CurOp. The ClientCursor constructor std::moves the
- // queryStatsKey so it uniquely owns it (and also makes the queryStatsKey on CurOp now a
- // nullptr) and copies over the queryStatsKeyHash as the latter is a cheap copy.
- // In the case of sharded $search, two cursors will be created per mongod. In this way,
- // two cursors are part of the same thread/operation, and therefore share a OpCtx/CurOp/OpDebug.
- // The first cursor that is created will own the queryStatsKey and have a copy of the
- // queryStatsKeyHash. On the other hand, the second one will only have a copy of the hash since
- // the queryStatsKey will be null on CurOp from being std::move'd in the first cursor
- // construction call. To not trip the tassert in writeQueryStats and because all cursors are
- // guaranteed to have a copy of the hash, we check that the cursor has a key
- if (key && opCtx) {
- query_stats::writeQueryStats(opCtx,
- queryStatsKeyHash,
- std::move(key),
- queryExecMicros,
- firstResponseExecMicros,
- docsReturned,
- willNeverExhaust);
- } else if (willNeverExhaust && opCtx) {
- // Since we already recorded information about the possible getMores associated with a
- // cursor that never ends, the only information left to record is about the kill/dispose
- // cursor operation. This operation is not timed and does not have any metrics associated
- // with it.
- query_stats::writeQueryStats(opCtx, queryStatsKeyHash, nullptr, 0, 0, 0, willNeverExhaust);
- }
-}
-
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/query_stats.h b/src/mongo/db/query/query_stats/query_stats.h
deleted file mode 100644
index fc96a8be179..00000000000
--- a/src/mongo/db/query/query_stats/query_stats.h
+++ /dev/null
@@ -1,211 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/base/status.h"
-#include "mongo/bson/bsonobj.h"
-#include "mongo/db/concurrency/d_concurrency.h"
-#include "mongo/db/curop.h"
-#include "mongo/db/namespace_string.h"
-#include "mongo/db/query/partitioned_cache.h"
-#include "mongo/db/query/plan_explainer.h"
-#include "mongo/db/query/query_stats/key.h"
-#include "mongo/db/query/query_stats/query_stats_entry.h"
-#include "mongo/db/query/query_stats/rate_limiting.h"
-#include "mongo/db/service_context.h"
-#include "mongo/db/views/view.h"
-#include <cstdint>
-#include <memory>
-
-namespace mongo::query_stats {
-
-extern Counter64 queryStatsStoreSizeEstimateBytesMetric;
-
-struct QueryStatsPartitioner {
- // The partitioning function for use with the 'Partitioned' utility.
- std::size_t operator()(const std::size_t hash, const std::size_t nPartitions) const {
- return hash % nPartitions;
- }
-};
-
-struct QueryStatsStoreEntryBudgetor {
- size_t operator()(const std::size_t hash, const QueryStatsEntry& value) {
- return sizeof(decltype(value)) + sizeof(decltype(hash)) + value.key->size();
- }
-};
-
-/*
- * 'QueryStatsStore insertion and eviction listener implementation. This class adjusts the
- * 'queryStatsStoreSize' serverStatus metric when entries are inserted or evicted.
- */
-struct QueryStatsStoreInsertionEvictionListener {
- void onInsert(const std::size_t&, const QueryStatsEntry&, size_t estimatedSize) {
- queryStatsStoreSizeEstimateBytesMetric.increment(estimatedSize);
- }
-
- void onEvict(const std::size_t&, const QueryStatsEntry&, size_t estimatedSize) {
- queryStatsStoreSizeEstimateBytesMetric.decrement(estimatedSize);
- }
-
- void onClear(size_t estimatedSize) {
- queryStatsStoreSizeEstimateBytesMetric.decrement(estimatedSize);
- }
-};
-using QueryStatsStore = PartitionedCache<std::size_t,
- QueryStatsEntry,
- QueryStatsStoreEntryBudgetor,
- QueryStatsPartitioner,
- QueryStatsStoreInsertionEvictionListener>;
-
-/**
- * A manager for the queryStats store allows a "pointer swap" on the queryStats store itself. The
- * usage patterns are as follows:
- *
- * - Updating the queryStats store uses the `getQueryStatsStore()` method. The queryStats store
- * instance is obtained, entries are looked up and mutated, or created anew.
- * - The queryStats store is "reset". This involves atomically allocating a new instance, once
- * there are no more updaters (readers of the store "pointer"), and returning the existing
- * instance.
- */
-class QueryStatsStoreManager {
-public:
- // The query stats store can be configured using these objects on a per-ServiceContext level.
- // This is essentially global, but can be manipulated by unit tests.
- static const ServiceContext::Decoration<std::unique_ptr<QueryStatsStoreManager>> get;
- static const ServiceContext::Decoration<std::unique_ptr<RateLimiting>> getRateLimiter;
-
- template <typename... QueryStatsStoreArgs>
- QueryStatsStoreManager(size_t cacheSize, size_t numPartitions)
- : _queryStatsStore(std::make_unique<QueryStatsStore>(cacheSize, numPartitions)),
- _maxSize(cacheSize) {}
-
- /**
- * Acquire the instance of the queryStats store.
- */
- QueryStatsStore& getQueryStatsStore() {
- return *_queryStatsStore;
- }
-
- size_t getMaxSize() {
- return _maxSize.load();
- }
-
- /**
- * Resize the queryStats store and return the number of evicted
- * entries.
- */
- size_t resetSize(size_t cacheSize) {
- _maxSize.store(cacheSize);
- return _queryStatsStore->reset(cacheSize);
- }
-
-private:
- std::unique_ptr<QueryStatsStore> _queryStatsStore;
-
- /**
- * Max size of the queryStats store. Tracked here to avoid having to recompute after it's
- * divided up into partitions.
- */
- AtomicWord<size_t> _maxSize;
-};
-
-/**
- * Acquire a reference to the global queryStats store.
- */
-QueryStatsStore& getQueryStatsStore(OperationContext* opCtx);
-
-/**
- * Registers a request for query stats collection. The function may decide not to collect anything,
- * so this should be called for all requests. The decision is made based on the feature flag and
- * query stats rate limiting.
- *
- * The originating command/query does not persist through the end of query execution due to
- * optimizations made to the original query and the expiration of OpCtx across getMores. In order
- * to pair the query stats metrics that are collected at the end of execution with the original
- * query, it is necessary to store the original query during planning and persist it through
- * getMores.
- *
- * During planning, registerRequest is called to serialize the query stats key and save it to
- * OpDebug. If a query's execution is complete within the original operation,
- * collectQueryStatsMongod/collectQueryStatsMongos will call writeQueryStats() and pass along the
- * query stats key to be saved in the query stats store alongside metrics collected.
- *
- * However, OpDebug does not persist through cursor iteration, so if a query's execution will span
- * more than one request/operation, it's necessary to save the query stats context to the cursor
- * upon cursor registration. In these cases, collectQueryStatsMongod/collectQueryStatsMongos will
- * aggregate each operation's metrics within the cursor. Once the request is eventually complete,
- * the cursor calls writeQueryStats() on its destruction.
- *
- * Notes:
- * - It's important to call registerRequest with the original request, before canonicalizing or
- * optimizing it, in order to preserve the user's input for the query shape.
- * - Calling this affects internal state. It should be called exactly once for each request for
- * which query stats may be collected.
- * - The std::function argument to construct an abstracted Key is provided to break
- * library cycles so this library does not need to know how to parse everything. It is done as a
- * deferred construction callback to ensure that this feature does not impact performance if
- * collecting stats is not needed due to the feature being disabled or the request being rate
- * limited.
- */
-void registerRequest(OperationContext* opCtx,
- const NamespaceString& collection,
- std::function<std::unique_ptr<Key>(void)> makeKey,
- bool willNeverExhaust = false);
-
-/**
- * Writes query stats to the query stats store for the operation identified by `queryStatsKeyHash`.
- *
- * Direct calls to writeQueryStats in new code should be avoided in favor of calling existing
- * functions:
- * - collectQueryStatsMongod/collectQueryStatsMongos in the case of requests that span one
- * operation
- * - writeQueryStatsOnCursorDisposeOrKill() in the case of requests that span
- * multiple operations (via getMore)
- */
-void writeQueryStats(OperationContext* opCtx,
- boost::optional<size_t> queryStatsKeyHash,
- std::unique_ptr<Key> key,
- uint64_t queryExecMicros,
- uint64_t firstResponseExecMicros,
- uint64_t docsReturned,
- bool willNeverExhaust = false);
-
-/**
- * Called from ClientCursor::dispose/ClusterClientCursorImpl::kill to set up and writeQueryStats()
- * at the end of life of a cursor.
- */
-void writeQueryStatsOnCursorDisposeOrKill(OperationContext* opCtx,
- boost::optional<size_t> queryStatsKeyHash,
- std::unique_ptr<Key> key,
- bool willNeverExhaust,
- uint64_t queryExecMicros,
- uint64_t firstResponseExecMicros,
- uint64_t docsReturned);
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/query_stats_entry.cpp b/src/mongo/db/query/query_stats/query_stats_entry.cpp
deleted file mode 100644
index f69f0a6ee2a..00000000000
--- a/src/mongo/db/query/query_stats/query_stats_entry.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_stats/query_stats_entry.h"
-
-#include <boost/optional.hpp>
-
-#include "mongo/crypto/hash_block.h"
-#include "mongo/crypto/sha256_block.h"
-
-#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
-
-namespace mongo::query_stats {
-
-BSONObj QueryStatsEntry::toBSON() const {
- BSONObjBuilder builder{sizeof(QueryStatsEntry) + 100};
- builder.append("lastExecutionMicros", (long long)lastExecutionMicros);
- builder.append("execCount", (long long)execCount);
- totalExecMicros.appendTo(builder, "totalExecMicros");
- firstResponseExecMicros.appendTo(builder, "firstResponseExecMicros");
- docsReturned.appendTo(builder, "docsReturned");
- builder.append("firstSeenTimestamp", firstSeenTimestamp);
- builder.append("latestSeenTimestamp", latestSeenTimestamp);
- return builder.obj();
-}
-
-
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/query_stats_entry.h b/src/mongo/db/query/query_stats/query_stats_entry.h
deleted file mode 100644
index 6b61a6a6dcf..00000000000
--- a/src/mongo/db/query/query_stats/query_stats_entry.h
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include <algorithm>
-#include <cstdint>
-#include <memory>
-
-#include "mongo/db/commands/server_status_metric.h"
-#include "mongo/db/query/query_stats/aggregated_metric.h"
-#include "mongo/db/query/query_stats/key.h"
-#include "mongo/db/query/query_stats/transform_algorithm_gen.h"
-#include "mongo/util/time_support.h"
-
-namespace mongo::query_stats {
-
-/**
- * The value stored in the query stats store. It contains a Key representing this "kind" of
- * query, and some metrics about that shape. This class is responsible for knowing its size and
- * updating our server status metrics about the size of the query stats store accordingly. At the
- * time of this writing, the LRUCache utility does not easily expose its size in a way we could use
- * as server status metrics.
- */
-struct QueryStatsEntry {
- QueryStatsEntry(std::unique_ptr<const Key> key_)
- : firstSeenTimestamp(Date_t::now()), key(std::move(key_)) {}
-
- BSONObj toBSON() const;
-
- /**
- * Timestamp for when this query shape was added to the store. Set on construction.
- */
- const Date_t firstSeenTimestamp;
-
- /**
- * Timestamp for when the latest time this query shape was seen.
- */
- Date_t latestSeenTimestamp;
-
- /**
- * Last execution time in microseconds.
- */
- uint64_t lastExecutionMicros = 0;
-
- /**
- * Number of query executions.
- */
- uint64_t execCount = 0;
-
- /**
- * Aggregates the total time for execution including getMore requests.
- */
- AggregatedMetric totalExecMicros;
-
- /**
- * Aggregates the time for execution for first batch only.
- */
- AggregatedMetric firstResponseExecMicros;
-
- AggregatedMetric docsReturned;
-
- /**
- * The Key that can generate the query stats key for this request.
- */
- std::shared_ptr<const Key> key;
-};
-
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/query_stats_helpers.h b/src/mongo/db/query/query_stats/query_stats_helpers.h
deleted file mode 100644
index 6d53cc8d4ce..00000000000
--- a/src/mongo/db/query/query_stats/query_stats_helpers.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include <absl/hash/hash.h>
-#include <boost/optional.hpp>
-
-#include "mongo/bson/bsonobj.h"
-#include "mongo/bson/simple_bsonobj_comparator.h"
-#include "mongo/db/query/query_shape/shape_helpers.h"
-
-namespace mongo::query_stats {
-
-/**
- * An abseil compatible hash function for BSONObjects. Note that this hasher ignores any collation
- * and uses the "simple" comparisons. This is fine and correct for query stats, but this is
- * intentionally placed within the 'query_stats' namespace to avoid polluting the whole codebase
- * with this helper which could cause an accidental bug where we ignore the request's collation.
- */
-template <typename H>
-H AbslHashValue(H h, const BSONObj& obj) {
- return H::combine(std::move(h), simpleHash(obj));
-}
-
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/query_stats_on_parameter_change.cpp b/src/mongo/db/query/query_stats/query_stats_on_parameter_change.cpp
deleted file mode 100644
index a8b7df9fccb..00000000000
--- a/src/mongo/db/query/query_stats/query_stats_on_parameter_change.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-
-#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
-
-#include "mongo/db/query/query_stats/query_stats_on_parameter_change.h"
-
-#include "mongo/base/status.h"
-#include "mongo/db/concurrency/d_concurrency.h"
-#include "mongo/db/query/partitioned_cache.h"
-#include "mongo/db/query/query_knobs_gen.h"
-#include "mongo/db/query/util/memory_util.h"
-#include "mongo/db/service_context.h"
-#include "mongo/logv2/log.h"
-
-namespace mongo::query_stats_util {
-
-namespace {
-/**
- * Given the current 'Client', returns a pointer to the 'ServiceContext' and an interface for
- * updating the queryStats store.
- */
-std::pair<ServiceContext*, OnParamChangeUpdater*> getUpdater(const Client& client) {
- auto serviceCtx = client.getServiceContext();
- tassert(7106500, "ServiceContext must be non null", serviceCtx);
-
- auto updater = queryStatsStoreOnParamChangeUpdater(serviceCtx).get();
- tassert(7106501, "queryStats store size updater must be non null", updater);
- return {serviceCtx, updater};
-}
-} // namespace
-
-
-Status onQueryStatsStoreSizeUpdate(const std::string& str) {
- auto newSize = memory_util::MemorySize::parse(str);
- if (!newSize.isOK()) {
- return newSize.getStatus();
- }
-
- // The client is nullptr if the parameter is supplied from the command line. In this case, we
- // ignore the update event, the parameter will be processed when initializing the service
- // context.
- if (auto client = Client::getCurrent()) {
- auto&& [serviceCtx, updater] = getUpdater(*client);
- updater->updateCacheSize(serviceCtx, newSize.getValue());
- }
-
- return Status::OK();
-}
-
-Status validateQueryStatsStoreSize(const std::string& str) {
- return memory_util::MemorySize::parse(str).getStatus();
-}
-
-Status onQueryStatsSamplingRateUpdate(int samplingRate) {
- // The client is nullptr if the parameter is supplied from the command line. In this case, we
- // ignore the update event, the parameter will be processed when initializing the service
- // context.
- if (auto client = Client::getCurrent()) {
- auto&& [serviceCtx, updater] = getUpdater(*client);
- updater->updateSamplingRate(serviceCtx, samplingRate < 0 ? INT_MAX : samplingRate);
- }
-
- return Status::OK();
-}
-
-const Decorable<ServiceContext>::Decoration<std::unique_ptr<OnParamChangeUpdater>>
- queryStatsStoreOnParamChangeUpdater =
- ServiceContext::declareDecoration<std::unique_ptr<OnParamChangeUpdater>>();
-} // namespace mongo::query_stats_util
diff --git a/src/mongo/db/query/query_stats/query_stats_on_parameter_change.h b/src/mongo/db/query/query_stats/query_stats_on_parameter_change.h
deleted file mode 100644
index 2a824961b34..00000000000
--- a/src/mongo/db/query/query_stats/query_stats_on_parameter_change.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/base/status.h"
-#include "mongo/db/concurrency/d_concurrency.h"
-#include "mongo/db/query/partitioned_cache.h"
-#include "mongo/db/query/util/memory_util.h"
-
-
-namespace mongo::query_stats_util {
-
-Status onQueryStatsStoreSizeUpdate(const std::string& str);
-
-
-Status validateQueryStatsStoreSize(const std::string& str);
-
-Status onQueryStatsSamplingRateUpdate(int samplingRate);
-
-/**
- * An interface used to modify the queryStats store when query setParameters are modified. This is
- * done via an interface decorating the 'ServiceContext' in order to avoid a link-time dependency of
- * the query knobs library on the queryStats code.
- */
-class OnParamChangeUpdater {
-public:
- virtual ~OnParamChangeUpdater() = default;
-
- /**
- * Resizes the queryStats store decorating 'serviceCtx' to the new size given by 'memSize'. If
- * the new size is smaller than the old, cache entries are evicted in order to ensure the
- * cache fits within the new size bound.
- */
- virtual void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) = 0;
-
- /**
- * Updates the sampling rate for the queryStats rate limiter.
- */
- virtual void updateSamplingRate(ServiceContext* serviceCtx, int samplingRate) = 0;
-};
-
-/**
- * Decorated accessor to the 'OnParamChangeUpdater' stored in 'ServiceContext'. Again, this is done
- * via a decoration and interface to avoid a link-time dependency from the query knobs library on
- * the queryStats code.
- */
-extern const Decorable<ServiceContext>::Decoration<std::unique_ptr<OnParamChangeUpdater>>
- queryStatsStoreOnParamChangeUpdater;
-} // namespace mongo::query_stats_util
diff --git a/src/mongo/db/query/query_stats/query_stats_store_test.cpp b/src/mongo/db/query/query_stats/query_stats_store_test.cpp
deleted file mode 100644
index 90cb073eafc..00000000000
--- a/src/mongo/db/query/query_stats/query_stats_store_test.cpp
+++ /dev/null
@@ -1,1406 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/bson/simple_bsonobj_comparator.h"
-#include "mongo/db/catalog/rename_collection.h"
-#include "mongo/db/collection_type.h"
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/query_feature_flags_gen.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
-#include "mongo/db/query/query_stats/agg_key.h"
-#include "mongo/db/query/query_stats/find_key.h"
-#include "mongo/db/query/query_stats/key.h"
-#include "mongo/db/query/query_stats/query_stats.h"
-#include "mongo/db/service_context_test_fixture.h"
-#include "mongo/idl/server_parameter_test_util.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo::query_stats {
-
-int countAllEntries(const QueryStatsStore& store) {
- int numKeys = 0;
- store.forEach([&](auto&& key, auto&& entry) { numKeys++; });
- return numKeys;
-}
-
-static const NamespaceStringOrUUID kDefaultTestNss = NamespaceString("testDB.testColl");
-class QueryStatsStoreTest : public ServiceContextTest {
-public:
- static std::unique_ptr<const Key> makeFindKeyFromQuery(BSONObj filter) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
- auto fcr = std::make_unique<FindCommandRequest>(kDefaultTestNss);
- fcr->setFilter(filter.getOwned());
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcr)));
- return std::make_unique<FindKey>(expCtx, *parsedFind, collectionType);
- }
-
- static constexpr auto collectionType = query_shape::CollectionType::kCollection;
- BSONObj makeQueryStatsKeyFindRequest(const FindCommandRequest& fcr,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- bool applyHmac) {
- auto fcrCopy = std::make_unique<FindCommandRequest>(fcr);
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcrCopy)));
- FindKey findKey(expCtx, *parsedFind, collectionType);
- SerializationOptions opts = SerializationOptions::kDebugShapeAndMarkIdentifiers_FOR_TEST;
- if (!applyHmac) {
- opts.transformIdentifiers = false;
- opts.transformIdentifiersCallback = defaultHmacStrategy;
- }
- return findKey.toBson(expCtx->opCtx, opts);
- }
-
- BSONObj makeQueryStatsKeyAggregateRequest(AggregateCommandRequest acr,
- const Pipeline& pipeline,
- const boost::intrusive_ptr<ExpressionContext>& expCtx,
- LiteralSerializationPolicy literalPolicy,
- bool applyHmac = false) {
- auto aggKey = std::make_unique<AggKey>(acr,
- pipeline,
- expCtx,
- pipeline.getInvolvedCollections(),
- acr.getNamespace(),
- collectionType);
-
- // SerializationOptions opts{.literalPolicy = literalPolicy};
- SerializationOptions opts = SerializationOptions::kMarkIdentifiers_FOR_TEST;
- opts.literalPolicy = literalPolicy;
- if (!applyHmac) {
- opts.transformIdentifiers = false;
- opts.transformIdentifiersCallback = defaultHmacStrategy;
- }
- return aggKey->toBson(expCtx->opCtx, opts);
- }
-};
-
-TEST_F(QueryStatsStoreTest, BasicUsage) {
- QueryStatsStore queryStatsStore{5000000, 1000};
-
- auto getMetrics = [&](BSONObj query) {
- auto key = makeFindKeyFromQuery(query);
- auto lookupResult = queryStatsStore.lookup(absl::Hash<query_stats::Key>{}(*key));
- ASSERT_OK(lookupResult);
- return *lookupResult.getValue();
- };
-
- auto collectMetrics = [&](BSONObj query) {
- auto key = makeFindKeyFromQuery(query);
- auto lookupHash = absl::Hash<query_stats::Key>{}(*key);
- auto lookupResult = queryStatsStore.lookup(lookupHash);
- if (!lookupResult.isOK()) {
- queryStatsStore.put(lookupHash, QueryStatsEntry{std::move(key)});
- lookupResult = queryStatsStore.lookup(lookupHash);
- }
- auto metrics = lookupResult.getValue();
- metrics->execCount += 1;
- metrics->lastExecutionMicros += 123456;
- };
-
- auto query1 = BSON("query" << 1 << "xEquals" << 42);
- // same value, different instance (tests hashing & equality)
- auto query1x = BSON("query" << 1 << "xEquals" << 42);
- auto query2 = BSON("query" << 2 << "yEquals" << 43);
-
- collectMetrics(query1);
- collectMetrics(query1);
- collectMetrics(query1x);
- collectMetrics(query2);
-
- ASSERT_EQ(getMetrics(query1).execCount, 3);
- ASSERT_EQ(getMetrics(query1x).execCount, 3);
- ASSERT_EQ(getMetrics(query2).execCount, 1);
-
- auto collectMetricsWithLock = [&](BSONObj& filter) {
- auto key = makeFindKeyFromQuery(filter);
- auto [lookupResult, lock] =
- queryStatsStore.getWithPartitionLock(absl::Hash<query_stats::Key>{}(*key));
- ASSERT_OK(lookupResult);
- auto& metrics = *lookupResult.getValue();
- metrics.execCount += 1;
- metrics.lastExecutionMicros += 123456;
- };
-
- collectMetricsWithLock(query1x);
- collectMetricsWithLock(query2);
-
- ASSERT_EQ(getMetrics(query1).execCount, 4);
- ASSERT_EQ(getMetrics(query1x).execCount, 4);
- ASSERT_EQ(getMetrics(query2).execCount, 2);
-
- ASSERT_EQ(2, countAllEntries(queryStatsStore));
-}
-
-TEST_F(QueryStatsStoreTest, EvictionTest) {
- // This creates a queryStats store with a single partition to specifically test the eviction
- // behavior with very large queries.
- // Add an entry that is smaller than the max partition size.
- auto query = BSON("query" << 1 << "xEquals" << 42);
- auto key = makeFindKeyFromQuery(query);
-
- const size_t cacheSize = key->size() + sizeof(QueryStatsEntry) + 100;
- const auto numPartitions = 1;
- QueryStatsStore queryStatsStore{cacheSize, numPartitions};
-
- auto hash = absl::Hash<query_stats::Key>{}(*key);
- queryStatsStore.put(hash, QueryStatsEntry{std::move(key)});
- ASSERT_EQ(countAllEntries(queryStatsStore), 1);
-
- // We'll do this again later so save this as a helper function.
- auto addLargeEntry = [&](auto& queryStatsStore) {
- // Add an entry that is larger than the max partition size to the non-empty partition. This
- // should evict both entries, the first small entry written to the partition and the current
- // too large entry we wish to write to the partition. The reason is because entries are
- // evicted from the partition in order of least recently used. Thus, the small entry will be
- // evicted first but the partition will still be over budget so the final, too large entry
- // will also be evicted.
- auto opCtx = makeOperationContext();
- auto fcr = std::make_unique<FindCommandRequest>(
- NamespaceStringOrUUID(NamespaceString("testDB.testColl")));
- fcr->setLet(BSON("var" << 2));
- fcr->setFilter(fromjson("{$expr: [{$eq: ['$a', '$$var']}]}"));
- fcr->setProjection(fromjson("{varIs: '$$var'}"));
- fcr->setLimit(5);
- fcr->setSkip(2);
- fcr->setBatchSize(25);
- fcr->setMaxTimeMS(1000);
- fcr->setNoCursorTimeout(false);
- opCtx->setComment(BSON("comment"
- << " foo bar baz"));
- fcr->setSingleBatch(false);
- fcr->setAllowDiskUse(false);
- fcr->setAllowPartialResults(true);
- fcr->setAllowDiskUse(false);
- fcr->setShowRecordId(true);
- fcr->setHint(BSON("z" << 1 << "c" << 1));
- fcr->setMax(BSON("z" << 25));
- fcr->setMin(BSON("z" << 80));
- fcr->setSort(BSON("sortVal" << 1 << "otherSort" << -1));
- auto&& [expCtx, parsedFind] =
- uassertStatusOK(parsed_find_command::parse(opCtx.get(), std::move(fcr)));
-
- key = std::make_unique<query_stats::FindKey>(expCtx, *parsedFind, collectionType);
- auto lookupHash = absl::Hash<query_stats::Key>{}(*key);
- QueryStatsEntry testMetrics{std::move(key)};
- queryStatsStore.put(lookupHash, testMetrics);
- };
-
- addLargeEntry(queryStatsStore);
- ASSERT_EQ(countAllEntries(queryStatsStore), 0);
-
- // This creates a queryStats store where each partition has a max size of 500 bytes.
- QueryStatsStore queryStatsStoreTwo{/*cacheSize*/ cacheSize * 3, /*numPartitions*/ 3};
- // Adding a queryStats store entry that is smaller than the overal cache size but larger
- // than a single partition max size, will cause an eviction. testMetrics is larger than 500
- // bytes and thus over budget for the partitions of this cache.
- addLargeEntry(queryStatsStoreTwo);
- ASSERT_EQ(countAllEntries(queryStatsStoreTwo), 0);
-}
-
-TEST_F(QueryStatsStoreTest, GenerateMaxBsonSizeQueryShape) {
- const NamespaceString nss = NamespaceString("testDB.testColl");
- FindCommandRequest fcr((NamespaceStringOrUUID(nss)));
- // This creates a query that is just below the 16 MB memory limit.
- int limit = 225500;
- BSONObjBuilder bob;
- BSONArrayBuilder andBob(bob.subarrayStart("$and"));
- for (int i = 1; i <= limit; i++) {
- BSONObjBuilder childrenBob;
- childrenBob.append("x", BSON("$lt" << i << "$gte" << i));
- andBob.append(childrenBob.obj());
- }
- andBob.doneFast();
- fcr.setFilter(bob.obj());
- auto fcrCopy = std::make_unique<FindCommandRequest>(fcr);
- auto opCtx = makeOperationContext();
- auto parsedFindPair =
- uassertStatusOK(parsed_find_command::parse(opCtx.get(), std::move(fcrCopy)));
-
- auto&& globalQueryStatsStoreManager = QueryStatsStoreManager::get(opCtx->getServiceContext());
- globalQueryStatsStoreManager = std::make_unique<QueryStatsStoreManager>(500000, 1000);
-
- // The shapification process will bloat the input query over the 16 MB memory limit. Assert that
- // calling registerRequest() doesn't throw and that the opDebug isn't registered with a key hash
- // (thus metrics won't be tracked for this query).
- ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() {
- return std::make_unique<query_stats::FindKey>(
- parsedFindPair.first, *parsedFindPair.second, query_shape::CollectionType::kCollection);
- }));
- auto& opDebug = CurOp::get(*opCtx)->debug();
- ASSERT_FALSE(opDebug.queryStatsInfo.keyHash.has_value());
-}
-
-TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestAllFields) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
- FindCommandRequest fcr(kDefaultTestNss);
-
- fcr.setFilter(BSON("a" << 1));
-
- auto key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "find",
- "filter": {
- "HASH<a>": {
- "$eq": "?number"
- }
- }
- },
- "collectionType": "collection"
- })",
- key);
-
- // Add sort.
- fcr.setSort(BSON("sortVal" << 1 << "otherSort" << -1));
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "find",
- "filter": {
- "HASH<a>": {
- "$eq": "?number"
- }
- },
- "sort": {
- "HASH<sortVal>": 1,
- "HASH<otherSort>": -1
- }
- },
- "collectionType": "collection"
- })",
- key);
-
- // Add inclusion projection.
- fcr.setProjection(BSON("e" << true << "f" << true));
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "find",
- "filter": {
- "HASH<a>": {
- "$eq": "?number"
- }
- },
- "projection": {
- "HASH<e>": true,
- "HASH<f>": true,
- "HASH<_id>": true
- },
- "sort": {
- "HASH<sortVal>": 1,
- "HASH<otherSort>": -1
- }
- },
- "collectionType": "collection"
- })",
- key);
-
- // Add let.
- fcr.setLet(BSON("var1" << 1 << "var2"
- << "const1"));
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "let": {
- "HASH<var1>": "?number",
- "HASH<var2>": "?string"
- },
- "command": "find",
- "filter": {
- "HASH<a>": {
- "$eq": "?number"
- }
- },
- "projection": {
- "HASH<e>": true,
- "HASH<f>": true,
- "HASH<_id>": true
- },
- "sort": {
- "HASH<sortVal>": 1,
- "HASH<otherSort>": -1
- }
- },
- "collectionType": "collection"
- })",
- key);
-
- // Add hinting fields.
- fcr.setHint(BSON("z" << 1 << "c" << 1));
- fcr.setMax(BSON("z" << 25));
- fcr.setMin(BSON("z" << 80));
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "let": {
- "HASH<var1>": "?number",
- "HASH<var2>": "?string"
- },
- "command": "find",
- "filter": {
- "HASH<a>": {
- "$eq": "?number"
- }
- },
- "projection": {
- "HASH<e>": true,
- "HASH<f>": true,
- "HASH<_id>": true
- },
- "max": {
- "HASH<z>": "?number"
- },
- "min": {
- "HASH<z>": "?number"
- },
- "sort": {
- "HASH<sortVal>": 1,
- "HASH<otherSort>": -1
- }
- },
- "collectionType": "collection",
- "hint": {
- "HASH<z>": 1,
- "HASH<c>": 1
- }
- })",
- key);
-
- // Add the literal redaction fields.
- fcr.setLimit(5);
- fcr.setSkip(2);
- fcr.setBatchSize(25);
- fcr.setMaxTimeMS(1000);
- fcr.setNoCursorTimeout(false);
-
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "let": {
- "HASH<var1>": "?number",
- "HASH<var2>": "?string"
- },
- "command": "find",
- "filter": {
- "HASH<a>": {
- "$eq": "?number"
- }
- },
- "projection": {
- "HASH<e>": true,
- "HASH<f>": true,
- "HASH<_id>": true
- },
- "max": {
- "HASH<z>": "?number"
- },
- "min": {
- "HASH<z>": "?number"
- },
- "sort": {
- "HASH<sortVal>": 1,
- "HASH<otherSort>": -1
- },
- "limit": "?number",
- "skip": "?number"
- },
- "collectionType": "collection",
- "hint": {
- "HASH<z>": 1,
- "HASH<c>": 1
- },
- "maxTimeMS": "?number",
- "noCursorTimeout": false,
- "batchSize": "?number"
- })",
- key);
-
- // Add the fields that shouldn't be hmacApplied.
- fcr.setSingleBatch(true);
- fcr.setAllowDiskUse(false);
- fcr.setAllowPartialResults(true);
- fcr.setAllowDiskUse(false);
- fcr.setShowRecordId(true);
- auto readPreference = BSON("mode"
- << "nearest"
- << "tags"
- << BSON_ARRAY(BSON("some"
- << "tag")
- << BSON("some"
- << "other tag")));
- ReadPreferenceSetting::get(expCtx->opCtx) =
- uassertStatusOK(ReadPreferenceSetting::fromInnerBSON(readPreference));
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "let": {
- "HASH<var1>": "?number",
- "HASH<var2>": "?string"
- },
- "command": "find",
- "filter": {
- "HASH<a>": {
- "$eq": "?number"
- }
- },
- "projection": {
- "HASH<e>": true,
- "HASH<f>": true,
- "HASH<_id>": true
- },
- "max": {
- "HASH<z>": "?number"
- },
- "min": {
- "HASH<z>": "?number"
- },
- "sort": {
- "HASH<sortVal>": 1,
- "HASH<otherSort>": -1
- },
- "limit": "?number",
- "skip": "?number",
- "singleBatch": true,
- "allowDiskUse": false,
- "showRecordId": true
- },
- "$readPreference": {
- "mode": "nearest",
- "tags": [ { "some": "other tag" }, { "some": "tag" } ],
- "hedge": { "enabled": true }
- },
- "collectionType": "collection",
- "hint": {
- "HASH<z>": 1,
- "HASH<c>": 1
- },
- "maxTimeMS": "?number",
- "allowPartialResults": true,
- "noCursorTimeout": false,
- "batchSize": "?number"
- })",
- key);
-
- fcr.setAllowPartialResults(false);
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
- // Make sure that a false allowPartialResults is also accurately captured.
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "let": {
- "HASH<var1>": "?number",
- "HASH<var2>": "?string"
- },
- "command": "find",
- "filter": {
- "HASH<a>": {
- "$eq": "?number"
- }
- },
- "projection": {
- "HASH<e>": true,
- "HASH<f>": true,
- "HASH<_id>": true
- },
- "max": {
- "HASH<z>": "?number"
- },
- "min": {
- "HASH<z>": "?number"
- },
- "sort": {
- "HASH<sortVal>": 1,
- "HASH<otherSort>": -1
- },
- "limit": "?number",
- "skip": "?number",
- "singleBatch": true,
- "allowDiskUse": false,
- "showRecordId": true
- },
- "$readPreference": {
- "mode": "nearest",
- "tags": [ { "some": "other tag" }, { "some": "tag" } ],
- "hedge": { "enabled": true }
- },
- "collectionType": "collection",
- "hint": {
- "HASH<z>": 1,
- "HASH<c>": 1
- },
- "maxTimeMS": "?number",
- "allowPartialResults": false,
- "noCursorTimeout": false,
- "batchSize": "?number"
- })",
- key);
-}
-
-TEST_F(QueryStatsStoreTest, CorrectlyRedactsTailableFindCommandRequest) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
-
- FindCommandRequest fcr(NamespaceStringOrUUID(NamespaceString("testDB.testColl")));
- fcr.setAwaitData(true);
- fcr.setTailable(true);
- fcr.setSort(BSON("$natural" << 1));
- auto key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "find",
- "filter": {},
- "tailable": true,
- "awaitData": true
- },
- "collectionType": "collection",
- "hint": {
- "$natural": 1
- }
- })",
- key);
-}
-
-TEST_F(QueryStatsStoreTest, CorrectlyRedactsFindCommandRequestEmptyFields) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
- FindCommandRequest fcr(NamespaceStringOrUUID(NamespaceString("testDB.testColl")));
- fcr.setFilter(BSONObj());
- fcr.setSort(BSONObj());
- fcr.setProjection(BSONObj());
-
- auto hmacApplied = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "find",
- "filter": {}
- },
- "collectionType": "collection"
- })",
- hmacApplied); // NOLINT (test auto-update)
-}
-
-TEST_F(QueryStatsStoreTest, CorrectlyRedactsHintsWithOptions) {
- auto expCtx = make_intrusive<ExpressionContextForTest>();
- FindCommandRequest fcr(NamespaceStringOrUUID(NamespaceString("testDB.testColl")));
-
- fcr.setFilter(BSON("b" << 1));
- fcr.setHint(BSON("z" << 1 << "c" << 1));
- fcr.setMax(BSON("z" << 25));
- fcr.setMin(BSON("z" << 80));
-
- auto key = makeQueryStatsKeyFindRequest(fcr, expCtx, false);
-
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "testDB",
- "coll": "testColl"
- },
- "command": "find",
- "filter": {
- "b": {
- "$eq": "?number"
- }
- },
- "max": {
- "z": "?number"
- },
- "min": {
- "z": "?number"
- }
- },
- "collectionType": "collection",
- "hint": {
- "z": 1,
- "c": 1
- }
- })",
- key);
- // Test with a string hint. Note that this is the internal representation of the string hint
- // generated at parse time.
- fcr.setHint(BSON("$hint"
- << "z"));
-
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, false);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "testDB",
- "coll": "testColl"
- },
- "command": "find",
- "filter": {
- "b": {
- "$eq": "?number"
- }
- },
- "max": {
- "z": "?number"
- },
- "min": {
- "z": "?number"
- }
- },
- "collectionType": "collection",
- "hint": {
- "$hint": "z"
- }
- })",
- key);
-
- fcr.setHint(BSON("z" << 1 << "c" << 1));
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "find",
- "filter": {
- "HASH<b>": {
- "$eq": "?number"
- }
- },
- "max": {
- "HASH<z>": "?number"
- },
- "min": {
- "HASH<z>": "?number"
- }
- },
- "collectionType": "collection",
- "hint": {
- "HASH<z>": 1,
- "HASH<c>": 1
- }
- })",
- key);
-
- // Test that $natural comes through unmodified.
- fcr.setHint(BSON("$natural" << -1));
- key = makeQueryStatsKeyFindRequest(fcr, expCtx, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "find",
- "filter": {
- "HASH<b>": {
- "$eq": "?number"
- }
- },
- "max": {
- "HASH<z>": "?number"
- },
- "min": {
- "HASH<z>": "?number"
- }
- },
- "collectionType": "collection",
- "hint": {
- "$natural": -1
- }
- })",
- key);
-}
-
-TEST_F(QueryStatsStoreTest, DefinesLetVariables) {
- // Test that the expression context we use to apply hmac will understand the 'let' part of
- // the find command while parsing the other pieces of the command.
-
- // Note that this ExpressionContext will not have the let variables defined - we expect the
- // 'makeQueryStatsKey' call to do that.
- auto opCtx = makeOperationContext();
- auto fcr = std::make_unique<FindCommandRequest>(NamespaceString("testDB.testColl"));
- fcr->setLet(BSON("var" << 2));
- fcr->setFilter(fromjson("{$expr: [{$eq: ['$a', '$$var']}]}"));
- fcr->setProjection(fromjson("{varIs: '$$var'}"));
-
- auto expCtx = make_intrusive<ExpressionContextForTest>(opCtx.get());
- expCtx->variables.seedVariablesWithLetParameters(expCtx.get(), *fcr->getLet());
- auto hmacApplied = makeQueryStatsKeyFindRequest(*fcr, expCtx, false);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "testDB",
- "coll": "testColl"
- },
- "let": {
- "var": "?number"
- },
- "command": "find",
- "filter": {
- "$expr": [
- {
- "$eq": [
- "$a",
- "$$var"
- ]
- }
- ]
- },
- "projection": {
- "varIs": "$$var",
- "_id": true
- }
- },
- "collectionType": "collection"
- })",
- hmacApplied);
-
- hmacApplied = makeQueryStatsKeyFindRequest(*fcr, expCtx, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "let": {
- "HASH<var>": "?number"
- },
- "command": "find",
- "filter": {
- "$expr": [
- {
- "$eq": [
- "$HASH<a>",
- "$$HASH<var>"
- ]
- }
- ]
- },
- "projection": {
- "HASH<varIs>": "$$HASH<var>",
- "HASH<_id>": true
- }
- },
- "collectionType": "collection"
- })",
- hmacApplied);
-}
-
-TEST_F(QueryStatsStoreTest, CorrectlyTokenizesAggregateCommandRequestAllFieldsSimplePipeline) {
- auto expCtx = make_intrusive<ExpressionContextForTest>(*kDefaultTestNss.nss());
- AggregateCommandRequest acr(*kDefaultTestNss.nss());
- auto matchStage = fromjson(R"({
- $match: {
- foo: { $in: ["a", "b"] },
- bar: { $gte: { $date: "2022-01-01T00:00:00Z" } }
- }
- })");
- auto unwindStage = fromjson("{$unwind: '$x'}");
- auto groupStage = fromjson(R"({
- $group: {
- _id: "$_id",
- c: { $first: "$d.e" },
- f: { $sum: 1 }
- }
- })");
- auto limitStage = fromjson("{$limit: 10}");
- auto outStage = fromjson(R"({$out: 'outColl'})");
- auto rawPipeline = {matchStage, unwindStage, groupStage, limitStage, outStage};
- acr.setPipeline(rawPipeline);
- auto pipeline = Pipeline::parse(rawPipeline, expCtx);
-
- auto shapified = makeQueryStatsKeyAggregateRequest(
- acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$match": {
- "$and": [
- {
- "HASH<foo>": {
- "$in": "?array<?string>"
- }
- },
- {
- "HASH<bar>": {
- "$gte": "?date"
- }
- }
- ]
- }
- },
- {
- "$unwind": {
- "path": "$HASH<x>"
- }
- },
- {
- "$group": {
- "_id": "$HASH<_id>",
- "HASH<c>": {
- "$first": "$HASH<d>.HASH<e>"
- },
- "HASH<f>": {
- "$sum": "?number"
- }
- }
- },
- {
- "$limit": "?number"
- },
- {
- "$out": {
- "coll": "HASH<outColl>",
- "db": "HASH<testDB>"
- }
- }
- ]
- },
- "collectionType": "collection"
- })",
- shapified);
-
- // Add the fields that shouldn't be abstracted.
- acr.setAllowDiskUse(false);
- acr.setHint(BSON("z" << 1 << "c" << 1));
- acr.setCollation(BSON("locale"
- << "simple"));
- shapified = makeQueryStatsKeyAggregateRequest(
- acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "collation": {
- "locale": "simple"
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$match": {
- "$and": [
- {
- "HASH<foo>": {
- "$in": "?array<?string>"
- }
- },
- {
- "HASH<bar>": {
- "$gte": "?date"
- }
- }
- ]
- }
- },
- {
- "$unwind": {
- "path": "$HASH<x>"
- }
- },
- {
- "$group": {
- "_id": "$HASH<_id>",
- "HASH<c>": {
- "$first": "$HASH<d>.HASH<e>"
- },
- "HASH<f>": {
- "$sum": "?number"
- }
- }
- },
- {
- "$limit": "?number"
- },
- {
- "$out": {
- "coll": "HASH<outColl>",
- "db": "HASH<testDB>"
- }
- }
- ],
- "allowDiskUse": false
- },
- "collectionType": "collection",
- "hint": {
- "HASH<z>": 1,
- "HASH<c>": 1
- }
- })",
- shapified);
-
- // Add let.
- acr.setLet(BSON("var1" << BSON("$literal"
- << "$foo")
- << "var2"
- << "bar"));
- shapified = makeQueryStatsKeyAggregateRequest(
- acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "collation": {
- "locale": "simple"
- },
- "let": {
- "HASH<var1>": "?string",
- "HASH<var2>": "?string"
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$match": {
- "$and": [
- {
- "HASH<foo>": {
- "$in": "?array<?string>"
- }
- },
- {
- "HASH<bar>": {
- "$gte": "?date"
- }
- }
- ]
- }
- },
- {
- "$unwind": {
- "path": "$HASH<x>"
- }
- },
- {
- "$group": {
- "_id": "$HASH<_id>",
- "HASH<c>": {
- "$first": "$HASH<d>.HASH<e>"
- },
- "HASH<f>": {
- "$sum": "?number"
- }
- }
- },
- {
- "$limit": "?number"
- },
- {
- "$out": {
- "coll": "HASH<outColl>",
- "db": "HASH<testDB>"
- }
- }
- ],
- "allowDiskUse": false
- },
- "collectionType": "collection",
- "hint": {
- "HASH<z>": 1,
- "HASH<c>": 1
- }
- })",
- shapified);
-
- // Add the fields that should be abstracted.
- auto cursorOptions = SimpleCursorOptions();
- cursorOptions.setBatchSize(10);
- acr.setCursor(cursorOptions);
- acr.setMaxTimeMS(500);
- acr.setBypassDocumentValidation(true);
- expCtx->opCtx->setComment(BSON("comment"
- << "note to self"));
- shapified = makeQueryStatsKeyAggregateRequest(
- acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "collation": {
- "locale": "simple"
- },
- "let": {
- "HASH<var1>": "?string",
- "HASH<var2>": "?string"
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$match": {
- "$and": [
- {
- "HASH<foo>": {
- "$in": "?array<?string>"
- }
- },
- {
- "HASH<bar>": {
- "$gte": "?date"
- }
- }
- ]
- }
- },
- {
- "$unwind": {
- "path": "$HASH<x>"
- }
- },
- {
- "$group": {
- "_id": "$HASH<_id>",
- "HASH<c>": {
- "$first": "$HASH<d>.HASH<e>"
- },
- "HASH<f>": {
- "$sum": "?number"
- }
- }
- },
- {
- "$limit": "?number"
- },
- {
- "$out": {
- "coll": "HASH<outColl>",
- "db": "HASH<testDB>"
- }
- }
- ],
- "allowDiskUse": false
- },
- "comment": "?string",
- "collectionType": "collection",
- "hint": {
- "HASH<z>": 1,
- "HASH<c>": 1
- },
- "maxTimeMS": "?number",
- "bypassDocumentValidation": true,
- "cursor": {
- "batchSize": "?number"
- }
- })",
- shapified);
-
- // Test again but with the representative query shape.
- shapified = makeQueryStatsKeyAggregateRequest(
- acr, *pipeline, expCtx, LiteralSerializationPolicy::kToRepresentativeParseableValue, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "collation": {
- "locale": "simple"
- },
- "let": {
- "HASH<var1>": {
- "$const": "?"
- },
- "HASH<var2>": {
- "$const": "?"
- }
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$match": {
- "$and": [
- {
- "HASH<foo>": {
- "$in": [
- "?"
- ]
- }
- },
- {
- "HASH<bar>": {
- "$gte": {"$date":"1970-01-01T00:00:00.000Z"}
- }
- }
- ]
- }
- },
- {
- "$unwind": {
- "path": "$HASH<x>"
- }
- },
- {
- "$group": {
- "_id": "$HASH<_id>",
- "HASH<c>": {
- "$first": "$HASH<d>.HASH<e>"
- },
- "HASH<f>": {
- "$sum": {
- "$const": 1
- }
- }
- }
- },
- {
- "$limit": 1
- },
- {
- "$out": {
- "coll": "HASH<outColl>",
- "db": "HASH<testDB>"
- }
- }
- ],
- "allowDiskUse": false
- },
- "comment": "?",
- "collectionType": "collection",
- "hint": {
- "HASH<z>": 1,
- "HASH<c>": 1
- },
- "maxTimeMS": 1,
- "bypassDocumentValidation": true,
- "cursor": {
- "batchSize": 1
- }
- })",
- shapified);
-}
-
-TEST_F(QueryStatsStoreTest, CorrectlyTokenizesAggregateCommandRequestEmptyFields) {
- auto expCtx = make_intrusive<ExpressionContextForTest>(*kDefaultTestNss.nss());
- AggregateCommandRequest acr(*kDefaultTestNss.nss());
- acr.setPipeline({});
- auto pipeline = Pipeline::parse({}, expCtx);
-
- auto shapified = makeQueryStatsKeyAggregateRequest(
- acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "aggregate",
- "pipeline": []
- },
- "collectionType": "collection"
- })",
- shapified); // NOLINT (test auto-update)
-
- // Test again with the representative query shape.
- shapified = makeQueryStatsKeyAggregateRequest(
- acr, *pipeline, expCtx, LiteralSerializationPolicy::kToRepresentativeParseableValue, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "aggregate",
- "pipeline": []
- },
- "collectionType": "collection"
- })",
- shapified); // NOLINT (test auto-update)
-}
-
-TEST_F(QueryStatsStoreTest,
- CorrectlyTokenizesAggregateCommandRequestPipelineWithSecondaryNamespaces) {
- auto expCtx = make_intrusive<ExpressionContextForTest>(*kDefaultTestNss.nss());
- auto nsToUnionWith = NamespaceString(expCtx->ns.db(), "otherColl");
- expCtx->addResolvedNamespaces({nsToUnionWith});
-
- AggregateCommandRequest acr(*kDefaultTestNss.nss());
- auto unionWithStage = fromjson(R"({
- $unionWith: {
- coll: "otherColl",
- pipeline: [{$match: {val: "foo"}}]
- }
- })");
- auto sortStage = fromjson("{$sort: {age: 1}}");
- auto rawPipeline = {unionWithStage, sortStage};
- acr.setPipeline(rawPipeline);
- auto pipeline = Pipeline::parse(rawPipeline, expCtx);
-
- auto shapified = makeQueryStatsKeyAggregateRequest(
- acr, *pipeline, expCtx, LiteralSerializationPolicy::kToDebugTypeString, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$unionWith": {
- "coll": "HASH<otherColl>",
- "pipeline": [
- {
- "$match": {
- "HASH<val>": {
- "$eq": "?string"
- }
- }
- }
- ]
- }
- },
- {
- "$sort": {
- "HASH<age>": 1
- }
- }
- ]
- },
- "collectionType": "collection",
- "otherNss": [
- {
- "db": "HASH<testDB>",
- "coll": "HASH<otherColl>"
- }
- ]
- })",
- shapified);
-
- // Do the same thing with the representative query shape.
- shapified = makeQueryStatsKeyAggregateRequest(
- acr, *pipeline, expCtx, LiteralSerializationPolicy::kToRepresentativeParseableValue, true);
- ASSERT_BSONOBJ_EQ_AUTO( // NOLINT
- R"({
- "queryShape": {
- "cmdNs": {
- "db": "HASH<testDB>",
- "coll": "HASH<testColl>"
- },
- "command": "aggregate",
- "pipeline": [
- {
- "$unionWith": {
- "coll": "HASH<otherColl>",
- "pipeline": [
- {
- "$match": {
- "HASH<val>": {
- "$eq": "?"
- }
- }
- }
- ]
- }
- },
- {
- "$sort": {
- "HASH<age>": 1
- }
- }
- ]
- },
- "collectionType": "collection",
- "otherNss": [
- {
- "db": "HASH<testDB>",
- "coll": "HASH<otherColl>"
- }
- ]
- })",
- shapified);
-}
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/query_stats_test.cpp b/src/mongo/db/query/query_stats/query_stats_test.cpp
deleted file mode 100644
index 366716afdf6..00000000000
--- a/src/mongo/db/query/query_stats/query_stats_test.cpp
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/bson/bsonobj.h"
-#include "mongo/db/collection_type.h"
-#include "mongo/db/namespace_string.h"
-#include "mongo/db/operation_context.h"
-#include "mongo/db/pipeline/expression_context.h"
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/parsed_find_command.h"
-#include "mongo/db/query/query_stats/find_key.h"
-#include "mongo/db/query/query_stats/query_stats.h"
-#include "mongo/db/service_context_test_fixture.h"
-#include "mongo/idl/server_parameter_test_util.h"
-#include "mongo/unittest/unittest.h"
-#include "mongo/util/assert_util.h"
-
-#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQueryStats
-
-namespace mongo::query_stats {
-class QueryStatsTest : public ServiceContextTest {};
-
-TEST_F(QueryStatsTest, TwoRegisterRequestsWithSameOpCtxRateLimitedFirstCall) {
- // This test simulates what happens with queries over views where two calls to registerRequest()
- // can be made with the same opCtx.
-
- // Make query for query stats.
- const NamespaceString nss = NamespaceString("testDB.testColl");
- FindCommandRequest fcr((NamespaceStringOrUUID(nss)));
- fcr.setFilter(BSONObj());
-
- auto fcrCopy = std::make_unique<FindCommandRequest>(fcr);
- auto opCtx = makeOperationContext();
- auto expCtx = make_intrusive<ExpressionContextForTest>();
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrCopy)}));
-
- auto& opDebug = CurOp::get(*opCtx)->debug();
- ASSERT_EQ(opDebug.queryStatsInfo.wasRateLimited, false);
-
- // First call to registerRequest() should be rate limited.
- QueryStatsStoreManager::getRateLimiter(opCtx->getServiceContext()) =
- std::make_unique<RateLimiting>(0, Seconds{1});
- ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() {
- return std::make_unique<query_stats::FindKey>(
- expCtx, *parsedFind, query_shape::CollectionType::kCollection);
- }));
-
- // Since the query was rate limited, no key should have been created.
- ASSERT(opDebug.queryStatsInfo.key == nullptr);
- ASSERT_EQ(opDebug.queryStatsInfo.wasRateLimited, true);
-
- // Second call should not be rate limited.
- QueryStatsStoreManager::getRateLimiter(opCtx->getServiceContext())
- .get()
- ->setSamplingRate(INT_MAX);
-
- ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() {
- return std::make_unique<query_stats::FindKey>(
- expCtx, *parsedFind, query_shape::CollectionType::kCollection);
- }));
-
- // queryStatsKey should not be created for previously rate limited query.
- ASSERT(opDebug.queryStatsInfo.key == nullptr);
- ASSERT_EQ(opDebug.queryStatsInfo.wasRateLimited, true);
- ASSERT_FALSE(opDebug.queryStatsInfo.keyHash.has_value());
-}
-
-TEST_F(QueryStatsTest, TwoRegisterRequestsWithSameOpCtxDisabledBetween) {
- // This test simulates an observed bug where an opCtx is used for two requests, and between the
- // first and the second the query stats store is emptied/disabled.
-
- // Make query for query stats.
- const NamespaceString nss = NamespaceString("testDB.testColl");
- FindCommandRequest fcr((NamespaceStringOrUUID(nss)));
- fcr.setFilter(BSONObj());
-
- auto serviceCtx = getServiceContext();
- auto opCtx = makeOperationContext();
-
- auto& opDebug = CurOp::get(*opCtx)->debug();
- ASSERT(opDebug.queryStatsInfo.key == nullptr);
- ASSERT_FALSE(opDebug.queryStatsInfo.keyHash.has_value());
- QueryStatsStoreManager::get(serviceCtx) =
- std::make_unique<QueryStatsStoreManager>(16 * 1024 * 1024, 1);
-
- QueryStatsStoreManager::getRateLimiter(serviceCtx) =
- std::make_unique<RateLimiting>(-1, Seconds{1});
-
- {
- auto fcrCopy = std::make_unique<FindCommandRequest>(fcr);
- auto expCtx = make_intrusive<ExpressionContext>(
- opCtx.get(), *fcrCopy, nullptr, true /* mayDbProfile*/);
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrCopy)}));
- ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() {
- return std::make_unique<query_stats::FindKey>(
- expCtx, *parsedFind, query_shape::CollectionType::kCollection);
- }));
-
- ASSERT(opDebug.queryStatsInfo.key != nullptr);
- ASSERT(opDebug.queryStatsInfo.keyHash.has_value());
-
- ASSERT_DOES_NOT_THROW(query_stats::writeQueryStats(opCtx.get(),
- opDebug.queryStatsInfo.keyHash,
- std::move(opDebug.queryStatsInfo.key),
- 0 /*queryExecMicros*/,
- 0 /*firstResponseExecMicros*/,
- 0 /*docsReturned*/));
- }
-
- // Second call should see that query stats are now disabled.
- {
- // To reproduce SERVER-84730 we need to clear out the query stats store so that writing the
- // stats at the end will attempt to insert a new entry.
- QueryStatsStoreManager::get(serviceCtx)->resetSize(0);
-
- auto fcrCopy = std::make_unique<FindCommandRequest>(fcr);
- fcrCopy->setFilter(BSON("x" << 1));
- auto expCtx = make_intrusive<ExpressionContext>(
- opCtx.get(), *fcrCopy, nullptr, true /* mayDbProfile*/);
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, {std::move(fcrCopy)}));
-
- ASSERT_DOES_NOT_THROW(query_stats::registerRequest(opCtx.get(), nss, [&]() {
- return std::make_unique<query_stats::FindKey>(
- expCtx, *parsedFind, query_shape::CollectionType::kCollection);
- }));
-
- // queryStatsKey should not be created since we have a size budget of 0.
- ASSERT(opDebug.queryStatsInfo.key == nullptr);
- // This is not a rate limit, but rather a lack of space rendering it entirely disabled.
- ASSERT_FALSE(opDebug.queryStatsInfo.wasRateLimited);
-
- // Interestingly, we purposefully leave the hash value around on the OperationContext after
- // the previous operation finishes. This is because we think it may have value in being
- // logged in the future, even after query stats have been written. Excepting obscure
- // internal use-cases, most OperationContexts will die shortly after the query stats are
- // written, so this isn't expected to be a large issue.
- ASSERT(opDebug.queryStatsInfo.keyHash.has_value());
-
- QueryStatsStoreManager::get(serviceCtx)->resetSize(16 * 1024 * 1024);
- // SERVER-84730 this assertion used to throw since there is no key, but there is a hash.
- ASSERT_DOES_NOT_THROW(query_stats::writeQueryStats(opCtx.get(),
- opDebug.queryStatsInfo.keyHash,
- std::move(opDebug.queryStatsInfo.key),
- 0 /*queryExecMicros*/,
- 0 /*firstResponseExecMicros*/,
- 0 /*docsReturned*/));
- }
-}
-} // namespace mongo::query_stats
diff --git a/src/mongo/db/query/query_stats/rate_limiting.cpp b/src/mongo/db/query/query_stats/rate_limiting.cpp
deleted file mode 100644
index aa8ca645bf1..00000000000
--- a/src/mongo/db/query/query_stats/rate_limiting.cpp
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "rate_limiting.h"
-#include "mongo/stdx/mutex.h"
-#include "mongo/util/clock_source.h"
-
-namespace mongo {
-RateLimiting::RateLimiting(RequestCount samplingRate,
- Milliseconds timePeriod,
- ClockSource* clockSource)
- : _clockSource(clockSource != nullptr ? clockSource : SystemClockSource::get()),
- _samplingRate(samplingRate),
- _timePeriod(timePeriod),
- _windowStart(_clockSource->now()),
- _prevCount(0),
- _currentCount(0) {}
-
-Date_t RateLimiting::tickWindow() {
- Date_t currentTime = _clockSource->now();
-
- // Elapsed time since window start exceeds the time period. Start a new window.
- if (currentTime - _windowStart > _timePeriod) {
- _windowStart = currentTime;
- _prevCount = _currentCount;
- _currentCount = 0;
- }
- return currentTime;
-}
-
-bool RateLimiting::handleRequestFixedWindow() {
- stdx::unique_lock windowLock{_windowMutex};
- tickWindow();
-
- if (_currentCount < _samplingRate.load()) {
- _currentCount += 1;
- return true;
- }
- return false;
-}
-
-bool RateLimiting::handleRequestSlidingWindow() {
- stdx::unique_lock windowLock{_windowMutex};
-
- Date_t currentTime = tickWindow();
- auto windowStart = _windowStart;
- auto prevCount = _prevCount;
-
- // Sliding window is implemented over fixed size time periods/blocks as follows. Instead of
- // making the decision to limit the rate using only the current time period, we look to the rate
- // of the previous period to predicate the rate of the current. This smooths the "sampling" of
- // the events by predicting a constant rate and limiting accordingly.
-
- // Percentage of time remaining in current window.
- double percentRemainingOfCurrentWindow =
- ((double)(_timePeriod.count() - (currentTime - windowStart).count())) / _timePeriod.count();
- // Estimate the number of requests remaining in the current period. We assume the requests in
- // the previous time block occurred at a constant rate. We multiply the total number of requests
- // in the previous period by the percentage of time remaining in the current period.
- double estimatedRemaining = prevCount * percentRemainingOfCurrentWindow;
- // Add this estimate to the requests we know have taken place within the current time block.
- double estimatedCount = _currentCount + estimatedRemaining;
-
- if (estimatedCount < _samplingRate.load()) {
- _currentCount += 1;
- return true;
- }
- return false;
-}
-} // namespace mongo
diff --git a/src/mongo/db/query/query_stats/rate_limiting.h b/src/mongo/db/query/query_stats/rate_limiting.h
deleted file mode 100644
index 66e38d7119b..00000000000
--- a/src/mongo/db/query/query_stats/rate_limiting.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/util/clock_source.h"
-#include "mongo/util/concurrency/mutex.h"
-#include "mongo/util/system_clock_source.h"
-
-namespace mongo {
-
-/**
- * Rate limiting is used to put a bound on the number of requests to a certain resource over a fixed
- * time window. This implementation is approximate in the sense that it may permit the bound to
- * exceeded. The bound is approximate as a trade off to reduce contention on internal resources.
- */
-class RateLimiting {
- using RequestCount = uint32_t;
-
-public:
- /*
- * Constructor for a rate limiter. Specify the number of requests you want to take place, as
- * well as the time period in milliseconds.
- */
- RateLimiting(RequestCount samplingRate,
- Milliseconds timePeriod = Seconds{1},
- ClockSource* clockSource = nullptr);
-
- /*
- * Getter for the sampling rate.
- */
- RequestCount getSamplingRate() {
- return _samplingRate.load();
- }
-
- /*
- * Setter for the sampling rate.
- */
- void setSamplingRate(RequestCount samplingRate) {
- _samplingRate.store(samplingRate);
- }
-
- /*
- * A simple method for rate limiting. Returns false if we have reached the request limit for the
- * current time window; otherwise, returns true and adds the request to the count for the
- * current window. If we have passed the end of the previous window, the slate is wiped clean.
- */
- bool handleRequestFixedWindow();
-
- /*
- * A method that ensures a more steady rate of requests. Rather than only looking at the current
- * time block, this method simulates a sliding window to estimate how many requests occurred in
- * the last full time period. Like the above, returns whether the request should be handled, and
- * resets the window if enough time has passed.
- */
- bool handleRequestSlidingWindow();
-
-private:
- /*
- * Resets the current window if it has ended. Returns the current time. This must be called in
- * the beginning of each handleRequest...() method.
- */
- Date_t tickWindow();
-
- /*
- * Clock source used to track time.
- */
- ClockSource* const _clockSource;
-
- /*
- * Sampling rate is the bound on the number of requests we want to admit per window.
- */
- AtomicWord<RequestCount> _samplingRate;
-
- /*
- * Time period is the window size in ms.
- */
- const Milliseconds _timePeriod;
-
- /*
- * Window start.
- */
- Date_t _windowStart;
-
- /*
- * Count of requests handled in the previous window.
- */
- RequestCount _prevCount;
-
- /*
- * Count of requests handled in the current window.
- */
- RequestCount _currentCount;
-
- /*
- * Mutex used when reading/writing the window.
- */
- SimpleMutex _windowMutex;
-};
-} // namespace mongo
diff --git a/src/mongo/db/query/query_stats/rate_limiting_bm.cpp b/src/mongo/db/query/query_stats/rate_limiting_bm.cpp
deleted file mode 100644
index 06308e6b0d8..00000000000
--- a/src/mongo/db/query/query_stats/rate_limiting_bm.cpp
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-
-#include <benchmark/benchmark.h>
-#include <climits>
-#include <memory>
-
-#include "mongo/bson/json.h"
-#include "mongo/db/matcher/expression_leaf.h"
-#include "mongo/db/matcher/expression_parser.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_stats/rate_limiting.h"
-#include "mongo/util/duration.h"
-#include "mongo/util/processinfo.h"
-#include "mongo/util/time_support.h"
-
-namespace mongo {
-namespace {
-
-// Local testing determined that these parameter values drove the most lock contention, which is
-// what we want to capture in this benchmark.
-constexpr long long rateLimitedWorkTimeMicros = 5;
-constexpr long long consistentWorkTimeMicros = 10;
-
-constexpr long long numThreads = 256;
-
-// Rate limit some fraction of the overall work for a request with a sliding window.
-int requestWithSlidingWindow(RateLimiting& limit) {
- if (limit.handleRequestSlidingWindow()) {
- sleepmicros(rateLimitedWorkTimeMicros);
- }
- sleepmicros(consistentWorkTimeMicros);
- return 0;
-}
-
-// Represent a request that bypasses the rate limiter.
-int requestUnlimited() {
- constexpr long long totalTime = rateLimitedWorkTimeMicros + consistentWorkTimeMicros;
- sleepmicros(totalTime);
- return 0;
-}
-
-// Represent a request without the rate limited work.
-int requestDeactivated() {
- sleepmicros(consistentWorkTimeMicros);
- return 0;
-}
-
-// Benchmark sliding window rate limiting.
-void BM_SlidingWindow(benchmark::State& state) {
- // The rate limiter needs a clock source passed in.
- static std::unique_ptr<ClockSource> clockSource;
- static std::unique_ptr<RateLimiting> rateLimit;
-
- // Initialize the rate limiter only on the first thread to start up.
- if (state.thread_index == 0) {
- clockSource = std::make_unique<SystemClockSource>();
- rateLimit =
- std::make_unique<RateLimiting>(state.range(0), Milliseconds(1), clockSource.get());
- }
-
- // Run the benchmark.
- for (auto keepRunning : state) {
- benchmark::DoNotOptimize(requestWithSlidingWindow(*rateLimit));
- }
-
- // Clean up the rate limiter when the benchmark is done.
- if (state.thread_index == 0) {
- rateLimit.reset();
- clockSource.reset();
- }
-}
-
-// "Control" benchmark that does not rate limit requests. In other words, the extra work is always
-// done for every request. This benchmark can be thought of as the "goal" performance for the peak,
-// or the highest rate limit in BM_SlidingWindow, to compare against.
-void BM_Unlimited(benchmark::State& state) {
- for (auto keepRunning : state) {
- benchmark::DoNotOptimize(requestUnlimited());
- }
-}
-// Another control benchmark, where the extra work is never done for any request. This can be
-// thought of as the goal performance for when rate limit equals 0.
-void BM_Deactivated(benchmark::State& state) {
- for (auto keepRunning : state) {
- benchmark::DoNotOptimize(requestDeactivated());
- }
-}
-
-// Google microbenchmarks report time T (in nanoseconds) spent per operation. But at Mongo we are
-// interested in total opereations performed per second. The former can easily be converted to the
-// latter by diving 10^6 by T. Use this benchmark to determine the natural throughput of the
-// operation. This can be compared to the rate limited benchmarks (BM_SlidingWindow) to determine
-// the overhead of rate limiting. Looking at the percentage change in throughput between the control
-// benchmarks and the rate limited benchmark, will indicate how much overhead is due to lock
-// contention.
-BENCHMARK(BM_Unlimited)->Threads(numThreads);
-
-BENCHMARK(BM_Deactivated)->Threads(numThreads);
-
-// Local testing has confirmed that the higher the rate limit, the worse the throughput. This makes
-// sense as putting a higher upper bound on number of requests allowed in a given time period, means
-// longer wait times for the lock.
-BENCHMARK(BM_SlidingWindow)
- ->ArgName("rate limit")
- ->Arg(0)
- ->Arg(64)
- ->Arg(128)
- ->Arg(256)
- ->Arg(512)
- ->Arg(1024)
- ->Arg(2048)
- ->Arg(4816)
- ->Threads(numThreads);
-
-} // namespace
-} // namespace mongo
diff --git a/src/mongo/db/query/query_stats/rate_limiting_test.cpp b/src/mongo/db/query/query_stats/rate_limiting_test.cpp
deleted file mode 100644
index 380636a2a20..00000000000
--- a/src/mongo/db/query/query_stats/rate_limiting_test.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/query_stats/rate_limiting.h"
-#include "mongo/unittest/unittest.h"
-#include "mongo/util/time_support.h"
-
-namespace mongo {
-TEST(RateLimitingTest, FixedWindowSucceeds) {
- auto rl = RateLimiting(1);
- ASSERT_TRUE(rl.handleRequestFixedWindow());
-}
-
-TEST(RateLimitingTest, SlidingWindowSucceeds) {
- auto rl = RateLimiting(1);
- ASSERT_TRUE(rl.handleRequestSlidingWindow());
-}
-
-TEST(RateLimitingTest, FixedWindowFails) {
- auto rl = RateLimiting(0);
- ASSERT_FALSE(rl.handleRequestFixedWindow());
-}
-
-TEST(RateLimitingTest, SlidingWindowFails) {
- auto rl = RateLimiting(0);
- ASSERT_FALSE(rl.handleRequestSlidingWindow());
-}
-
-TEST(RateLimitingTest, FixedWindowSucceedsThenFails) {
- auto rl = RateLimiting(1, Hours{1});
- ASSERT_TRUE(rl.handleRequestFixedWindow());
- ASSERT_FALSE(rl.handleRequestFixedWindow());
- ASSERT_FALSE(rl.handleRequestFixedWindow());
-}
-
-TEST(RateLimitingTest, SlidingWindowSucceedsThenFails) {
- auto rl = RateLimiting(1, Hours{1});
- ASSERT_TRUE(rl.handleRequestSlidingWindow());
- ASSERT_FALSE(rl.handleRequestSlidingWindow());
- ASSERT_FALSE(rl.handleRequestSlidingWindow());
-}
-
-TEST(RateLimitingTest, FixedWindowPermitsRequestAfterWindowExpires) {
- auto rl = RateLimiting(1, Milliseconds{10});
- ASSERT_TRUE(rl.handleRequestFixedWindow());
- ASSERT_FALSE(rl.handleRequestFixedWindow());
- sleepmillis(11);
- ASSERT_TRUE(rl.handleRequestFixedWindow());
-}
-
-} // namespace mongo
diff --git a/src/mongo/db/query/query_stats/shapifying_bm.cpp b/src/mongo/db/query/query_stats/shapifying_bm.cpp
deleted file mode 100644
index fd7f605c855..00000000000
--- a/src/mongo/db/query/query_stats/shapifying_bm.cpp
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-
-#include <benchmark/benchmark.h>
-#include <climits>
-#include <memory>
-
-#include "mongo/bson/json.h"
-#include "mongo/db/concurrency/locker_noop_client_observer.h"
-#include "mongo/db/matcher/expression_leaf.h"
-#include "mongo/db/matcher/expression_parser.h"
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/query_shape/query_shape.h"
-#include "mongo/db/query/query_stats/find_key.h"
-#include "mongo/db/query/query_stats/query_stats.h"
-#include "mongo/db/query/query_stats/rate_limiting.h"
-#include "mongo/idl/server_parameter_test_util.h"
-#include "mongo/rpc/metadata/client_metadata.h"
-#include "mongo/util/duration.h"
-#include "mongo/util/processinfo.h"
-#include "mongo/util/testing_proctor.h"
-#include "mongo/util/time_support.h"
-
-namespace mongo {
-namespace {
-
-static const NamespaceStringOrUUID kDefaultTestNss =
- NamespaceStringOrUUID{NamespaceString("testDB.testColl")};
-
-static constexpr auto kCollectionType = query_shape::CollectionType::kCollection;
-
-// This is a snapshot of the client metadata generated from our IDHACK genny workload. The
-// specifics aren't so important, but it chosen in an attempt to be indicative of the size/shape
-// of this kind of thing "in the wild".
-const auto kMetadataWrapper = fromjson(R"({metadata: {
- "application" : {
- "name" : "Genny"
- },
- "driver" : {
- "name" : "mongoc / mongocxx",
- "version" : "1.23.2 / 3.7.0"
- },
- "os" : {
- "type" : "Linux",
- "name" : "Ubuntu",
- "version" : "22.04",
- "architecture" : "aarch64"
- },
- "platform" : "cfg=0x03215e88e9 posix=200809 stdc=201710 CC=GCC 11.3.0 CFLAGS=\"-fPIC\" LDFLAGS=\"\""
- }})");
-auto kMockClientMetadataElem = kMetadataWrapper["metadata"];
-
-auto makeFindKey(const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const ParsedFindCommand& parsedFind) {
- return std::make_unique<const query_stats::FindKey>(expCtx, parsedFind, kCollectionType);
-}
-
-int shapifyAndHashRequest(const boost::intrusive_ptr<ExpressionContext>& expCtx,
- const ParsedFindCommand& parsedFind) {
- auto key = makeFindKey(expCtx, parsedFind);
- [[maybe_unused]] auto hash = absl::Hash<query_stats::Key>{}(*key);
- return 0;
-}
-
-// Benchmark the performance of computing and hashing the query stats key for an IDHACK query.
-void BM_ShapfiyIDHack(benchmark::State& state) {
- auto serviceCtx = ServiceContext::make();
- serviceCtx->registerClientObserver(std::make_unique<LockerNoopClientObserver>());
-
- auto client = serviceCtx->makeClient("query_test");
-
- auto opCtx = client->makeOperationContext();
- auto expCtx = make_intrusive<ExpressionContextForTest>(opCtx.get());
- auto fcr = std::make_unique<FindCommandRequest>(expCtx->ns);
- fcr->setFilter(fromjson("{_id: 4}"));
- ClientMetadata::setFromMetadata(opCtx->getClient(), kMockClientMetadataElem);
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcr)));
-
- // Run the benchmark.
- for (auto keepRunning : state) {
- benchmark::DoNotOptimize(shapifyAndHashRequest(expCtx, *parsedFind));
- }
-}
-
-// Benchmark computing the query stats key and its hash for a mildly complex query predicate.
-void BM_ShapfiyMildlyComplex(benchmark::State& state) {
- auto serviceCtx = ServiceContext::make();
- serviceCtx->registerClientObserver(std::make_unique<LockerNoopClientObserver>());
-
- auto client = serviceCtx->makeClient("query_test");
-
- auto opCtx = client->makeOperationContext();
- auto expCtx = make_intrusive<ExpressionContextForTest>(opCtx.get());
- auto fcr = std::make_unique<FindCommandRequest>(expCtx->ns);
- fcr->setFilter(fromjson(R"({
- clientId: {$nin: ["432345", "4386945", "111111"]},
- nEmployees: {$gte: 4, $lt: 20},
- deactivated: false,
- region: "US",
- yearlySpend: {$lte: 1000}
- })"));
- ClientMetadata::setFromMetadata(opCtx->getClient(), kMockClientMetadataElem);
- auto parsedFind = uassertStatusOK(parsed_find_command::parse(expCtx, std::move(fcr)));
-
- // Run the benchmark.
- for (auto keepRunning : state) {
- benchmark::DoNotOptimize(shapifyAndHashRequest(expCtx, *parsedFind));
- }
-}
-
-BENCHMARK(BM_ShapfiyIDHack)->Threads(1);
-BENCHMARK(BM_ShapfiyMildlyComplex)->Threads(1);
-
-} // namespace
-} // namespace mongo
diff --git a/src/mongo/db/query/query_stats/transform_algorithm.idl b/src/mongo/db/query/query_stats/transform_algorithm.idl
deleted file mode 100644
index cd0a5ba43db..00000000000
--- a/src/mongo/db/query/query_stats/transform_algorithm.idl
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright (C) 2023-present MongoDB, Inc.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the Server Side Public License, version 1,
-# as published by MongoDB, Inc.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Server Side Public License for more details.
-#
-# You should have received a copy of the Server Side Public License
-# along with this program. If not, see
-# <http://www.mongodb.com/licensing/server-side-public-license>.
-#
-# As a special exception, the copyright holders give permission to link the
-# code of portions of this program with the OpenSSL library under certain
-# conditions as described in each individual source file and distribute
-# linked combinations including the program with the OpenSSL library. You
-# must comply with the Server Side Public License in all respects for
-# all of the code used other than as permitted herein. If you modify file(s)
-# with this exception, you may extend this exception to your version of the
-# file(s), but you are not obligated to do so. If you do not wish to do so,
-# delete this exception statement from your version. If you delete this
-# exception statement from all source files in the program, then also delete
-# it in the license file.
-#
-global:
- cpp_namespace: "mongo"
-
-enums:
- TransformAlgorithm:
- description: "The type of algorithm to be used for the transformIdentifiers field of $queryStats."
- type: string
- values:
- kHmacSha256: "hmac-sha-256"
- kNone: "none"
diff --git a/src/mongo/db/query/record_id_bound.h b/src/mongo/db/query/record_id_bound.h
index d6a37617a4f..99400ae938d 100644
--- a/src/mongo/db/query/record_id_bound.h
+++ b/src/mongo/db/query/record_id_bound.h
@@ -29,7 +29,6 @@
#pragma once
-#include <boost/operators.hpp>
#include <boost/optional.hpp>
#include <fmt/format.h>
#include <ostream>
@@ -45,7 +44,7 @@ namespace mongo {
/**
* A RecordId bound for a collection scan, with an optional BSON representation for pretty printing.
*/
-class RecordIdBound : boost::totally_ordered<RecordIdBound> {
+class RecordIdBound {
public:
RecordIdBound() = default;
@@ -82,14 +81,6 @@ public:
return _recordId.compare(rhs._recordId);
}
- bool operator==(const RecordIdBound& rhs) const {
- return compare(rhs) == 0;
- }
-
- bool operator<(const RecordIdBound& rhs) const {
- return compare(rhs) < 0;
- }
-
private:
RecordId _recordId;
boost::optional<BSONObj> _bson;
diff --git a/src/mongo/db/query/record_id_range.cpp b/src/mongo/db/query/record_id_range.cpp
deleted file mode 100644
index 88720183ef4..00000000000
--- a/src/mongo/db/query/record_id_range.cpp
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include <boost/optional.hpp>
-
-#include "mongo/db/query/record_id_range.h"
-
-namespace mongo {
-void RecordIdRange::maybeNarrowMin(const BSONObj& newMin, bool inclusive) {
- maybeNarrowMin(RecordIdBound(record_id_helpers::keyForObj(newMin), newMin), inclusive);
-}
-
-void RecordIdRange::maybeNarrowMin(const RecordIdBound& newMin, bool inclusive) {
- if (_min) {
- auto cmp = _min->compare(newMin);
- // The range only needs updating if:
- // * There's no existing _min
- // * The provided value is greater than the current _min
- // * The value == _min, but is _not_ inclusive, but the existing value is
-
- if (cmp > 0) {
- // Current min is strictly greater than the provided value (and existing value has been
- // initialised), nothing to do.
- return;
- }
-
- if (cmp == 0) {
- // Inclusivity moving true -> false narrows the range.
- _minInclusive = _minInclusive && inclusive;
- return;
- }
- }
- _min = newMin;
- // The bound value changed, so the previous value of _minInclusive is irrelevant.
- _minInclusive = inclusive;
-}
-
-void RecordIdRange::maybeNarrowMax(const BSONObj& newMax, bool inclusive) {
- maybeNarrowMax(RecordIdBound(record_id_helpers::keyForObj(newMax), newMax), inclusive);
-}
-
-void RecordIdRange::maybeNarrowMax(const RecordIdBound& newMax, bool inclusive) {
- if (_max) {
- auto cmp = _max->compare(newMax);
- // The range only needs updating if:
- // * There's no existing _max
- // * The provided value is less than the current _max
- // * The value == _max, but is _not_ inclusive, but the existing value is
-
- if (cmp < 0) {
- // Current max is strictly less than the provided value (and existing value has been
- // initialised), nothing to do.
- return;
- }
-
- if (cmp == 0) {
- // Inclusivity moving true -> false narrows the range.
- _maxInclusive = _maxInclusive && inclusive;
- return;
- }
- }
- _max = newMax;
- // The bound value changed, so the previous value of _maxInclusive is irrelevant.
- _maxInclusive = inclusive;
-}
-
-void RecordIdRange::intersectRange(const RecordIdRange& other) {
- intersectRange(other._min, other._max, other._minInclusive, other._maxInclusive);
-}
-
-void RecordIdRange::intersectRange(const boost::optional<RecordIdBound>& min,
- const boost::optional<RecordIdBound>& max,
- bool minInclusive,
- bool maxInclusive) {
- if (min) {
- maybeNarrowMin(*min, minInclusive);
- }
- if (max) {
- maybeNarrowMax(*max, maxInclusive);
- }
-}
-
-} // namespace mongo
diff --git a/src/mongo/db/query/record_id_range.h b/src/mongo/db/query/record_id_range.h
deleted file mode 100644
index d25e17356c1..00000000000
--- a/src/mongo/db/query/record_id_range.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-
-#include <boost/optional.hpp>
-
-#include "mongo/bson/bsonobj.h"
-#include "mongo/db/query/record_id_bound.h"
-
-namespace mongo {
-
-class RecordIdRange {
-public:
- /**
- * If the provided value @p newMin is greater than the existing min,
- * update the lower bound to equal @p newMin
- *
- * @return true if range was adjusted
- */
- void maybeNarrowMin(const BSONObj& newMin, bool inclusive);
- void maybeNarrowMin(const RecordIdBound& newMin, bool inclusive);
-
- /**
- * If the provided value @p newMax is less than the existing max,
- * update the upper bound to equal @p newMax
- *
- * @return true if range was adjusted
- */
- void maybeNarrowMax(const BSONObj& newMax, bool inclusive);
- void maybeNarrowMax(const RecordIdBound& newMax, bool inclusive);
-
- /**
- * Update this range to the intersection of this range
- * and @p other. This may update both, one of, or neither of
- * min and max.
- *
- * Results in a range which is either unchanged, or made
- * narrower (possibly becoming an empty range).
- */
- void intersectRange(const RecordIdRange& other);
- /**
- * Overload of intersectRange taking the components of a RecordIdRange,
- * for convenience when the other range is not handled as a RecordIdRange.
- */
- void intersectRange(const boost::optional<RecordIdBound>& min,
- const boost::optional<RecordIdBound>& max,
- bool minInclusive = true,
- bool maxInclusive = true);
-
- bool isEmpty() const;
-
-
- const auto& getMin() const {
- return _min;
- }
-
- const auto& getMax() const {
- return _max;
- }
-
- bool isMinInclusive() const {
- return _minInclusive;
- }
-
- bool isMaxInclusive() const {
- return _maxInclusive;
- }
-
-
-private:
- // If present, this parameter sets the start point of a forward scan or the end point of a
- // reverse scan.
- boost::optional<RecordIdBound> _min;
-
- // If present, this parameter sets the start point of a reverse scan or the end point of a
- // forward scan.
- boost::optional<RecordIdBound> _max;
-
- // TODO: investigate folding this into RecordIdBound; many other usages pair RecordIdBound
- // with ScanBoundInclusion to convey this information
- // If min is present, this indicates whether the range is inclusive or exclusive of the
- // set min value
- bool _minInclusive = true;
- // If max is present, this indicates whether the range is inclusive or exclusive of the
- // set max value
- bool _maxInclusive = true;
-};
-
-} // namespace mongo
diff --git a/src/mongo/db/query/record_id_range_test.cpp b/src/mongo/db/query/record_id_range_test.cpp
deleted file mode 100644
index 5db866a73d3..00000000000
--- a/src/mongo/db/query/record_id_range_test.cpp
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/base/error_extra_info.h"
-#include "mongo/bson/bsonmisc.h"
-#include "mongo/db/query/record_id_range.h"
-
-#include "mongo/bson/bsonmisc.h"
-#include "mongo/unittest/unittest.h"
-
-namespace {
-
-using namespace mongo;
-
-template <class BoundsCheck, class NarrowingCheck, class InclusivityCheck>
-void testRange(const BoundsCheck& getBoundOptional,
- const NarrowingCheck& maybeNarrowBound,
- const InclusivityCheck& checkInclusivity,
- int initialValue,
- int narrowerValue,
- int widerValue) {
- ASSERT_FALSE(getBoundOptional());
-
- auto assertValueEq = [&](auto value) {
- auto bson = BSON("value" << value);
- auto recordId = record_id_helpers::keyForObj(bson);
- ASSERT_EQ(recordId, getBoundOptional()->recordId());
- };
-
- // narrow from unset
- maybeNarrowBound(BSON("value" << initialValue), true /* inclusive */);
- ASSERT_TRUE(getBoundOptional());
- assertValueEq(initialValue);
- ASSERT_TRUE(checkInclusivity());
-
- // narrow by removing inclusivity of the bound
- maybeNarrowBound(BSON("value" << initialValue), false /* not inclusive */);
- ASSERT_TRUE(getBoundOptional());
- assertValueEq(initialValue);
- ASSERT_FALSE(checkInclusivity());
-
- // cannot widen by re-adding inclusivity
- maybeNarrowBound(BSON("value" << initialValue), true /* inclusive */);
- ASSERT_TRUE(getBoundOptional());
- assertValueEq(initialValue);
- ASSERT_FALSE(checkInclusivity());
-
- // cannot widen by setting a wider bound
- maybeNarrowBound(BSON("value" << widerValue), true /* inclusive */);
- ASSERT_TRUE(getBoundOptional());
- assertValueEq(initialValue);
- ASSERT_FALSE(checkInclusivity());
-
- // cannot widen by setting a wider bound, regardless of inclusivity
- maybeNarrowBound(BSON("value" << widerValue), false /* not inclusive */);
- ASSERT_TRUE(getBoundOptional());
- assertValueEq(initialValue);
- ASSERT_FALSE(checkInclusivity());
-
- // narrow to a non-inclusive bound at a narrower value
- maybeNarrowBound(BSON("value" << narrowerValue), false /* not inclusive */);
- ASSERT_TRUE(getBoundOptional());
- assertValueEq(narrowerValue);
- ASSERT_FALSE(checkInclusivity());
-}
-
-TEST(RecordIdRangeTest, NarrowMin) {
- RecordIdRange range;
-
- testRange([&] { return range.getMin(); },
- [&](const BSONObj& newVal, bool inclusive) {
- return range.maybeNarrowMin(newVal, inclusive);
- },
- [&] { return range.isMinInclusive(); },
- 10,
- 11,
- 9);
-}
-
-TEST(RecordIdRangeTest, NarrowMax) {
- RecordIdRange range;
-
- testRange([&] { return range.getMax(); },
- [&](const BSONObj& newVal, bool inclusive) {
- return range.maybeNarrowMax(newVal, inclusive);
- },
- [&] { return range.isMaxInclusive(); },
- 10,
- 9,
- 11);
-}
-
-} // namespace
diff --git a/src/mongo/db/query/sbe_cached_solution_planner.cpp b/src/mongo/db/query/sbe_cached_solution_planner.cpp
index cd7f23016eb..c594c70f22e 100644
--- a/src/mongo/db/query/sbe_cached_solution_planner.cpp
+++ b/src/mongo/db/query/sbe_cached_solution_planner.cpp
@@ -74,7 +74,6 @@ CandidatePlans CachedSolutionPlanner::plan(
std::move(roots[0].first),
std::move(roots[0].second),
maxReadsBeforeReplan);
-
auto explainer = plan_explainer_factory::make(
candidate.root.get(),
&candidate.data,
diff --git a/src/mongo/db/query/sbe_multi_planner.cpp b/src/mongo/db/query/sbe_multi_planner.cpp
index a0d6e975694..5f2104a0668 100644
--- a/src/mongo/db/query/sbe_multi_planner.cpp
+++ b/src/mongo/db/query/sbe_multi_planner.cpp
@@ -32,6 +32,7 @@
#include "mongo/db/query/sbe_multi_planner.h"
+#include "mongo/db/exec/multi_plan.h"
#include "mongo/db/exec/sbe/expressions/expression.h"
#include "mongo/db/exec/sbe/values/bson.h"
#include "mongo/db/query/collection_query_info.h"
diff --git a/src/mongo/db/query/sbe_plan_cache.cpp b/src/mongo/db/query/sbe_plan_cache.cpp
index 70d864e7074..b699387cc47 100644
--- a/src/mongo/db/query/sbe_plan_cache.cpp
+++ b/src/mongo/db/query/sbe_plan_cache.cpp
@@ -31,7 +31,7 @@
#include "mongo/db/query/sbe_plan_cache.h"
-#include "mongo/db/query/util/memory_util.h"
+#include "mongo/db/query/plan_cache_size_parameter.h"
#include "mongo/db/server_options.h"
#include "mongo/logv2/log.h"
#include "mongo/util/processinfo.h"
@@ -42,23 +42,71 @@ namespace {
const auto sbePlanCacheDecoration =
ServiceContext::declareDecoration<std::unique_ptr<sbe::PlanCache>>();
+size_t convertToSizeInBytes(const plan_cache_util::PlanCacheSizeParameter& param) {
+ constexpr size_t kBytesInMB = 1014 * 1024;
+ constexpr size_t kMBytesInGB = 1014;
+
+ double sizeInMB = param.size;
+
+ switch (param.units) {
+ case plan_cache_util::PlanCacheSizeUnits::kPercent:
+ sizeInMB *= ProcessInfo::getMemSizeMB() / 100.0;
+ break;
+ case plan_cache_util::PlanCacheSizeUnits::kMB:
+ break;
+ case plan_cache_util::PlanCacheSizeUnits::kGB:
+ sizeInMB *= kMBytesInGB;
+ break;
+ }
+
+ return static_cast<size_t>(sizeInMB * kBytesInMB);
+}
+
+/**
+ * Sets upper size limit on the PlanCache size to 500GB or 25% of the system's memory, whichever is
+ * smaller.
+ */
+size_t capPlanCacheSize(size_t planCacheSize) {
+ constexpr size_t kBytesInGB = 1024 * 1024 * 1024;
+
+ // Maximum size of the plan cache expressed in bytes.
+ constexpr size_t kMaximumPlanCacheSize = 500 * kBytesInGB;
+
+ // Maximum size of the plan cache expressed as a share of the memory available to the process.
+ const plan_cache_util::PlanCacheSizeParameter limitToProcessSize{
+ 25, plan_cache_util::PlanCacheSizeUnits::kPercent};
+ const size_t limitToProcessSizeInBytes = convertToSizeInBytes(limitToProcessSize);
+
+ // The size will be capped by the minimum of the two values defined above.
+ const size_t maxPlanCacheSize = std::min(kMaximumPlanCacheSize, limitToProcessSizeInBytes);
+
+ if (planCacheSize > maxPlanCacheSize) {
+ planCacheSize = maxPlanCacheSize;
+ LOGV2_DEBUG(6007000,
+ 1,
+ "The plan cache size has been capped",
+ "maxPlanCacheSize"_attr = maxPlanCacheSize);
+ }
+
+ return planCacheSize;
+}
+
+size_t getPlanCacheSizeInBytes(const plan_cache_util::PlanCacheSizeParameter& param) {
+ size_t planCacheSize = convertToSizeInBytes(param);
+ uassert(5968001,
+ "Cache size must be at least 1KB * number of cores",
+ planCacheSize >= 1024 * ProcessInfo::getNumCores());
+ return capPlanCacheSize(planCacheSize);
+}
class PlanCacheOnParamChangeUpdaterImpl final : public plan_cache_util::OnParamChangeUpdater {
public:
- void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) final {
+ void updateCacheSize(ServiceContext* serviceCtx,
+ plan_cache_util::PlanCacheSizeParameter parameter) final {
if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) {
- auto newSizeBytes = memory_util::getRequestedMemSizeInBytes(memSize);
- auto cappedCacheSize = memory_util::capMemorySize(newSizeBytes /*requestedSizeBytes*/,
- 500 /*maximumSizeGB*/,
- 25 /*percentTotalSystemMemory*/);
- if (cappedCacheSize < newSizeBytes) {
- LOGV2_DEBUG(6007001,
- 1,
- "The plan cache size has been capped",
- "cappedSize"_attr = cappedCacheSize);
- }
+ auto size = getPlanCacheSizeInBytes(parameter);
auto& globalPlanCache = sbePlanCacheDecoration(serviceCtx);
- globalPlanCache->reset(cappedCacheSize);
+ globalPlanCache->reset(size);
}
}
@@ -76,21 +124,12 @@ ServiceContext::ConstructorActionRegisterer planCacheRegisterer{
std::make_unique<PlanCacheOnParamChangeUpdaterImpl>();
if (feature_flags::gFeatureFlagSbePlanCache.isEnabledAndIgnoreFCV()) {
- auto status = memory_util::MemorySize::parse(planCacheSize.get());
+ auto status = plan_cache_util::PlanCacheSizeParameter::parse(planCacheSize.get());
uassertStatusOK(status);
- auto size = memory_util::getRequestedMemSizeInBytes(status.getValue());
- auto cappedCacheSize = memory_util::capMemorySize(size /*requestedSizeBytes*/,
- 500 /*maximumSizeGB*/,
- 25 /*percentTotalSystemMemory*/);
- if (cappedCacheSize < size) {
- LOGV2_DEBUG(6007000,
- 1,
- "The plan cache size has been capped",
- "cappedSize"_attr = cappedCacheSize);
- }
+
+ auto size = getPlanCacheSizeInBytes(status.getValue());
auto& globalPlanCache = sbePlanCacheDecoration(serviceCtx);
- globalPlanCache =
- std::make_unique<sbe::PlanCache>(cappedCacheSize, ProcessInfo::getNumCores());
+ globalPlanCache = std::make_unique<sbe::PlanCache>(size, ProcessInfo::getNumCores());
}
}};
diff --git a/src/mongo/db/query/sbe_plan_cache.h b/src/mongo/db/query/sbe_plan_cache.h
index 090df251bdf..6e7853fa817 100644
--- a/src/mongo/db/query/sbe_plan_cache.h
+++ b/src/mongo/db/query/sbe_plan_cache.h
@@ -190,14 +190,7 @@ struct CachedSbePlan {
using PlanCacheEntry = PlanCacheEntryBase<CachedSbePlan, plan_cache_debug_info::DebugInfoSBE>;
struct BudgetEstimator {
- /**
- * This estimator function is called when an entry is added or removed to LRU cache in order to
- * make sure the total plan cache size does not exceed the maximum size.
- */
- size_t operator()(const sbe::PlanCacheKey& key,
- const std::shared_ptr<const PlanCacheEntry>& entry) {
- // TODO: SERVER-73649 include size of underlying query shape and size of int_32 key hash in
- // total size estimation.
+ size_t operator()(const std::shared_ptr<const PlanCacheEntry>& entry) {
return entry->estimatedEntrySizeBytes;
}
};
diff --git a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.cpp b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.cpp
index 9f6cbef6d4f..399d5210128 100644
--- a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.cpp
+++ b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.cpp
@@ -56,7 +56,7 @@ Status clearSbeCacheOnParameterChangeHelper() {
}
Status onPlanCacheSizeUpdate(const std::string& str) {
- auto newSize = memory_util::MemorySize::parse(str);
+ auto newSize = PlanCacheSizeParameter::parse(str);
if (!newSize.isOK()) {
return newSize.getStatus();
}
@@ -73,7 +73,7 @@ Status onPlanCacheSizeUpdate(const std::string& str) {
}
Status validatePlanCacheSize(const std::string& str) {
- uasserted(7529500, "planCacheSize is not a supported parameter in 6.0");
+ return PlanCacheSizeParameter::parse(str).getStatus();
}
const Decorable<ServiceContext>::Decoration<std::unique_ptr<OnParamChangeUpdater>>
diff --git a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h
index d0f8a026a05..e126dda9f48 100644
--- a/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h
+++ b/src/mongo/db/query/sbe_plan_cache_on_parameter_change.h
@@ -32,7 +32,7 @@
#include <string>
#include "mongo/base/status.h"
-#include "mongo/db/query/util/memory_util.h"
+#include "mongo/db/query/plan_cache_size_parameter.h"
#include "mongo/db/service_context.h"
namespace mongo::plan_cache_util {
@@ -70,11 +70,11 @@ public:
virtual ~OnParamChangeUpdater() = default;
/**
- * Resizes the SBE plan cache decorating 'serviceCtx' to the new size given by 'memSize'. If
+ * Resizes the SBE plan cache decorating 'serviceCtx' to the new size given by 'parameter'. If
* the new cache size is smaller than the old, cache entries are evicted in order to ensure the
* cache fits within the new size bound.
*/
- virtual void updateCacheSize(ServiceContext* serviceCtx, memory_util::MemorySize memSize) = 0;
+ virtual void updateCacheSize(ServiceContext* serviceCtx, PlanCacheSizeParameter parameter) = 0;
/**
* Deletes all plans from the SBE plan cache decorating 'serviceCtx'.
diff --git a/src/mongo/db/query/sbe_stage_builder.cpp b/src/mongo/db/query/sbe_stage_builder.cpp
index 7a2b2a9bdef..d35fcc774b0 100644
--- a/src/mongo/db/query/sbe_stage_builder.cpp
+++ b/src/mongo/db/query/sbe_stage_builder.cpp
@@ -769,9 +769,12 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
}
// If the slots necessary for performing an index consistency check were not requested in
- // 'reqs', then set 'doIndexConsistencyCheck' to false to avoid generating unnecessary logic.
- bool doIndexConsistencyCheck =
- reqs.has(kSnapshotId) && reqs.has(kIndexId) && reqs.has(kIndexKey);
+ // 'reqs', then don't pass a pointer to 'iamMap' so 'generateIndexScan' doesn't generate the
+ // necessary slots.
+ auto iamMap = &_data.iamMap;
+ if (!(reqs.has(kSnapshotId) && reqs.has(kIndexId) && reqs.has(kIndexKey))) {
+ iamMap = nullptr;
+ }
const auto generateIndexScanFunc =
ixn->iets.empty() ? generateIndexScan : generateIndexScanWithDynamicBounds;
@@ -780,7 +783,7 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
ixn,
indexKeyBitset,
_yieldPolicy,
- doIndexConsistencyCheck,
+ iamMap,
reqs.has(kIndexKeyPattern));
if (reqs.has(PlanStageSlots::kReturnKey)) {
@@ -922,6 +925,7 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
auto [stage, outputs] = build(fn->children[0], childReqs);
+ auto iamMap = _data.iamMap;
uassert(4822880, "RecordId slot is not defined", outputs.has(kRecordId));
uassert(
4953600, "ReturnKey slot is not defined", !reqs.has(kReturnKey) || outputs.has(kReturnKey));
@@ -949,6 +953,7 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
outputs.get(kIndexKey),
outputs.get(kIndexKeyPattern),
getCurrentCollection(reqs),
+ std::move(iamMap),
root->nodeId(),
std::move(relevantSlots),
_slotIdGenerator);
@@ -1997,17 +2002,27 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
auto outerChild = andSortedNode->children[0];
auto innerChild = andSortedNode->children[1];
- auto outerChildReqs = childReqs.copy()
- .clear(kSnapshotId)
- .clear(kIndexId)
- .clear(kIndexKey)
- .clear(kIndexKeyPattern);
- auto [outerStage, outerOutputs] = build(outerChild, outerChildReqs);
+ auto [outerStage, outerOutputs] = build(outerChild, childReqs);
auto outerIdSlot = outerOutputs.get(kRecordId);
auto outerResultSlot = outerOutputs.get(kResult);
auto outerKeySlots = sbe::makeSV(outerIdSlot);
auto outerProjectSlots = sbe::makeSV(outerResultSlot);
+ if (outerOutputs.has(kSnapshotId)) {
+ outerProjectSlots.push_back(outerOutputs.get(kSnapshotId));
+ }
+
+ if (outerOutputs.has(kIndexId)) {
+ outerProjectSlots.push_back(outerOutputs.get(kIndexId));
+ }
+
+ if (outerOutputs.has(kIndexKey)) {
+ outerProjectSlots.push_back(outerOutputs.get(kIndexKey));
+ }
+
+ if (outerOutputs.has(kIndexKeyPattern)) {
+ outerProjectSlots.push_back(outerOutputs.get(kIndexKeyPattern));
+ }
auto [innerStage, innerOutputs] = build(innerChild, childReqs);
tassert(5073707, "innerOutputs must contain kRecordId slot", innerOutputs.has(kRecordId));
@@ -2085,104 +2100,44 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
namespace {
template <typename F>
-struct FieldPathAndCondPreVisitor : public SelectiveConstExpressionVisitorBase {
+struct FieldPathVisitor : public SelectiveConstExpressionVisitorBase {
// To avoid overloaded-virtual warnings.
using SelectiveConstExpressionVisitorBase::visit;
- FieldPathAndCondPreVisitor(const F& fn, int32_t& nestedCondLevel)
- : _fn(fn), _nestedCondLevel(nestedCondLevel) {}
+ FieldPathVisitor(const F& fn) : _fn(fn) {}
void visit(const ExpressionFieldPath* expr) final {
- _fn(expr, _nestedCondLevel);
- }
-
- void visit(const ExpressionCond* expr) final {
- ++_nestedCondLevel;
- }
-
- void visit(const ExpressionSwitch* expr) final {
- ++_nestedCondLevel;
- }
-
- void visit(const ExpressionIfNull* expr) final {
- ++_nestedCondLevel;
- }
-
- void visit(const ExpressionAnd* expr) final {
- ++_nestedCondLevel;
- }
-
- void visit(const ExpressionOr* expr) final {
- ++_nestedCondLevel;
+ _fn(expr);
}
F _fn;
- // Tracks the number of conditional expressions like $cond or $ifNull that are above us in the
- // tree.
- int32_t& _nestedCondLevel;
-};
-
-struct CondPostVisitor : public SelectiveConstExpressionVisitorBase {
- // To avoid overloaded-virtual warnings.
- using SelectiveConstExpressionVisitorBase::visit;
-
- CondPostVisitor(int32_t& nestedCondLevel) : _nestedCondLevel(nestedCondLevel) {}
-
- void visit(const ExpressionCond* expr) final {
- --_nestedCondLevel;
- }
-
- void visit(const ExpressionSwitch* expr) final {
- --_nestedCondLevel;
- }
-
- void visit(const ExpressionIfNull* expr) final {
- --_nestedCondLevel;
- }
-
- void visit(const ExpressionAnd* expr) final {
- --_nestedCondLevel;
- }
-
- void visit(const ExpressionOr* expr) final {
- --_nestedCondLevel;
- }
-
- int32_t& _nestedCondLevel;
};
/**
* Walks through the 'expr' expression tree and whenever finds an 'ExpressionFieldPath', calls
* the 'fn' function. Type requirement for 'fn' is it must have a const 'ExpressionFieldPath'
- * pointer parameter and 'nestedCondLevel' parameter.
+ * pointer parameter.
*/
template <typename F>
void walkAndActOnFieldPaths(Expression* expr, const F& fn) {
- int32_t nestedCondLevel = 0;
- FieldPathAndCondPreVisitor<F> preVisitor(fn, nestedCondLevel);
- CondPostVisitor postVisitor(nestedCondLevel);
- ExpressionWalker walker(&preVisitor, nullptr /*inVisitor*/, &postVisitor);
+ FieldPathVisitor<F> visitor(fn);
+ ExpressionWalker walker(&visitor, nullptr /*inVisitor*/, nullptr /*postVisitor*/);
expression_walker::walk(expr, &walker);
}
/**
* Checks whether all field paths in 'idExpr' and all accumulator expressions are top-level ones.
*/
-bool areAllFieldPathsOptimizable(const boost::intrusive_ptr<Expression>& idExpr,
- const std::vector<AccumulationStatement>& accStmts) {
- auto areFieldPathsOptimizable = true;
+bool checkAllFieldPathsAreTopLevel(const boost::intrusive_ptr<Expression>& idExpr,
+ const std::vector<AccumulationStatement>& accStmts) {
+ auto areAllTopLevelFields = true;
- auto checkFieldPath = [&](const ExpressionFieldPath* fieldExpr, int32_t nestedCondLevel) {
+ auto checkFieldPath = [&](const ExpressionFieldPath* fieldExpr) {
// We optimize neither a field path for the top-level document itself (getPathLength() == 1)
// nor a field path that refers to a variable. We can optimize only top-level fields
// (getPathLength() == 2).
- //
- // The 'nestedCondLevel' being > 0 means that a field path is refered to below conditional
- // expressions at the parent $group node, when we cannot optimize field path access and
- // therefore, cannot avoid materialization.
- if (nestedCondLevel > 0 || fieldExpr->getFieldPath().getPathLength() != 2 ||
- fieldExpr->isVariableReference()) {
- areFieldPathsOptimizable = false;
+ if (fieldExpr->getFieldPath().getPathLength() != 2 || fieldExpr->isVariableReference()) {
+ areAllTopLevelFields = false;
return;
}
};
@@ -2195,7 +2150,7 @@ bool areAllFieldPathsOptimizable(const boost::intrusive_ptr<Expression>& idExpr,
walkAndActOnFieldPaths(accStmt.expr.argument.get(), checkFieldPath);
}
- return areFieldPathsOptimizable;
+ return areAllTopLevelFields;
}
/**
@@ -2221,7 +2176,7 @@ EvalStage optimizeFieldPaths(StageBuilderState& state,
auto searchInChildOutputs = !optionalRootSlot.has_value();
auto retEvalStage = std::move(childEvalStage);
- walkAndActOnFieldPaths(expr.get(), [&](const ExpressionFieldPath* fieldExpr, int32_t) {
+ walkAndActOnFieldPaths(expr.get(), [&](const ExpressionFieldPath* fieldExpr) {
// We optimize neither a field path for the top-level document itself nor a field path that
// refers to a variable instead of calling getField().
if (fieldExpr->getFieldPath().getPathLength() == 1 || fieldExpr->isVariableReference()) {
@@ -2352,12 +2307,20 @@ std::tuple<sbe::value::SlotVector, EvalStage, std::unique_ptr<sbe::EExpression>>
nodeId,
slotIdGenerator);
- // The group-by field may end up being 'Nothing' and in that case _id: null will be
- // returned. Calling 'makeFillEmptyNull' for the group-by field takes care of that.
- auto fillEmptyNullExpr = makeFillEmptyNull(groupByEvalExpr.extractExpr());
sbe::value::SlotId slot;
- std::tie(slot, retEvalStage) = projectEvalExpr(
- std::move(fillEmptyNullExpr), std::move(groupByEvalStage), nodeId, slotIdGenerator);
+ if (auto isConstIdExpr = dynamic_cast<ExpressionConstant*>(idExpr.get()) != nullptr;
+ isConstIdExpr) {
+ std::tie(slot, retEvalStage) = projectEvalExpr(
+ std::move(groupByEvalExpr), std::move(groupByEvalStage), nodeId, slotIdGenerator);
+ } else {
+ // The group-by field may end up being 'Nothing' and in that case _id: null will be
+ // returned. Calling 'makeFillEmptyNull' for the group-by field takes care of that.
+ std::tie(slot, retEvalStage) =
+ projectEvalExpr(makeFillEmptyNull(groupByEvalExpr.extractExpr()),
+ std::move(groupByEvalStage),
+ nodeId,
+ slotIdGenerator);
+ }
return {sbe::value::SlotVector{slot}, std::move(retEvalStage), nullptr};
}
@@ -2369,7 +2332,7 @@ std::tuple<sbe::value::SlotVector, EvalStage> generateAccumulator(
const PlanStageSlots& childOutputs,
PlanNodeId nodeId,
sbe::value::SlotIdGenerator* slotIdGenerator,
- sbe::SlotExprPairVector& accSlotExprPairs) {
+ sbe::value::SlotMap<std::unique_ptr<sbe::EExpression>>& accSlotToExprMap) {
// Input fields may need field traversal which ends up being a complex tree.
auto evalStage = optimizeFieldPaths(
state, accStmt.expr.argument, std::move(childEvalStage), childOutputs, nodeId);
@@ -2380,54 +2343,17 @@ std::tuple<sbe::value::SlotVector, EvalStage> generateAccumulator(
// One accumulator may be translated to multiple accumulator expressions. For example, The
// $avg will have two accumulators expressions, a sum(..) and a count which is implemented
// as sum(1).
- auto collatorSlot = state.data->env->getSlotIfExists("collator"_sd);
- auto accExprs = stage_builder::buildAccumulator(
- accStmt, std::move(argExpr), collatorSlot, *state.frameIdGenerator);
+ auto [accExprs, accProjEvalStage] = stage_builder::buildAccumulator(
+ state, accStmt, std::move(accArgEvalStage), std::move(argExpr), nodeId);
sbe::value::SlotVector aggSlots;
for (auto& accExpr : accExprs) {
auto slot = slotIdGenerator->generate();
aggSlots.push_back(slot);
- accSlotExprPairs.push_back({slot, std::move(accExpr)});
+ accSlotToExprMap.emplace(slot, std::move(accExpr));
}
- return {std::move(aggSlots), std::move(accArgEvalStage)};
-}
-
-/**
- * Generate a vector of (inputSlot, mergingExpression) pairs. The slot (whose id is allocated by
- * this function) will be used to store spilled partial aggregate values that have been recovered
- * from disk and deserialized. The merging expression is an agg function which combines these
- * partial aggregates.
- *
- * Usually the returned vector will be of length 1, but in some cases the MQL accumulation statement
- * is implemented by calculating multiple separate aggregates in the SBE plan, which are finalized
- * by a subsequent project stage to produce the ultimate value.
- */
-sbe::SlotExprPairVector generateMergingExpressions(StageBuilderState& state,
- const AccumulationStatement& accStmt,
- int numInputSlots) {
- tassert(7039555, "'numInputSlots' must be positive", numInputSlots > 0);
- auto slotIdGenerator = state.slotIdGenerator;
- tassert(7039556, "expected non-null 'slotIdGenerator' pointer", slotIdGenerator);
- auto frameIdGenerator = state.frameIdGenerator;
- tassert(7039557, "expected non-null 'frameIdGenerator' pointer", frameIdGenerator);
-
- auto spillSlots = slotIdGenerator->generateMultiple(numInputSlots);
- auto collatorSlot = state.data->env->getSlotIfExists("collator"_sd);
- auto mergingExprs =
- buildCombinePartialAggregates(accStmt, spillSlots, collatorSlot, *frameIdGenerator);
-
- // Zip the slot vector and expression vector into a vector of pairs.
- tassert(7039550,
- "expected same number of slots and input exprs",
- spillSlots.size() == mergingExprs.size());
- sbe::SlotExprPairVector result;
- result.reserve(spillSlots.size());
- for (size_t i = 0; i < spillSlots.size(); ++i) {
- result.push_back({spillSlots[i], std::move(mergingExprs[i])});
- }
- return result;
+ return {std::move(aggSlots), std::move(accProjEvalStage)};
}
std::tuple<std::vector<std::string>, sbe::value::SlotVector, EvalStage> generateGroupFinalStage(
@@ -2464,11 +2390,13 @@ std::tuple<std::vector<std::string>, sbe::value::SlotVector, EvalStage> generate
auto finalSlots{sbe::value::SlotVector{finalGroupBySlot}};
std::vector<std::string> fieldNames{"_id"};
+ auto groupFinalEvalStage = std::move(groupEvalStage);
size_t idxAccFirstSlot = dedupedGroupBySlots.size();
for (size_t idxAcc = 0; idxAcc < accStmts.size(); ++idxAcc) {
// Gathers field names for the output object from accumulator statements.
fieldNames.push_back(accStmts[idxAcc].fieldName);
- auto finalExpr = stage_builder::buildFinalize(state, accStmts[idxAcc], aggSlotsVec[idxAcc]);
+ auto [finalExpr, tempEvalStage] = stage_builder::buildFinalize(
+ state, accStmts[idxAcc], aggSlotsVec[idxAcc], std::move(groupFinalEvalStage), nodeId);
// The final step may not return an expression if it's trivial. For example, $first and
// $last's final steps are trivial.
@@ -2483,13 +2411,15 @@ std::tuple<std::vector<std::string>, sbe::value::SlotVector, EvalStage> generate
// Some accumulator(s) like $avg generate multiple expressions and slots. So, need to
// advance this index by the number of those slots for each accumulator.
idxAccFirstSlot += aggSlotsVec[idxAcc].size();
+
+ groupFinalEvalStage = std::move(tempEvalStage);
}
// Gathers all accumulator results. If there're no project expressions, does not add a project
// stage.
auto retEvalStage = prjSlotToExprMap.empty()
- ? std::move(groupEvalStage)
- : makeProject(std::move(groupEvalStage), std::move(prjSlotToExprMap), nodeId);
+ ? std::move(groupFinalEvalStage)
+ : makeProject(std::move(groupFinalEvalStage), std::move(prjSlotToExprMap), nodeId);
return {std::move(fieldNames), std::move(finalSlots), std::move(retEvalStage)};
}
@@ -2548,8 +2478,10 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
const auto& accStmts = groupNode->accumulators;
auto childStageType = childNode->getType();
- auto childReqs = reqs.copy().set(kResult);
- if (childStageType == StageType::STAGE_GROUP && areAllFieldPathsOptimizable(idExpr, accStmts)) {
+ auto areAllTopLevelFields = checkAllFieldPathsAreTopLevel(idExpr, accStmts);
+
+ auto childReqs = reqs.copy();
+ if (childStageType == StageType::STAGE_GROUP && areAllTopLevelFields) {
// Does not ask the GROUP child for the result slot to avoid unnecessary materialization if
// all fields are top-level fields. See the end of this function. For example, GROUP - GROUP
// - COLLSCAN case.
@@ -2572,28 +2504,17 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
// Translates accumulators which are executed inside the group stage and gets slots for
// accumulators.
stage_builder::EvalStage accProjEvalStage = std::move(groupByEvalStage);
- sbe::SlotExprPairVector accSlotExprPairs;
+ sbe::value::SlotMap<std::unique_ptr<sbe::EExpression>> accSlotToExprMap;
std::vector<sbe::value::SlotVector> aggSlotsVec;
- // Since partial accumulator state may be spilled to disk and then merged, we must construct not
- // only the basic agg expressions for each accumulator, but also agg expressions that are used
- // to combine partial aggregates that have been spilled to disk.
- sbe::SlotExprPairVector mergingExprs;
for (const auto& accStmt : accStmts) {
- auto [curAggSlots, tempEvalStage] = generateAccumulator(_state,
- accStmt,
- std::move(accProjEvalStage),
- childOutputs,
- nodeId,
- &_slotIdGenerator,
- accSlotExprPairs);
-
- sbe::SlotExprPairVector curMergingExprs =
- generateMergingExpressions(_state, accStmt, curAggSlots.size());
-
- aggSlotsVec.emplace_back(std::move(curAggSlots));
- mergingExprs.insert(mergingExprs.end(),
- std::make_move_iterator(curMergingExprs.begin()),
- std::make_move_iterator(curMergingExprs.end()));
+ auto [aggSlots, tempEvalStage] = generateAccumulator(_state,
+ accStmt,
+ std::move(accProjEvalStage),
+ childOutputs,
+ nodeId,
+ &_slotIdGenerator,
+ accSlotToExprMap);
+ aggSlotsVec.emplace_back(std::move(aggSlots));
accProjEvalStage = std::move(tempEvalStage);
}
@@ -2604,10 +2525,9 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
// Builds a group stage with accumulator expressions and group-by slot(s).
auto groupEvalStage = makeHashAgg(std::move(accProjEvalStage),
dedupedGroupBySlots,
- std::move(accSlotExprPairs),
+ std::move(accSlotToExprMap),
_state.data->env->getSlotIfExists("collator"_sd),
_cq.getExpCtx()->allowDiskUse,
- std::move(mergingExprs),
nodeId);
tassert(
diff --git a/src/mongo/db/query/sbe_stage_builder.h b/src/mongo/db/query/sbe_stage_builder.h
index d1acc2c765f..7abd0e2fa46 100644
--- a/src/mongo/db/query/sbe_stage_builder.h
+++ b/src/mongo/db/query/sbe_stage_builder.h
@@ -329,6 +329,9 @@ struct PlanStageData {
// This holds the output slots produced by SBE plan (resultSlot, recordIdSlot, etc).
PlanStageSlots outputs;
+ // Map from index name to IAM.
+ StringMap<const IndexAccessMethod*> iamMap;
+
// The CompileCtx object owns the RuntimeEnvironment. The RuntimeEnvironment owns various
// SlotAccessors which are accessed when the SBE plan is executed.
sbe::RuntimeEnvironment* env{nullptr};
@@ -376,6 +379,7 @@ private:
// RuntimeEnvironment and CompileCtx.
void copyFrom(const PlanStageData& other) {
outputs = other.outputs;
+ iamMap = other.iamMap;
shouldTrackLatestOplogTimestamp = other.shouldTrackLatestOplogTimestamp;
shouldTrackResumeToken = other.shouldTrackResumeToken;
shouldUseTailableScan = other.shouldUseTailableScan;
diff --git a/src/mongo/db/query/sbe_stage_builder_accumulator.cpp b/src/mongo/db/query/sbe_stage_builder_accumulator.cpp
index 0cf745c15a1..f01886c4213 100644
--- a/src/mongo/db/query/sbe_stage_builder_accumulator.cpp
+++ b/src/mongo/db/query/sbe_stage_builder_accumulator.cpp
@@ -42,9 +42,9 @@
namespace mongo::stage_builder {
namespace {
-std::unique_ptr<sbe::EExpression> wrapMinMaxArg(std::unique_ptr<sbe::EExpression> arg,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- return makeLocalBind(&frameIdGenerator,
+std::unique_ptr<sbe::EExpression> wrapMinMaxArg(StageBuilderState& state,
+ std::unique_ptr<sbe::EExpression> arg) {
+ return makeLocalBind(state.frameIdGenerator,
[](sbe::EVariable input) {
return sbe::makeE<sbe::EIf>(
generateNullOrMissing(input),
@@ -54,37 +54,30 @@ std::unique_ptr<sbe::EExpression> wrapMinMaxArg(std::unique_ptr<sbe::EExpression
std::move(arg));
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorMin(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorMin(
+ StageBuilderState& state,
const AccumulationExpression& expr,
std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
+ auto collatorSlot = state.data->env->getSlotIfExists("collator"_sd);
if (collatorSlot) {
aggs.push_back(makeFunction("collMin"_sd,
sbe::makeE<sbe::EVariable>(*collatorSlot),
- wrapMinMaxArg(std::move(arg), frameIdGenerator)));
+ wrapMinMaxArg(state, std::move(arg))));
} else {
- aggs.push_back(makeFunction("min"_sd, wrapMinMaxArg(std::move(arg), frameIdGenerator)));
+ aggs.push_back(makeFunction("min"_sd, wrapMinMaxArg(state, std::move(arg))));
}
- return aggs;
+ return {std::move(aggs), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsMin(
+std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildFinalizeMin(
+ StageBuilderState& state,
const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039501,
- "partial agg combiner for $min should have exactly one input slot",
- inputSlots.size() == 1);
- auto arg = makeVariable(inputSlots[0]);
- return buildAccumulatorMin(expr, std::move(arg), collatorSlot, frameIdGenerator);
-}
-
-std::unique_ptr<sbe::EExpression> buildFinalizeMin(StageBuilderState& state,
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& minSlots) {
+ const sbe::value::SlotVector& minSlots,
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
// We can get away with not building a project stage since there's no finalize step but we
// will stick the slot into an EVariable in case a $min is one of many group clauses and it
// can be combined into a final project stage.
@@ -92,104 +85,76 @@ std::unique_ptr<sbe::EExpression> buildFinalizeMin(StageBuilderState& state,
str::stream() << "Expected one input slot for finalization of min, got: "
<< minSlots.size(),
minSlots.size() == 1);
- return makeFillEmptyNull(makeVariable(minSlots[0]));
+ return {makeFillEmptyNull(makeVariable(minSlots[0])), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorMax(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorMax(
+ StageBuilderState& state,
const AccumulationExpression& expr,
std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
+ auto collatorSlot = state.data->env->getSlotIfExists("collator"_sd);
if (collatorSlot) {
aggs.push_back(makeFunction("collMax"_sd,
sbe::makeE<sbe::EVariable>(*collatorSlot),
- wrapMinMaxArg(std::move(arg), frameIdGenerator)));
+ wrapMinMaxArg(state, std::move(arg))));
} else {
- aggs.push_back(makeFunction("max"_sd, wrapMinMaxArg(std::move(arg), frameIdGenerator)));
+ aggs.push_back(makeFunction("max"_sd, wrapMinMaxArg(state, std::move(arg))));
}
- return aggs;
+ return {std::move(aggs), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsMax(
+std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildFinalizeMax(
+ StageBuilderState& state,
const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039502,
- "partial agg combiner for $max should have exactly one input slot",
- inputSlots.size() == 1);
- auto arg = makeVariable(inputSlots[0]);
- return buildAccumulatorMax(expr, std::move(arg), collatorSlot, frameIdGenerator);
-}
-
-std::unique_ptr<sbe::EExpression> buildFinalizeMax(StageBuilderState& state,
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& maxSlots) {
+ const sbe::value::SlotVector& maxSlots,
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
tassert(5755100,
str::stream() << "Expected one input slot for finalization of max, got: "
<< maxSlots.size(),
maxSlots.size() == 1);
- return makeFillEmptyNull(makeVariable(maxSlots[0]));
+ return {makeFillEmptyNull(makeVariable(maxSlots[0])), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorFirst(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorFirst(
+ StageBuilderState& state,
const AccumulationExpression& expr,
std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
aggs.push_back(makeFunction("first", makeFillEmptyNull(std::move(arg))));
- return aggs;
-}
-
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsFirst(
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039503,
- "partial agg combiner for $first should have exactly one input slot",
- inputSlots.size() == 1);
- auto arg = makeVariable(inputSlots[0]);
- return buildAccumulatorFirst(expr, std::move(arg), collatorSlot, frameIdGenerator);
+ return {std::move(aggs), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorLast(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorLast(
+ StageBuilderState& state,
const AccumulationExpression& expr,
std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
aggs.push_back(makeFunction("last", makeFillEmptyNull(std::move(arg))));
- return aggs;
-}
-
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsLast(
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039504,
- "partial agg combiner for $last should have exactly one input slot",
- inputSlots.size() == 1);
- auto arg = makeVariable(inputSlots[0]);
- return buildAccumulatorLast(expr, std::move(arg), collatorSlot, frameIdGenerator);
+ return {std::move(aggs), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorAvg(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorAvg(
+ StageBuilderState& state,
const AccumulationExpression& expr,
std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
// 'aggDoubleDoubleSum' will ignore non-numeric values automatically.
aggs.push_back(makeFunction("aggDoubleDoubleSum", arg->clone()));
// For the counter we need to skip non-numeric values ourselves.
- auto addend = makeLocalBind(&frameIdGenerator,
+ auto addend = makeLocalBind(state.frameIdGenerator,
[](sbe::EVariable input) {
return sbe::makeE<sbe::EIf>(
makeBinaryOp(sbe::EPrimBinary::logicOr,
@@ -202,27 +167,15 @@ std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorAvg(
auto counterExpr = makeFunction("sum", std::move(addend));
aggs.push_back(std::move(counterExpr));
- return aggs;
+ return {std::move(aggs), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsAvg(
+std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildFinalizeAvg(
+ StageBuilderState& state,
const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039539,
- "partial agg combiner for $avg should have exactly two input slots",
- inputSlots.size() == 2);
-
- std::vector<std::unique_ptr<sbe::EExpression>> aggs;
- aggs.push_back(makeFunction("aggMergeDoubleDoubleSums", makeVariable(inputSlots[0])));
- aggs.push_back(makeFunction("sum", makeVariable(inputSlots[1])));
- return aggs;
-}
-
-std::unique_ptr<sbe::EExpression> buildFinalizeAvg(StageBuilderState& state,
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& aggSlots) {
+ const sbe::value::SlotVector& aggSlots,
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
// Slot 0 contains the accumulated sum, and slot 1 contains the count of summed items.
tassert(5754703,
str::stream() << "Expected two slots to finalize avg, got: " << aggSlots.size(),
@@ -277,7 +230,7 @@ std::unique_ptr<sbe::EExpression> buildFinalizeAvg(StageBuilderState& state,
auto partialAvgFinalize =
sbe::makeE<sbe::EIf>(std::move(ifCondExpr), std::move(thenExpr), std::move(elseExpr));
- return partialAvgFinalize;
+ return {std::move(partialAvgFinalize), std::move(inputStage)};
} else {
// If we've encountered any numeric input, the counter would contain a positive integer.
// Unlike $sum, when there is no numeric input, $avg should return null.
@@ -290,37 +243,27 @@ std::unique_ptr<sbe::EExpression> buildFinalizeAvg(StageBuilderState& state,
makeFunction("doubleDoubleSumFinalize", makeVariable(aggSlots[0])),
makeVariable(aggSlots[1])));
- return finalizingExpression;
+ return {std::move(finalizingExpression), std::move(inputStage)};
}
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorSum(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorSum(
+ StageBuilderState& state,
const AccumulationExpression& expr,
std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
aggs.push_back(makeFunction("aggDoubleDoubleSum", std::move(arg)));
- return aggs;
+ return {std::move(aggs), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsSum(
+std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildFinalizeSum(
+ StageBuilderState& state,
const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039530,
- "partial agg combiner for $sum should have exactly one input slot",
- inputSlots.size() == 1);
- auto arg = makeVariable(inputSlots[0]);
- std::vector<std::unique_ptr<sbe::EExpression>> aggs;
- aggs.push_back(makeFunction("aggMergeDoubleDoubleSums", std::move(arg)));
- return aggs;
-}
-
-std::unique_ptr<sbe::EExpression> buildFinalizeSum(StageBuilderState& state,
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& sumSlots) {
+ const sbe::value::SlotVector& sumSlots,
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
tassert(5755300,
str::stream() << "Expected one input slot for finalization of sum, got: "
<< sumSlots.size(),
@@ -349,7 +292,8 @@ std::unique_ptr<sbe::EExpression> buildFinalizeSum(StageBuilderState& state,
auto canUseNewPartialResultFormat = fcv.isVersionInitialized() &&
fcv.isGreaterThanOrEqualTo(multiversion::FeatureCompatibilityVersion::kVersion_6_0);
if (canUseNewPartialResultFormat) {
- return makeFunction("doubleDoublePartialSumFinalize", makeVariable(sumSlots[0]));
+ return {makeFunction("doubleDoublePartialSumFinalize", makeVariable(sumSlots[0])),
+ std::move(inputStage)};
}
// To support the sharding behavior, the mongos splits $group into two separate $group
@@ -387,60 +331,43 @@ std::unique_ptr<sbe::EExpression> buildFinalizeSum(StageBuilderState& state,
input.clone());
},
std::move(sumFinalize));
- return partialSumFinalize;
+ return {std::move(partialSumFinalize), std::move(inputStage)};
} else {
- return makeFunction("doubleDoubleSumFinalize", makeVariable(sumSlots[0]));
+ auto sumFinalize = makeFunction("doubleDoubleSumFinalize", makeVariable(sumSlots[0]));
+ return {std::move(sumFinalize), std::move(inputStage)};
}
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorAddToSetHelper(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorAddToSet(
+ StageBuilderState& state,
+ const AccumulationExpression& expr,
std::unique_ptr<sbe::EExpression> arg,
- StringData funcName,
- boost::optional<sbe::value::SlotId> collatorSlot,
- StringData funcNameWithCollator) {
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
const int cap = internalQueryMaxAddToSetBytes.load();
+ auto collatorSlot = state.data->env->getSlotIfExists("collator"_sd);
if (collatorSlot) {
aggs.push_back(makeFunction(
- funcNameWithCollator,
+ "collAddToSetCapped"_sd,
sbe::makeE<sbe::EVariable>(*collatorSlot),
std::move(arg),
makeConstant(sbe::value::TypeTags::NumberInt32, sbe::value::bitcastFrom<int>(cap))));
} else {
aggs.push_back(makeFunction(
- funcName,
+ "addToSetCapped",
std::move(arg),
makeConstant(sbe::value::TypeTags::NumberInt32, sbe::value::bitcastFrom<int>(cap))));
}
- return aggs;
-}
-
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorAddToSet(
- const AccumulationExpression& expr,
- std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- return buildAccumulatorAddToSetHelper(
- std::move(arg), "addToSetCapped"_sd, collatorSlot, "collAddToSetCapped"_sd);
-}
-
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsAddToSet(
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039506,
- "partial agg combiner for $addToSet should have exactly one input slot",
- inputSlots.size() == 1);
- auto arg = makeVariable(inputSlots[0]);
- return buildAccumulatorAddToSetHelper(
- std::move(arg), "aggSetUnionCapped"_sd, collatorSlot, "aggCollSetUnionCapped"_sd);
+ return {std::move(aggs), std::move(inputStage)};
}
-std::unique_ptr<sbe::EExpression> buildFinalizeCappedAccumulator(
+std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildFinalizeCappedAccumulator(
StageBuilderState& state,
const AccumulationExpression& expr,
- const sbe::value::SlotVector& accSlots) {
+ const sbe::value::SlotVector& accSlots,
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
tassert(6526500,
str::stream() << "Expected one input slot for finalization of capped accumulator, got: "
<< accSlots.size(),
@@ -455,62 +382,33 @@ std::unique_ptr<sbe::EExpression> buildFinalizeCappedAccumulator(
makeConstant(sbe::value::TypeTags::NumberInt32,
static_cast<int>(sbe::vm::AggArrayWithSize::kValues)));
- return pushFinalize;
+ return {std::move(pushFinalize), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorPushHelper(
- std::unique_ptr<sbe::EExpression> arg, StringData aggFuncName) {
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorPush(
+ StageBuilderState& state,
+ const AccumulationExpression& expr,
+ std::unique_ptr<sbe::EExpression> arg,
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
const int cap = internalQueryMaxPushBytes.load();
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
aggs.push_back(makeFunction(
- aggFuncName,
+ "addToArrayCapped"_sd,
std::move(arg),
makeConstant(sbe::value::TypeTags::NumberInt32, sbe::value::bitcastFrom<int>(cap))));
- return aggs;
+ return {std::move(aggs), std::move(inputStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorPush(
- const AccumulationExpression& expr,
- std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- return buildAccumulatorPushHelper(std::move(arg), "addToArrayCapped"_sd);
-}
-
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsPush(
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039505,
- "partial agg combiner for $push should have exactly one input slot",
- inputSlots.size() == 1);
- auto arg = makeVariable(inputSlots[0]);
- return buildAccumulatorPushHelper(std::move(arg), "aggConcatArraysCapped"_sd);
-}
-
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorStdDev(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorStdDev(
+ StageBuilderState& state,
const AccumulationExpression& expr,
std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
aggs.push_back(makeFunction("aggStdDev", std::move(arg)));
- return aggs;
-}
-
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsStdDev(
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039540,
- "partial agg combiner for stddev should have exactly one input slot",
- inputSlots.size() == 1);
- auto arg = makeVariable(inputSlots[0]);
- std::vector<std::unique_ptr<sbe::EExpression>> aggs;
- aggs.push_back(makeFunction("aggMergeStdDevs", std::move(arg)));
- return aggs;
+ return {std::move(aggs), std::move(inputStage)};
}
std::unique_ptr<sbe::EExpression> buildFinalizePartialStdDev(sbe::value::SlotId stdDevSlot) {
@@ -541,75 +439,69 @@ std::unique_ptr<sbe::EExpression> buildFinalizePartialStdDev(sbe::value::SlotId
static_cast<int>(sbe::vm::AggStdDevValueElems::kCount)))});
}
-std::unique_ptr<sbe::EExpression> buildFinalizeStdDevPop(
+std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildFinalizeStdDevPop(
StageBuilderState& state,
const AccumulationExpression& expr,
- const sbe::value::SlotVector& stdDevSlots) {
+ const sbe::value::SlotVector& stdDevSlots,
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
tassert(5755204,
str::stream() << "Expected one input slot for finalization of stdDevPop, got: "
<< stdDevSlots.size(),
stdDevSlots.size() == 1);
if (state.needsMerge) {
- return buildFinalizePartialStdDev(stdDevSlots[0]);
+ return {buildFinalizePartialStdDev(stdDevSlots[0]), std::move(inputStage)};
} else {
auto stdDevPopFinalize = makeFunction("stdDevPopFinalize", makeVariable(stdDevSlots[0]));
- return stdDevPopFinalize;
+ return {std::move(stdDevPopFinalize), std::move(inputStage)};
}
}
-std::unique_ptr<sbe::EExpression> buildFinalizeStdDevSamp(
+std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildFinalizeStdDevSamp(
StageBuilderState& state,
const AccumulationExpression& expr,
- const sbe::value::SlotVector& stdDevSlots) {
+ const sbe::value::SlotVector& stdDevSlots,
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
tassert(5755209,
str::stream() << "Expected one input slot for finalization of stdDevSamp, got: "
<< stdDevSlots.size(),
stdDevSlots.size() == 1);
if (state.needsMerge) {
- return buildFinalizePartialStdDev(stdDevSlots[0]);
+ return {buildFinalizePartialStdDev(stdDevSlots[0]), std::move(inputStage)};
} else {
- return makeFunction("stdDevSampFinalize", makeVariable(stdDevSlots[0]));
+ auto stdDevSampFinalize = makeFunction("stdDevSampFinalize", makeVariable(stdDevSlots[0]));
+ return {std::move(stdDevSampFinalize), std::move(inputStage)};
}
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulatorMergeObjects(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulatorMergeObjects(
+ StageBuilderState& state,
const AccumulationExpression& expr,
std::unique_ptr<sbe::EExpression> arg,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
std::vector<std::unique_ptr<sbe::EExpression>> aggs;
- auto filterExpr =
- makeLocalBind(&frameIdGenerator,
- [](sbe::EVariable input) {
- auto typeCheckExpr =
- makeBinaryOp(sbe::EPrimBinary::logicOr,
- generateNullOrMissing(input),
- makeFunction("isObject", input.clone()));
- return sbe::makeE<sbe::EIf>(
- std::move(typeCheckExpr),
- makeFunction("mergeObjects", input.clone()),
- sbe::makeE<sbe::EFail>(ErrorCodes::Error{5911200},
- "$mergeObjects only supports objects"));
- },
- std::move(arg));
-
- aggs.push_back(std::move(filterExpr));
- return aggs;
-}
+ auto filterExpr = makeLocalBind(
+ state.frameIdGenerator,
+ [](sbe::EVariable input) {
+ return makeBinaryOp(
+ sbe::EPrimBinary::logicOr,
+ generateNullOrMissing(input),
+ makeBinaryOp(sbe::EPrimBinary::logicOr,
+ makeFunction("isObject", input.clone()),
+ sbe::makeE<sbe::EFail>(ErrorCodes::Error{5911200},
+ "$mergeObjects only supports objects")));
+ },
+ arg->clone());
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggsMergeObjects(
- const AccumulationExpression& expr,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- tassert(7039507,
- "partial agg combiner for $mergeObjects should have exactly one input slot",
- inputSlots.size() == 1);
- auto arg = makeVariable(inputSlots[0]);
- return buildAccumulatorMergeObjects(expr, std::move(arg), collatorSlot, frameIdGenerator);
+ inputStage = makeFilter<false>(std::move(inputStage), std::move(filterExpr), planNodeId);
+
+ aggs.push_back(makeFunction("mergeObjects", std::move(arg)));
+ return {std::move(aggs), std::move(inputStage)};
}
}; // namespace
@@ -624,16 +516,19 @@ std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildArgument(
return {argExpr.extractExpr(), std::move(outStage)};
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulator(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulator(
+ StageBuilderState& state,
const AccumulationStatement& acc,
- std::unique_ptr<sbe::EExpression> argExpr,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- using BuildAccumulatorFn = std::function<std::vector<std::unique_ptr<sbe::EExpression>>(
- const AccumulationExpression&,
- std::unique_ptr<sbe::EExpression>,
- boost::optional<sbe::value::SlotId>,
- sbe::value::FrameIdGenerator&)>;
+ EvalStage inputStage,
+ std::unique_ptr<sbe::EExpression> inputExpr,
+ PlanNodeId planNodeId) {
+ using BuildAccumulatorFn =
+ std::function<std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage>(
+ StageBuilderState&,
+ const AccumulationExpression&,
+ std::unique_ptr<sbe::EExpression>,
+ EvalStage,
+ PlanNodeId)>;
static const StringDataMap<BuildAccumulatorFn> kAccumulatorBuilders = {
{AccumulatorMin::kName, &buildAccumulatorMin},
@@ -655,51 +550,25 @@ std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulator(
kAccumulatorBuilders.find(accExprName) != kAccumulatorBuilders.end());
return std::invoke(kAccumulatorBuilders.at(accExprName),
+ state,
acc.expr,
- std::move(argExpr),
- collatorSlot,
- frameIdGenerator);
+ std::move(inputExpr),
+ std::move(inputStage),
+ planNodeId);
}
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggregates(
+std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildFinalize(
+ StageBuilderState& state,
const AccumulationStatement& acc,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator& frameIdGenerator) {
- using BuildAggCombinerFn = std::function<std::vector<std::unique_ptr<sbe::EExpression>>(
+ const sbe::value::SlotVector& aggSlots,
+ EvalStage inputStage,
+ PlanNodeId planNodeId) {
+ using BuildFinalizeFn = std::function<std::pair<std::unique_ptr<sbe::EExpression>, EvalStage>(
+ StageBuilderState&,
const AccumulationExpression&,
- const sbe::value::SlotVector&,
- boost::optional<sbe::value::SlotId>,
- sbe::value::FrameIdGenerator&)>;
-
- static const StringDataMap<BuildAggCombinerFn> kAggCombinerBuilders = {
- {AccumulatorAddToSet::kName, &buildCombinePartialAggsAddToSet},
- {AccumulatorAvg::kName, &buildCombinePartialAggsAvg},
- {AccumulatorFirst::kName, &buildCombinePartialAggsFirst},
- {AccumulatorLast::kName, &buildCombinePartialAggsLast},
- {AccumulatorMax::kName, &buildCombinePartialAggsMax},
- {AccumulatorMergeObjects::kName, &buildCombinePartialAggsMergeObjects},
- {AccumulatorMin::kName, &buildCombinePartialAggsMin},
- {AccumulatorPush::kName, &buildCombinePartialAggsPush},
- {AccumulatorStdDevPop::kName, &buildCombinePartialAggsStdDev},
- {AccumulatorStdDevSamp::kName, &buildCombinePartialAggsStdDev},
- {AccumulatorSum::kName, &buildCombinePartialAggsSum},
- };
-
- auto accExprName = acc.expr.name;
- uassert(7039500,
- str::stream() << "Unsupported Accumulator in SBE accumulator builder: " << accExprName,
- kAggCombinerBuilders.find(accExprName) != kAggCombinerBuilders.end());
-
- return std::invoke(
- kAggCombinerBuilders.at(accExprName), acc.expr, inputSlots, collatorSlot, frameIdGenerator);
-}
-
-std::unique_ptr<sbe::EExpression> buildFinalize(StageBuilderState& state,
- const AccumulationStatement& acc,
- const sbe::value::SlotVector& aggSlots) {
- using BuildFinalizeFn = std::function<std::unique_ptr<sbe::EExpression>(
- StageBuilderState&, const AccumulationExpression&, sbe::value::SlotVector)>;
+ sbe::value::SlotVector,
+ EvalStage,
+ PlanNodeId)>;
static const StringDataMap<BuildFinalizeFn> kAccumulatorBuilders = {
{AccumulatorMin::kName, &buildFinalizeMin},
@@ -721,10 +590,10 @@ std::unique_ptr<sbe::EExpression> buildFinalize(StageBuilderState& state,
kAccumulatorBuilders.find(accExprName) != kAccumulatorBuilders.end());
if (auto fn = kAccumulatorBuilders.at(accExprName); fn) {
- return std::invoke(fn, state, acc.expr, aggSlots);
+ return std::invoke(fn, state, acc.expr, aggSlots, std::move(inputStage), planNodeId);
} else {
// nullptr for 'EExpression' signifies that no final project is necessary.
- return nullptr;
+ return {nullptr, std::move(inputStage)};
}
}
} // namespace mongo::stage_builder
diff --git a/src/mongo/db/query/sbe_stage_builder_accumulator.h b/src/mongo/db/query/sbe_stage_builder_accumulator.h
index 7477ffb2a3a..508a34f07f0 100644
--- a/src/mongo/db/query/sbe_stage_builder_accumulator.h
+++ b/src/mongo/db/query/sbe_stage_builder_accumulator.h
@@ -51,32 +51,24 @@ std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildArgument(
/**
* Translates an input AccumulationStatement into an SBE EExpression for accumulation expressions.
+ * The 'stage' parameter provides the input subtree to build on top of.
*/
-std::vector<std::unique_ptr<sbe::EExpression>> buildAccumulator(
+std::pair<std::vector<std::unique_ptr<sbe::EExpression>>, EvalStage> buildAccumulator(
+ StageBuilderState& state,
const AccumulationStatement& acc,
+ EvalStage stage,
std::unique_ptr<sbe::EExpression> argExpr,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator&);
-
-/**
- * When SBE hash aggregation spills to disk, it spills partial aggregates which need to be combined
- * later. This function returns the expressions that can be used to combine partial aggregates for
- * the given accumulator 'acc'. The aggregate-of-aggregates will be stored in a slots owned by the
- * hash agg stage, while the new partial aggregates to combine can be read from the given
- * 'inputSlots'.
- */
-std::vector<std::unique_ptr<sbe::EExpression>> buildCombinePartialAggregates(
- const AccumulationStatement& acc,
- const sbe::value::SlotVector& inputSlots,
- boost::optional<sbe::value::SlotId> collatorSlot,
- sbe::value::FrameIdGenerator&);
+ PlanNodeId planNodeId);
/**
* Translates an input AccumulationStatement into an SBE EExpression that represents an
* AccumulationStatement's finalization step. The 'stage' parameter provides the input subtree to
* build on top of.
*/
-std::unique_ptr<sbe::EExpression> buildFinalize(StageBuilderState& state,
- const AccumulationStatement& acc,
- const sbe::value::SlotVector& aggSlots);
+std::pair<std::unique_ptr<sbe::EExpression>, EvalStage> buildFinalize(
+ StageBuilderState& state,
+ const AccumulationStatement& acc,
+ const sbe::value::SlotVector& aggSlots,
+ EvalStage stage,
+ PlanNodeId planNodeId);
} // namespace mongo::stage_builder
diff --git a/src/mongo/db/query/sbe_stage_builder_accumulator_test.cpp b/src/mongo/db/query/sbe_stage_builder_accumulator_test.cpp
index 732b04f62a8..e4fd17df36d 100644
--- a/src/mongo/db/query/sbe_stage_builder_accumulator_test.cpp
+++ b/src/mongo/db/query/sbe_stage_builder_accumulator_test.cpp
@@ -27,22 +27,15 @@
* it in the license file.
*/
-#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
-
#include "mongo/platform/basic.h"
#include <fmt/printf.h>
-#include "mongo/db/exec/sbe/expression_test_base.h"
-#include "mongo/db/exec/sbe/values/value_printer.h"
#include "mongo/db/pipeline/document_source_group.h"
#include "mongo/db/pipeline/expression_context_for_test.h"
#include "mongo/db/query/collation/collator_interface_mock.h"
#include "mongo/db/query/query_solution.h"
-#include "mongo/db/query/sbe_stage_builder_accumulator.h"
#include "mongo/db/query/sbe_stage_builder_test_fixture.h"
-#include "mongo/idl/server_parameter_test_util.h"
-#include "mongo/logv2/log.h"
#include "mongo/unittest/unittest.h"
namespace mongo {
@@ -1684,724 +1677,4 @@ TEST_F(SbeStageBuilderGroupTest, SbeIncompatibleExpressionInGroup) {
}
}
-/**
- * A test fixture designed to test that the expressions generated to combine partial aggregates
- * that have been spilled to disk work correctly. We use 'EExpressionTestFixture' rather than
- * something like 'SbeStageBuilderTestFixture' so that the expressions can be tested in isolation,
- * without actually requiring a hash agg stage or without actually spilling any data to disk.
- */
-class SbeStageBuilderGroupAggCombinerTest : public sbe::EExpressionTestFixture {
-public:
- explicit SbeStageBuilderGroupAggCombinerTest()
- : _expCtx{make_intrusive<ExpressionContextForTest>()},
- _inputSlotId{bindAccessor(&_inputAccessor)},
- _collatorSlotId{bindAccessor(&_collatorAccessor)} {}
-
- AccumulationStatement makeAccumulationStatement(StringData accumName) {
- return makeAccumulationStatement(BSON("unused" << BSON(accumName << "unused")));
- }
-
- AccumulationStatement makeAccumulationStatement(BSONObj accumulationStmt) {
- _accumulationStmtBson = std::move(accumulationStmt);
- VariablesParseState vps = _expCtx->variablesParseState;
- return AccumulationStatement::parseAccumulationStatement(
- _expCtx.get(), _accumulationStmtBson.firstElement(), vps);
- }
-
- /**
- * Convenience method for producing bytecode which combines partial aggregates for the given
- * 'AccumulationStatement'.
- *
- * Requires that accumulation statement results in a single aggregate with one input and one
- * output. Furthermore, cannot be used when the test case involves a non-simple collation.
- */
- std::unique_ptr<sbe::vm::CodeFragment> compileSingleInputNoCollator(
- const AccumulationStatement& accStatement) {
- auto exprs = stage_builder::buildCombinePartialAggregates(
- accStatement, {_inputSlotId}, boost::none, _frameIdGenerator);
- ASSERT_EQ(exprs.size(), 1u);
- _expr = std::move(exprs[0]);
-
- return compileAggExpression(*_expr, &_aggAccessor);
- }
-
- /**
- * Verifies that executing the bytecode ('code') for combining partial aggregates for $group
- * spilling produces the 'expected' outputs given 'inputs'.
- *
- * The inputs and expected outputs are expressed as BSON arrays as a convenience to the caller,
- * and should have the same length. The bytecode is executed over each element of 'inputs'
- * one-by-one, with the result stored into a slot holding the aggregate value. At each step,
- * this function asserts that the current aggregate value is equal to the matching element in
- * 'expected'.
- *
- * The string "MISSING" can be used as a sentinel in either 'inputs' or 'outputs' in order to
- * represent the Nothing value (since nothingness cannot literally be stored in a BSON array).
- */
- void aggregateAndAssertResults(BSONArray inputs,
- BSONArray expected,
- const sbe::vm::CodeFragment* code) {
- auto [inputTag, inputVal] = makeArray(inputs);
- auto [expectedTag, expectedVal] = makeArray(expected);
- return aggregateAndAssertResults(inputTag, inputVal, expectedTag, expectedVal, code);
- }
-
- /**
- * Verifies that executing the bytecode ('code') for combining partial aggregates for $group
- * spilling produces the 'expectedVal' outputs given 'inputsVal'. Assumes ownership of both
- * 'expectedVal' and 'inputsVal'.
- *
- * Identical to the overload above, except the inputs and expected outputs are provided as SBE
- * arrays rather than BSON arrays. This is useful if the caller needs to construct input and
- * output ways in a special way that cannot be achieved by trivial conversion from BSON.
- */
- void aggregateAndAssertResults(sbe::value::TypeTags inputTag,
- sbe::value::Value inputVal,
- sbe::value::TypeTags expectedTag,
- sbe::value::Value expectedVal,
- const sbe::vm::CodeFragment* code) {
- // Make sure we are starting from a clean state.
- _inputAccessor.reset();
- _aggAccessor.reset();
-
- sbe::value::ValueGuard inputGuard{inputTag, inputVal};
- sbe::value::ValueGuard expectedGuard{expectedTag, expectedVal};
-
- sbe::value::ArrayEnumerator inputEnumerator{inputTag, inputVal};
- sbe::value::ArrayEnumerator expectedEnumerator{expectedTag, expectedVal};
-
- // Aggregate the inputs one-by-one, and at each step validate that the resulting accumulator
- // state is as expected.
- int index = 0;
- while (!inputEnumerator.atEnd()) {
- ASSERT_FALSE(expectedEnumerator.atEnd());
- auto [nextInputTag, nextInputVal] = inputEnumerator.getViewOfValue();
-
- // Feed in the input value, treating "MISSING" as a special sentinel to indicate the
- // Nothing value.
- if (sbe::value::isString(nextInputTag) &&
- sbe::value::getStringView(nextInputTag, nextInputVal) == "MISSING"_sd) {
- _inputAccessor.reset();
- } else {
- auto [copyTag, copyVal] = sbe::value::copyValue(nextInputTag, nextInputVal);
- _inputAccessor.reset(true, copyTag, copyVal);
- }
-
- auto [outputTag, outputVal] = runCompiledExpression(code);
-
- // Validate that the output value equals the expected value, and then put the output
- // value into the slot that holds the accumulation state.
- auto [expectedOutputTag, expectedOutputValue] = expectedEnumerator.getViewOfValue();
- if (sbe::value::isString(expectedOutputTag) &&
- sbe::value::getStringView(expectedOutputTag, expectedOutputValue) == "MISSING"_sd) {
- expectedOutputTag = sbe::value::TypeTags::Nothing;
- expectedOutputValue = 0;
- }
- auto [compareTag, compareValue] = sbe::value::compareValue(
- outputTag, outputVal, expectedOutputTag, expectedOutputValue);
- if (compareTag != sbe::value::TypeTags::NumberInt32 || compareValue != 0) {
- // The test failed, but dump the actual and expected values to the logs for ease of
- // debugging.
- str::stream actualBuilder;
- auto actualPrinter = makeValuePrinter(actualBuilder);
- actualPrinter.writeValueToStream(outputTag, outputVal);
-
- str::stream expectedBuilder;
- auto expectedPrinter = makeValuePrinter(expectedBuilder);
- expectedPrinter.writeValueToStream(expectedOutputTag, expectedOutputValue);
-
- LOGV2(7039529,
- "Actual value not equal to expected value",
- "actual"_attr = actualBuilder,
- "expected"_attr = expectedBuilder,
- "index"_attr = index);
- FAIL("accumulator did not have expected value");
- }
-
- _aggAccessor.reset(true, outputTag, outputVal);
-
- inputEnumerator.advance();
- expectedEnumerator.advance();
- ++index;
- }
- }
-
- /**
- * A helper for converting a sequence of accumulator states for $push or $addToSet into the
- * corresponding SBE value.
- */
- enum class Accumulator { kPush, kAddToSet };
- std::pair<sbe::value::TypeTags, sbe::value::Value> makeArrayAccumVal(BSONArray bsonArray,
- Accumulator accumType) {
- auto [resultTag, resultVal] = sbe::value::makeNewArray();
- sbe::value::ValueGuard resultGuard{resultTag, resultVal};
- auto resultArr = sbe::value::getArrayView(resultVal);
-
- for (auto&& elt : bsonArray) {
- ASSERT(elt.type() == BSONType::Array);
-
- BSONObjIterator arrayIt{elt.embeddedObject()};
- ASSERT_TRUE(arrayIt.more());
- auto firstElt = arrayIt.next();
- ASSERT(firstElt.type() == BSONType::Array);
- BSONArray partialBsonArr{firstElt.embeddedObject()};
-
- ASSERT_TRUE(arrayIt.more());
- auto secondElt = arrayIt.next();
- ASSERT(secondElt.isNumber());
- int64_t size = secondElt.safeNumberLong();
-
- ASSERT_FALSE(arrayIt.more());
-
- // Each partial aggregate is a two-element array whose first element is the partial
- // $push result (itself an array) and whose second element is the size.
- auto [partialAggTag, partialAggVal] = sbe::value::makeNewArray();
- auto partialAggArr = sbe::value::getArrayView(partialAggVal);
-
- auto [pushedValsTag, pushedValsVal] = accumType == Accumulator::kPush
- ? makeArray(partialBsonArr)
- : makeArraySet(partialBsonArr);
- partialAggArr->push_back(pushedValsTag, pushedValsVal);
-
- partialAggArr->push_back(sbe::value::TypeTags::NumberInt64,
- sbe::value::bitcastFrom<int64_t>(size));
-
- resultArr->push_back(partialAggTag, partialAggVal);
- }
-
- resultGuard.reset();
- return {resultTag, resultVal};
- }
-
- /**
- * Given the name of an SBE agg function ('aggFuncName') and an array of values expressed as a
- * BSON array, aggregates the values inside the array and returns the resulting SBE value.
- */
- std::pair<sbe::value::TypeTags, sbe::value::Value> makeOnePartialAggregate(
- StringData aggFuncName, BSONArray valuesToAgg) {
- // Make sure we are starting from a clean state.
- _inputAccessor.reset();
- _aggAccessor.reset();
-
- // Construct an expression which calls the given agg function, aggregating the values in
- // '_inputSlotId'.
- auto expr =
- stage_builder::makeFunction(aggFuncName, stage_builder::makeVariable(_inputSlotId));
- auto code = compileAggExpression(*expr, &_aggAccessor);
-
- // Find the first element by skipping the length.
- const char* bsonElt = valuesToAgg.objdata() + 4;
- const char* bsonEnd = bsonElt + valuesToAgg.objsize();
- while (*bsonElt != 0) {
- auto fieldName = sbe::bson::fieldNameView(bsonElt);
-
- // Convert the BSON value to an SBE value and put it inside the input slot.
- auto [tag, val] = sbe::bson::convertFrom<false>(bsonElt, bsonEnd, fieldName.size());
- _inputAccessor.reset(true, tag, val);
-
- // Run the agg function, and put the result in the slot holding the aggregate value.
- auto [outputTag, outputVal] = runCompiledExpression(code.get());
- _aggAccessor.reset(true, outputTag, outputVal);
-
- bsonElt = sbe::bson::advance(bsonElt, fieldName.size());
- }
-
- return _aggAccessor.copyOrMoveValue();
- }
-
- /**
- * Returns an SBE array which contains a sequence of partial aggregate values. Useful for
- * constructing a sequence of partial aggregates when those partial aggregates are not trivial
- * to describe using BSON. The input to this function is a BSON array of BSON arrays; each of
- * the inner arrays is aggregated using the given 'aggFuncName' in order to produce the output
- * SBE array.
- *
- * As an example, suppose the agg function is a simple sum. Given the input
- *
- * [[8, 1, 5], [6], [2,3]]
- *
- * the output will be the SBE array [14, 6, 5].
- */
- std::pair<sbe::value::TypeTags, sbe::value::Value> makePartialAggArray(
- StringData aggFuncName, BSONArray arrayOfArrays) {
- auto [arrTag, arrVal] = sbe::value::makeNewArray();
- sbe::value::ValueGuard guard{arrTag, arrVal};
-
- auto arr = sbe::value::getArrayView(arrVal);
-
- for (auto&& element : arrayOfArrays) {
- ASSERT(element.type() == BSONType::Array);
- auto [tag, val] =
- makeOnePartialAggregate(aggFuncName, BSONArray{element.embeddedObject()});
- arr->push_back(tag, val);
- }
-
- guard.reset();
- return {arrTag, arrVal};
- }
-
-protected:
- sbe::value::FrameIdGenerator _frameIdGenerator;
- boost::intrusive_ptr<ExpressionContextForTest> _expCtx;
-
- // Accessor and corresponding slot id that holds the input to the agg expression. Each time we
- // "turn the crank" this will hold the next partial aggregate to be aggregated into
- // '_aggAccessor'.
- sbe::value::OwnedValueAccessor _inputAccessor;
- sbe::value::SlotId _inputSlotId;
-
- // The accessor which holds the final output resulting from combining all partial outputs. We
- // check that the intermediate value is as expected after every turn of the crank.
- sbe::value::OwnedValueAccessor _aggAccessor;
-
- sbe::value::OwnedValueAccessor _collatorAccessor;
- sbe::value::SlotId _collatorSlotId;
-
-private:
- template <typename Stream>
- sbe::value::ValuePrinter<Stream> makeValuePrinter(Stream& stream) {
- return sbe::value::ValuePrinters::make(stream,
- sbe::PrintOptions().useTagForAmbiguousValues(true));
- }
-
- BSONObj _accumulationStmtBson;
- std::unique_ptr<sbe::EExpression> _expr;
-};
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsMin) {
- auto accStatement = makeAccumulationStatement("$min"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- auto inputValues = BSON_ARRAY(8 << 7 << 9 << BSONNULL << 6);
- auto expectedAggStates = BSON_ARRAY(8 << 7 << 7 << 7 << 6);
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-
- // Test that Nothing values are treated as expected.
- inputValues = BSON_ARRAY("MISSING" << 9 << 7 << "MISSING" << 6);
- expectedAggStates = BSON_ARRAY("MISSING" << 9 << 7 << 7 << 6);
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsMinWithCollation) {
- auto accStatement = makeAccumulationStatement("$min"_sd);
-
- auto exprs = stage_builder::buildCombinePartialAggregates(
- accStatement, {_inputSlotId}, {_collatorSlotId}, _frameIdGenerator);
- ASSERT_EQ(exprs.size(), 1u);
- auto expr = std::move(exprs[0]);
-
- CollatorInterfaceMock collator{CollatorInterfaceMock::MockType::kReverseString};
- _collatorAccessor.reset(false,
- sbe::value::TypeTags::collator,
- sbe::value::bitcastFrom<const CollatorInterface*>(&collator));
-
- auto compiledExpr = compileAggExpression(*expr, &_aggAccessor);
-
- // The strings in reverse have the opposite ordering as compared to forwards.
- auto inputValues = BSON_ARRAY("az"
- << "by"
- << "cx");
- auto expectedAggStates = BSON_ARRAY("az"
- << "by"
- << "cx");
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsMax) {
- auto accStatement = makeAccumulationStatement("$max"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- auto inputValues = BSON_ARRAY(3 << 1 << 4 << BSONNULL << 8);
- auto expectedAggStates = BSON_ARRAY(3 << 3 << 4 << 4 << 8);
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-
- // Test that Nothing values are treated as expected.
- inputValues = BSON_ARRAY("MISSING" << 7 << 9 << "MISSING" << 10);
- expectedAggStates = BSON_ARRAY("MISSING" << 7 << 9 << 9 << 10);
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsMaxWithCollation) {
- auto accStatement = makeAccumulationStatement("$max"_sd);
-
- auto exprs = stage_builder::buildCombinePartialAggregates(
- accStatement, {_inputSlotId}, {_collatorSlotId}, _frameIdGenerator);
- ASSERT_EQ(exprs.size(), 1u);
- auto expr = std::move(exprs[0]);
-
- CollatorInterfaceMock collator{CollatorInterfaceMock::MockType::kReverseString};
- _collatorAccessor.reset(false,
- sbe::value::TypeTags::collator,
- sbe::value::bitcastFrom<const CollatorInterface*>(&collator));
-
- auto compiledExpr = compileAggExpression(*expr, &_aggAccessor);
-
- // The strings in reverse have the opposite ordering as compared to forwards.
- auto inputValues = BSON_ARRAY("cx"
- << "by"
- << "az");
- auto expectedAggStates = BSON_ARRAY("cx"
- << "by"
- << "az");
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsFirst) {
- auto accStatement = makeAccumulationStatement("$first"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- auto inputValues = BSON_ARRAY(3 << 1 << BSONNULL << "MISSING" << 8);
- auto expectedAggStates = BSON_ARRAY(3 << 3 << 3 << 3 << 3);
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-
- // When the first value is missing, the resulting value is a literal null.
- inputValues = BSON_ARRAY("MISSING" << 1 << BSONNULL << "MISSING" << 8);
- expectedAggStates = BSON_ARRAY(BSONNULL << BSONNULL << BSONNULL << BSONNULL << BSONNULL);
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsLast) {
- auto accStatement = makeAccumulationStatement("$last"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- auto inputValues = BSON_ARRAY(3 << 1 << BSONNULL << "MISSING" << 8);
- auto expectedAggStates = BSON_ARRAY(3 << 1 << BSONNULL << BSONNULL << 8);
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsPush) {
- auto accStatement = makeAccumulationStatement("$push"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- auto [inputValuesTag, inputValuesVal] = makeArrayAccumVal(
- BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(5 << 4 << 3) << 10)
- << BSON_ARRAY(BSON_ARRAY(2 << 1) << 20) << BSON_ARRAY(BSONArray{} << 0)),
- Accumulator::kPush);
- auto [expectedTag, expectedVal] =
- makeArrayAccumVal(BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(5 << 4 << 3) << 10)
- << BSON_ARRAY(BSON_ARRAY(5 << 4 << 3 << 2 << 1) << 30)
- << BSON_ARRAY(BSON_ARRAY(5 << 4 << 3 << 2 << 1) << 30)),
- Accumulator::kPush);
- aggregateAndAssertResults(
- inputValuesTag, inputValuesVal, expectedTag, expectedVal, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsPushThrowsWhenExceedingSizeLimit) {
- auto accStatement = makeAccumulationStatement("$push"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- // If we inject a very large size, we expect the accumulator to throw. This cap prevents the
- // accumulator from consuming too much memory.
- const int64_t largeSize = 1000 * 1000 * 1000;
-
- auto input = makeArrayAccumVal(BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(5 << 4) << 3)
- << BSON_ARRAY(BSON_ARRAY(2 << 1) << largeSize)),
- Accumulator::kPush);
- auto expected = makeArrayAccumVal(
- BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(5 << 4) << 3) << BSON_ARRAY(BSON_ARRAY("unused") << -1)),
- Accumulator::kPush);
- ASSERT_THROWS_CODE(
- aggregateAndAssertResults(
- input.first, input.second, expected.first, expected.second, compiledExpr.get()),
- DBException,
- ErrorCodes::ExceededMemoryLimit);
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsAddToSet) {
- auto accStatement = makeAccumulationStatement("$addToSet"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- auto [inputValuesTag, inputValuesVal] =
- makeArrayAccumVal(BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(3 << 4 << 5) << 10)
- << BSON_ARRAY(BSON_ARRAY(1 << 3 << 5 << 8) << 20)
- << BSON_ARRAY(BSONArray{} << 0)),
- Accumulator::kAddToSet);
-
- // Each SBE value is 8 bytes and its tag is 1 byte. So we expect each unique element's size to
- // be calculated as 9 bytes. The sizes from the partial aggregates end up getting ignored, and
- // the total size is recalculated, since we cannot predict the size of the set union in advance.
- auto [expectedTag, expectedVal] =
- makeArrayAccumVal(BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(3 << 4 << 5) << 27)
- << BSON_ARRAY(BSON_ARRAY(1 << 3 << 4 << 5 << 8) << 45)
- << BSON_ARRAY(BSON_ARRAY(1 << 3 << 4 << 5 << 8) << 45)),
- Accumulator::kAddToSet);
- aggregateAndAssertResults(
- inputValuesTag, inputValuesVal, expectedTag, expectedVal, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsAddToSetWithCollation) {
- auto accStatement = makeAccumulationStatement("$addToSet"_sd);
-
- auto exprs = stage_builder::buildCombinePartialAggregates(
- accStatement, {_inputSlotId}, {_collatorSlotId}, _frameIdGenerator);
- ASSERT_EQ(exprs.size(), 1u);
- auto expr = std::move(exprs[0]);
-
- CollatorInterfaceMock collator{CollatorInterfaceMock::MockType::kToLowerString};
- _collatorAccessor.reset(false,
- sbe::value::TypeTags::collator,
- sbe::value::bitcastFrom<const CollatorInterface*>(&collator));
-
- auto compiledExpr = compileAggExpression(*expr, &_aggAccessor);
-
- auto [inputValuesTag, inputValuesVal] =
- makeArrayAccumVal(BSON_ARRAY(BSON_ARRAY(BSON_ARRAY("foo"
- << "bar")
- << 10)
- << BSON_ARRAY(BSON_ARRAY("FOO"
- << "BAR"
- << "baz")
- << 20)),
- Accumulator::kAddToSet);
-
- // These strings end up as big strings copied out of the BSON array, so the size accounts for
- // the value itself, the type tag, the 4-byte size of the string, and the string itself.
- auto [expectedTag, expectedVal] =
- makeArrayAccumVal(BSON_ARRAY(BSON_ARRAY(BSON_ARRAY("bar"
- << "foo")
- << 34)
- << BSON_ARRAY(BSON_ARRAY("bar"
- << "baz"
- << "foo")
- << 51)),
- Accumulator::kAddToSet);
- aggregateAndAssertResults(
- inputValuesTag, inputValuesVal, expectedTag, expectedVal, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest,
- CombinePartialAggsAddToSetThrowsWhenExceedingSizeLimit) {
- RAIIServerParameterControllerForTest queryKnobController("internalQueryMaxAddToSetBytes", 50);
-
- auto accStatement = makeAccumulationStatement("$addToSet"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- auto input = makeArrayAccumVal(BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(1 << 2) << 0)
- << BSON_ARRAY(BSON_ARRAY(3 << 4 << 5) << 0)
- << BSON_ARRAY(BSON_ARRAY(6) << 0)),
- Accumulator::kAddToSet);
-
- auto expected =
- makeArrayAccumVal(BSON_ARRAY(BSON_ARRAY(BSON_ARRAY(1 << 2) << 18)
- << BSON_ARRAY(BSON_ARRAY(1 << 2 << 3 << 4 << 5) << 45)
- << BSON_ARRAY(BSON_ARRAY("unused") << -1)),
- Accumulator::kAddToSet);
-
- ASSERT_THROWS_CODE(
- aggregateAndAssertResults(
- input.first, input.second, expected.first, expected.second, compiledExpr.get()),
- DBException,
- ErrorCodes::ExceededMemoryLimit);
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsMergeObjects) {
- auto accStatement = makeAccumulationStatement("$mergeObjects"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- auto inputValues = BSON_ARRAY(BSONNULL << BSONObj{} << BSON("a" << 1) << BSONNULL << "MISSING"
- << BSON("a" << 2 << "b" << 3 << "c" << 4) << BSONObj{});
- auto expectedAggStates =
- BSON_ARRAY(BSONObj{} << BSONObj{} << BSON("a" << 1) << BSON("a" << 1) << BSON("a" << 1)
- << BSON("a" << 2 << "b" << 3 << "c" << 4)
- << BSON("a" << 2 << "b" << 3 << "c" << 4));
- aggregateAndAssertResults(inputValues, expectedAggStates, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsSimpleCount) {
- // $sum:1 is a simple count of the incoming documents. SERVER-65465 changed this scenario to use
- // a simple summation rather than the DoubleDouble summation algorithm in more recent branches,
- // but the 6.0 branch still uses DoubleDouble sum.
- auto inputValues = BSON_ARRAY(5 << 8 << "MISSING" << 4);
- auto [inputTag, inputVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd, BSON_ARRAY(BSON_ARRAY(5) << BSON_ARRAY(8) << BSON_ARRAY(4)));
- auto [expectedTag, expectedVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(5) << BSON_ARRAY(5 << 8) << BSON_ARRAY(5 << 8 << 4)));
-
- auto accStatement = makeAccumulationStatement(BSON("unused" << BSON("$sum" << 1)));
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- aggregateAndAssertResults(inputTag, inputVal, expectedTag, expectedVal, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsDoubleDoubleSum) {
- auto [inputTag, inputVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(1 << 2 << 3) << BSON_ARRAY(4 << 6) << BSON_ARRAY(1 << 1 << 1)));
- auto [expectedTag, expectedVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd, BSON_ARRAY(BSON_ARRAY(6) << BSON_ARRAY(16) << BSON_ARRAY(19)));
-
- // A field path expression is needed so that the merging expression is constructed to combine
- // DoubleDouble summations rather than doing a simple sum. The actual field name "foo" is
- // irrelevant because the values are fed into the merging expression by the test fixture.
- auto accStatement = makeAccumulationStatement(BSON("unused" << BSON("$sum"
- << "$foo")));
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- aggregateAndAssertResults(inputTag, inputVal, expectedTag, expectedVal, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsDoubleDoubleSumInfAndNan) {
- auto [inputTag, inputVal] =
- makePartialAggArray("aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(1 << 2 << 3)
- << BSON_ARRAY(4 << std::numeric_limits<double>::infinity())
- << BSON_ARRAY(1 << 1 << 1)
- << BSON_ARRAY(std::numeric_limits<double>::quiet_NaN())));
- auto [expectedTag, expectedVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(6) << BSON_ARRAY(10 << std::numeric_limits<double>::infinity())
- << BSON_ARRAY(10 << std::numeric_limits<double>::infinity())
- << BSON_ARRAY(std::numeric_limits<double>::quiet_NaN())));
-
- // A field path expression is needed so that the merging expression is constructed to combine
- // DoubleDouble summations rather than doing a simple sum. The actual field name "foo" is
- // irrelevant because the values are fed into the merging expression by the test fixture.
- auto accStatement = makeAccumulationStatement(BSON("unused" << BSON("$sum"
- << "$foo")));
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- aggregateAndAssertResults(inputTag, inputVal, expectedTag, expectedVal, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsDoubleDoubleSumMixedTypes) {
- auto [inputTag, inputVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(1 << 2) << BSON_ARRAY(3ll << 4ll) << BSON_ARRAY(5.5 << 6.6)
- << BSON_ARRAY(Decimal128(7) << Decimal128(8))));
- auto [expectedTag, expectedVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(1 << 2) << BSON_ARRAY(1 << 2 << 3ll << 4ll)
- << BSON_ARRAY(1 << 2 << 3ll << 4ll << 5.5 << 6.6)
- << BSON_ARRAY(1 << 2 << 3ll << 4ll << 5.5 << 6.6
- << Decimal128(7) << Decimal128(8))));
-
- // A field path expression is needed so that the merging expression is constructed to combine
- // DoubleDouble summations rather than doing a simple sum. The actual field name "foo" is
- // irrelevant because the values are fed into the merging expression by the test fixture.
- auto accStatement = makeAccumulationStatement(BSON("unused" << BSON("$sum"
- << "$foo")));
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- aggregateAndAssertResults(inputTag, inputVal, expectedTag, expectedVal, compiledExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsDoubleDoubleSumLargeInts) {
- // Large 64-bit ints can't be represented precisely as doubles. This test demonstrates that when
- // summing such large longs, the sum is returned as a long and no precision is lost.
- const int64_t largeLong = std::numeric_limits<int64_t>::max() - 10;
-
- auto [inputTag, inputVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(largeLong << 1 << 1) << BSON_ARRAY(1ll << 1ll << 1ll)));
- auto [expectedTag, expectedVal] =
- makePartialAggArray("aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(largeLong + 2ll) << BSON_ARRAY(largeLong + 5ll)));
-
- // A field path expression is needed so that the merging expression is constructed to combine
- // DoubleDouble summations rather than doing a simple sum. The actual field name "foo" is
- // irrelevant because the values are fed into the merging expression by the test fixture.
- auto accStatement = makeAccumulationStatement(BSON("unused" << BSON("$sum"
- << "$foo")));
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
-
- aggregateAndAssertResults(inputTag, inputVal, expectedTag, expectedVal, compiledExpr.get());
-
- // Feed the result back into the input accessor. We finalize the resulting aggregate in order
- // to make sure that the resulting sum is mathematically correct.
- auto [resTag, resVal] = _aggAccessor.copyOrMoveValue();
- _inputAccessor.reset(true, resTag, resVal);
- auto finalizeExpr = stage_builder::makeFunction("doubleDoubleSumFinalize",
- stage_builder::makeVariable(_inputSlotId));
- auto finalizeCode = compileExpression(*finalizeExpr);
- auto [finalizedTag, finalizedRes] = runCompiledExpression(finalizeCode.get());
- ASSERT_EQ(finalizedTag, sbe::value::TypeTags::NumberInt64);
- ASSERT_EQ(sbe::value::bitcastTo<int64_t>(finalizedRes), largeLong + 5ll);
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsAvg) {
- auto accStatement = makeAccumulationStatement("$avg"_sd);
-
- // We expect $avg to result in two separate agg expressions: one for computing the sum and the
- // other for computing the count. Both agg expressions read from the same input slot.
- auto exprs = stage_builder::buildCombinePartialAggregates(
- accStatement, {_inputSlotId, _inputSlotId}, boost::none, _frameIdGenerator);
- ASSERT_EQ(exprs.size(), 2u);
-
- // Compile the first expression and make sure it can combine DoubleDouble summations as
- // expected.
- auto [inputTag, inputVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(1 << 2) << BSON_ARRAY(3ll << 4ll) << BSON_ARRAY(5.5 << 6.6)
- << BSON_ARRAY(Decimal128(7) << Decimal128(8))));
- auto [expectedTag, expectedVal] = makePartialAggArray(
- "aggDoubleDoubleSum"_sd,
- BSON_ARRAY(BSON_ARRAY(1 << 2) << BSON_ARRAY(1 << 2 << 3ll << 4ll)
- << BSON_ARRAY(1 << 2 << 3ll << 4ll << 5.5 << 6.6)
- << BSON_ARRAY(1 << 2 << 3ll << 4ll << 5.5 << 6.6
- << Decimal128(7) << Decimal128(8))));
- auto doubleDoubleSumExpr = compileAggExpression(*exprs[0], &_aggAccessor);
- aggregateAndAssertResults(
- inputTag, inputVal, expectedTag, expectedVal, doubleDoubleSumExpr.get());
-
- // Now compile the second expression and make sure it computes a simple sum.
- auto simpleSumExpr = compileAggExpression(*exprs[1], &_aggAccessor);
-
- auto inputValues = BSON_ARRAY(5 << 8 << 0 << 4);
- auto expectedAggStates = BSON_ARRAY(5 << 13 << 13 << 17);
- aggregateAndAssertResults(inputValues, expectedAggStates, simpleSumExpr.get());
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsStdDevPop) {
- auto [inputTag, inputVal] = makePartialAggArray(
- "aggStdDev"_sd,
- BSON_ARRAY(BSON_ARRAY(5 << 10)
- << BSON_ARRAY(6 << 8) << BSON_ARRAY("MISSING") << BSON_ARRAY(1 << 9 << 10)));
- auto [expectedTag, expectedVal] = makePartialAggArray(
- "aggStdDev"_sd,
- BSON_ARRAY(BSON_ARRAY(5 << 10)
- << BSON_ARRAY(5 << 10 << 6 << 8) << BSON_ARRAY(5 << 10 << 6 << 8)
- << BSON_ARRAY(5 << 10 << 6 << 8 << 1 << 9 << 10)));
-
- auto accStatement = makeAccumulationStatement("$stdDevPop"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
- aggregateAndAssertResults(inputTag, inputVal, expectedTag, expectedVal, compiledExpr.get());
-
- // Feed the result back into the input accessor.
- auto [resTag, resVal] = _aggAccessor.copyOrMoveValue();
- _inputAccessor.reset(true, resTag, resVal);
- auto finalizeExpr =
- stage_builder::makeFunction("stdDevPopFinalize", stage_builder::makeVariable(_inputSlotId));
- auto finalizeCode = compileExpression(*finalizeExpr);
- auto [finalizedTag, finalizedRes] = runCompiledExpression(finalizeCode.get());
- ASSERT_EQ(finalizedTag, sbe::value::TypeTags::NumberDouble);
- ASSERT_APPROX_EQUAL(sbe::value::bitcastTo<double>(finalizedRes), 3.0237, 0.0001);
-}
-
-TEST_F(SbeStageBuilderGroupAggCombinerTest, CombinePartialAggsStdDevSamp) {
- auto [inputTag, inputVal] = makePartialAggArray(
- "aggStdDev"_sd,
- BSON_ARRAY(BSON_ARRAY(5 << 10)
- << BSON_ARRAY(6 << 8) << BSON_ARRAY("MISSING") << BSON_ARRAY(1 << 9 << 10)));
- auto [expectedTag, expectedVal] = makePartialAggArray(
- "aggStdDev"_sd,
- BSON_ARRAY(BSON_ARRAY(5 << 10)
- << BSON_ARRAY(5 << 10 << 6 << 8) << BSON_ARRAY(5 << 10 << 6 << 8)
- << BSON_ARRAY(5 << 10 << 6 << 8 << 1 << 9 << 10)));
-
- auto accStatement = makeAccumulationStatement("$stdDevSamp"_sd);
- auto compiledExpr = compileSingleInputNoCollator(accStatement);
- aggregateAndAssertResults(inputTag, inputVal, expectedTag, expectedVal, compiledExpr.get());
-
- // Feed the result back into the input accessor.
- auto [resTag, resVal] = _aggAccessor.copyOrMoveValue();
- _inputAccessor.reset(true, resTag, resVal);
- auto finalizeExpr = stage_builder::makeFunction("stdDevSampFinalize",
- stage_builder::makeVariable(_inputSlotId));
- auto finalizeCode = compileExpression(*finalizeExpr);
- auto [finalizedTag, finalizedRes] = runCompiledExpression(finalizeCode.get());
- ASSERT_EQ(finalizedTag, sbe::value::TypeTags::NumberDouble);
- ASSERT_APPROX_EQUAL(sbe::value::bitcastTo<double>(finalizedRes), 3.2660, 0.0001);
-}
-
} // namespace mongo
diff --git a/src/mongo/db/query/sbe_stage_builder_expression.cpp b/src/mongo/db/query/sbe_stage_builder_expression.cpp
index 8ecc4961435..9fd4aba048e 100644
--- a/src/mongo/db/query/sbe_stage_builder_expression.cpp
+++ b/src/mongo/db/query/sbe_stage_builder_expression.cpp
@@ -27,12 +27,16 @@
* it in the license file.
*/
+#include "mongo/platform/basic.h"
+
#include "mongo/db/query/sbe_stage_builder_expression.h"
+#include "mongo/db/query/util/make_data_structure.h"
#include "mongo/base/string_data.h"
#include "mongo/db/exec/sbe/stages/branch.h"
#include "mongo/db/exec/sbe/stages/co_scan.h"
#include "mongo/db/exec/sbe/stages/filter.h"
+#include "mongo/db/exec/sbe/stages/hash_agg.h"
#include "mongo/db/exec/sbe/stages/limit_skip.h"
#include "mongo/db/exec/sbe/stages/loop_join.h"
#include "mongo/db/exec/sbe/stages/project.h"
@@ -48,7 +52,6 @@
#include "mongo/db/query/projection_parser.h"
#include "mongo/db/query/sbe_stage_builder.h"
#include "mongo/db/query/sbe_stage_builder_eval_frame.h"
-#include "mongo/db/query/util/make_data_structure.h"
#include "mongo/util/str.h"
#include <absl/container/flat_hash_map.h>
@@ -477,7 +480,6 @@ public:
void visit(const ExpressionSetField* expr) final {}
void visit(const ExpressionTsSecond* expr) final {}
void visit(const ExpressionTsIncrement* expr) final {}
- void visit(const ExpressionInternalKeyStringValue* expr) final {}
private:
void visitMultiBranchLogicExpression(const Expression* expr, sbe::EPrimBinary::Op logicOp) {
@@ -711,7 +713,6 @@ public:
void visit(const ExpressionSetField* expr) final {}
void visit(const ExpressionTsSecond* expr) final {}
void visit(const ExpressionTsIncrement* expr) final {}
- void visit(const ExpressionInternalKeyStringValue* expr) final {}
private:
void visitMultiBranchLogicExpression(const Expression* expr, sbe::EPrimBinary::Op logicOp) {
@@ -734,9 +735,6 @@ struct DoubleBound {
static DoubleBound plusInfinity() {
return DoubleBound(std::numeric_limits<double>::infinity(), false);
}
- static DoubleBound plusInfinityInclusive() {
- return DoubleBound(std::numeric_limits<double>::infinity(), true);
- }
std::string printLowerBound() const {
return str::stream() << (inclusive ? "[" : "(") << bound;
}
@@ -1183,46 +1181,103 @@ public:
return;
}
- auto binds = sbe::makeEs();
- for (size_t i = 0; i < numChildren; ++i) {
- binds.emplace_back(_context->popExpr());
+ sbe::EExpression::Vector nullChecks;
+ std::vector<EvalStage> unionBranches;
+ std::vector<sbe::value::SlotVector> unionInputSlots;
+ sbe::value::SlotMap<std::unique_ptr<sbe::EExpression>> projections;
+
+ nullChecks.reserve(numChildren);
+ unionBranches.reserve(numChildren);
+ unionInputSlots.reserve(numChildren);
+ for (size_t idx = 0; idx < numChildren; ++idx) {
+ auto outputSlot = _context->state.slotId();
+ projections.emplace(outputSlot, _context->popExpr());
+ unionBranches.emplace_back(
+ EvalStage{makeLimitCoScanTree(_context->planNodeId), sbe::makeSV()});
+ unionInputSlots.emplace_back(sbe::makeSV(outputSlot));
+ nullChecks.emplace_back(generateNullOrMissing(outputSlot));
}
- std::reverse(binds.begin(), binds.end());
- auto frameId = _context->state.frameId();
- auto args = sbe::makeEs();
+ // Build a project to capture our child expressions.
+ std::reverse(std::begin(unionInputSlots), std::end(unionInputSlots));
+ auto project = makeProject(
+ _context->extractCurrentEvalStage(), std::move(projections), _context->planNodeId);
- std::unique_ptr<sbe::EExpression> checkArgsForNull;
- for (size_t i = 0; i < numChildren; ++i) {
- sbe::EVariable argRef(frameId, i);
- args.emplace_back(argRef.clone());
+ // Build a union stage to consolidate array input branches into a stream.
+ auto unionOutputSlot = _context->state.slotId();
+ auto unionStage = makeUnion(std::move(unionBranches),
+ std::move(unionInputSlots),
+ sbe::makeSV(unionOutputSlot),
+ _context->planNodeId);
- checkArgsForNull = checkArgsForNull ? makeBinaryOp(sbe::EPrimBinary::logicOr,
- std::move(checkArgsForNull),
- generateNullOrMissing(argRef))
- : generateNullOrMissing(argRef);
- }
+ auto collatorSlot = _context->state.data->env->getSlotIfExists("collator"_sd);
- auto nullOrFailExpr =
- sbe::makeE<sbe::EIf>(std::move(checkArgsForNull),
- makeConstant(sbe::value::TypeTags::Null, 0),
- sbe::makeE<sbe::EFail>(ErrorCodes::Error{5153400},
- "$concatArrays only supports arrays"));
+ // Build a filter that will throw an 'EFail' if any element coming from the union is NOT
+ // an array.
+ auto filter = makeFilter<false, false>(
+ std::move(unionStage),
+ makeBinaryOp(sbe::EPrimBinary::logicOr,
+ makeFunction("isArray", makeVariable(unionOutputSlot)),
+ sbe::makeE<sbe::EFail>(ErrorCodes::Error{5153400},
+ "$concatArrays only supports arrays")),
+ _context->planNodeId);
- auto resultExpr = makeLocalBind(
- _context->state.frameIdGenerator,
- [&](sbe::EVariable concatArraysRef) {
- // We optimize for the case where all of the args are arrays. If concatArrays()
- // returns Nothing, then we deal with checking if any of the args are null and
- // either returning null or raising an error.
- return sbe::makeE<sbe::EIf>(makeFunction("exists", concatArraysRef.clone()),
- concatArraysRef.clone(),
- std::move(nullOrFailExpr));
- },
- sbe::makeE<sbe::EFunction>("concatArrays"_sd, std::move(args)));
+ // Build subtree to handle nulls. If an input is null, return null. Otherwise, unwind the
+ // input and concatenate it into an array using addToArray.
+ auto unwindEvalStage =
+ makeUnwind(std::move(filter), _context->state.slotIdGenerator, _context->planNodeId);
+ auto unwindSlot = unwindEvalStage.outSlots.front();
+
+ // Create a group stage to append all streamed elements into one array. This is the final
+ // output when the input consists entirely of arrays.
+ auto finalAddToArrayExpr = makeFunction("addToArray", makeVariable(unwindSlot));
+ auto finalGroupSlot = _context->state.slotId();
+ auto finalGroupStage =
+ makeHashAgg(std::move(unwindEvalStage),
+ sbe::makeSV(),
+ sbe::makeEM(finalGroupSlot, std::move(finalAddToArrayExpr)),
+ collatorSlot,
+ _context->state.allowDiskUse,
+ _context->planNodeId);
+
+ // Returns true if any of our input expressions return null.
+ using iter_t = sbe::EExpression::Vector::iterator;
+ auto checkPartsForNull = std::accumulate(
+ std::move_iterator<iter_t>(nullChecks.begin() + 1),
+ std::move_iterator<iter_t>(nullChecks.end()),
+ std::move(nullChecks.front()),
+ [](auto&& acc, auto&& b) {
+ return makeBinaryOp(sbe::EPrimBinary::logicOr, std::move(acc), std::move(b));
+ });
- _context->pushExpr(
- sbe::makeE<sbe::ELocalBind>(frameId, std::move(binds), std::move(resultExpr)));
+ // Create a branch stage to select between the branch that produces one null if any elements
+ // in the original input were null or missing, or otherwise select the branch that unwinds
+ // and concatenates elements into the output array.
+ auto [nullSlot, nullStage] = [&] {
+ auto outputSlot = _context->state.slotId();
+ auto nullEvalStage =
+ makeProject({makeLimitCoScanTree(_context->planNodeId), sbe::makeSV()},
+ _context->planNodeId,
+ outputSlot,
+ makeConstant(sbe::value::TypeTags::Null, 0));
+ return std::make_pair(outputSlot, std::move(nullEvalStage));
+ }();
+
+ auto branchSlot = _context->state.slotId();
+ auto branchNullEvalStage = makeBranch(std::move(nullStage),
+ std::move(finalGroupStage),
+ std::move(checkPartsForNull),
+ sbe::makeSV(nullSlot),
+ sbe::makeSV(finalGroupSlot),
+ sbe::makeSV(branchSlot),
+ _context->planNodeId);
+
+ // Create nlj to connect outer project with inner branch that handles null input.
+ _context->pushExpr(branchSlot,
+ makeLoopJoin(std::move(project),
+ std::move(branchNullEvalStage),
+ _context->planNodeId,
+ _context->getLexicalEnvironment()));
}
void visit(const ExpressionCond* expr) final {
visitConditionalExpression(expr);
@@ -2122,52 +2177,12 @@ public:
std::vector<EvalExprStagePair> branches;
branches.reserve(numChildren);
- auto childStageCount = 0;
for (size_t i = 0; i < numChildren; ++i) {
auto [expr, stage] = _context->popFrame();
- if (stage.stage.get() != nullptr) {
- childStageCount++;
- }
branches.emplace_back(std::move(expr), std::move(stage));
}
std::reverse(branches.begin(), branches.end());
- // If there is no separate child stage branch, then we can implement $ifNull as a simple
- // projection of SBE if expression, instead of with union stages.
- if (childStageCount == 0) {
- auto stage = _context->extractCurrentEvalStage();
-
- std::vector<sbe::value::SlotId> slots;
- slots.reserve(branches.size());
- sbe::value::SlotMap<std::unique_ptr<sbe::EExpression>> projects;
- for (auto& branch : branches) {
- if (branch.first.getSlot()) {
- slots.push_back(*branch.first.getSlot());
- } else {
- auto slot = _context->state.slotId();
- slots.push_back(slot);
- projects.emplace(slot, branch.first.extractExpr());
- }
- }
- if (!projects.empty()) {
- stage = makeProject(std::move(stage), std::move(projects), _context->planNodeId);
- }
-
- auto expr = sbe::makeE<sbe::EVariable>(slots[slots.size() - 1]);
- for (int i = slots.size() - 2; i >= 0; i--) {
- auto thenExpr = sbe::makeE<sbe::EVariable>(slots[i]);
- auto condExpr = makeNot(generateNullOrMissing(thenExpr->clone()));
- expr =
- sbe::makeE<sbe::EIf>(std::move(condExpr), std::move(thenExpr), std::move(expr));
- }
-
- auto outSlot = _context->state.slotId();
- stage = makeProject(std::move(stage), _context->planNodeId, outSlot, std::move(expr));
-
- _context->pushExpr(outSlot, std::move(stage));
- return;
- }
-
// Prepare to create limit-1/union with N branches (where N is the number of operands). Each
// branch will be evaluated from left to right until one of the branches produces a value.
auto branchFn = [](EvalExpr evalExpr,
@@ -2450,7 +2465,13 @@ public:
exprs[--i] = makeConstant(rit->first);
}
- _context->pushExpr(sbe::makeE<sbe::EFunction>("newObj"_sd, std::move(exprs)));
+ auto fieldSlot{_context->state.slotIdGenerator->generate()};
+ auto stage = makeProject(_context->extractCurrentEvalStage(),
+ _context->planNodeId,
+ fieldSlot,
+ sbe::makeE<sbe::EFunction>("newObj"_sd, std::move(exprs)));
+
+ _context->pushExpr(fieldSlot, std::move(stage));
}
void visit(const ExpressionOr* expr) final {
visitMultiBranchLogicExpression(expr, sbe::EPrimBinary::logicOr);
@@ -2889,7 +2910,7 @@ public:
}
void visit(const ExpressionHyperbolicArcCosine* expr) final {
generateTrigonometricExpressionWithBounds(
- "acosh", DoubleBound(1.0, true), DoubleBound::plusInfinityInclusive());
+ "acosh", DoubleBound(1.0, true), DoubleBound::plusInfinity());
}
void visit(const ExpressionHyperbolicArcSine* expr) final {
generateTrigonometricExpression("asinh");
@@ -3072,10 +3093,6 @@ public:
_context->pushExpr(std::move(tsIncrementExpr));
}
- void visit(const ExpressionInternalKeyStringValue* expr) final {
- unsupportedExpression(expr->getOpName());
- }
-
private:
/**
* Shared logic for $and, $or. Converts each child into an EExpression that evaluates to Boolean
@@ -3336,8 +3353,7 @@ private:
*/
void generateTrigonometricExpressionBinary(StringData exprName) {
_context->ensureArity(2);
- auto x = _context->popExpr();
- auto y = _context->popExpr();
+
auto genericTrignomentricExpr = makeLocalBind(
_context->state.frameIdGenerator,
[&](sbe::EVariable lhs, sbe::EVariable rhs) {
@@ -3356,8 +3372,8 @@ private:
str::stream() << "$" << exprName
<< " supports only numeric types"));
},
- std::move(y),
- std::move(x));
+ _context->popExpr(),
+ _context->popExpr());
_context->pushExpr(std::move(genericTrignomentricExpr));
}
@@ -3398,17 +3414,13 @@ private:
str::stream() << "$" << exprName.toString()
<< " supports only numeric types"),
sbe::makeE<sbe::EIf>(
- // return NaN when NaN is the input.
- generateNaNCheck(inputRef),
- inputRef.clone(),
- sbe::makeE<sbe::EIf>(
- std::move(checkBounds),
- makeFunction(exprName.toString(), inputRef.clone()),
- sbe::makeE<sbe::EFail>(
- ErrorCodes::Error{4995503},
- str::stream() << "Cannot apply $" << exprName.toString()
- << ", value must be in " << lowerBound.printLowerBound()
- << ", " << upperBound.printUpperBound())))));
+ std::move(checkBounds),
+ makeFunction(exprName.toString(), inputRef.clone()),
+ sbe::makeE<sbe::EFail>(ErrorCodes::Error{4995503},
+ str::stream() << "Cannot apply $" << exprName.toString()
+ << ", value must be in "
+ << lowerBound.printLowerBound() << ", "
+ << upperBound.printUpperBound()))));
_context->pushExpr(sbe::makeE<sbe::ELocalBind>(
frameId, std::move(binds), std::move(genericTrignomentricExpr)));
diff --git a/src/mongo/db/query/sbe_stage_builder_helpers.cpp b/src/mongo/db/query/sbe_stage_builder_helpers.cpp
index 3307a6d7597..c36947a23ad 100644
--- a/src/mongo/db/query/sbe_stage_builder_helpers.cpp
+++ b/src/mongo/db/query/sbe_stage_builder_helpers.cpp
@@ -36,8 +36,6 @@
#include <iterator>
#include <numeric>
-#include "mongo/db/catalog/health_log_gen.h"
-#include "mongo/db/catalog/health_log_interface.h"
#include "mongo/db/exec/sbe/expressions/expression.h"
#include "mongo/db/exec/sbe/stages/branch.h"
#include "mongo/db/exec/sbe/stages/co_scan.h"
@@ -55,7 +53,6 @@
#include "mongo/db/query/sbe_stage_builder.h"
#include "mongo/db/storage/execution_context.h"
#include "mongo/logv2/log.h"
-#include "mongo/util/stacktrace.h"
namespace mongo::stage_builder {
@@ -473,22 +470,14 @@ EvalStage makeUnion(std::vector<EvalStage> inputStages,
EvalStage makeHashAgg(EvalStage stage,
sbe::value::SlotVector gbs,
- sbe::SlotExprPairVector aggs,
+ sbe::value::SlotMap<std::unique_ptr<sbe::EExpression>> aggs,
boost::optional<sbe::value::SlotId> collatorSlot,
bool allowDiskUse,
- sbe::SlotExprPairVector mergingExprs,
PlanNodeId planNodeId) {
stage.outSlots = gbs;
for (auto& [slot, _] : aggs) {
stage.outSlots.push_back(slot);
}
-
- // In debug builds or when we explicitly set the query knob, we artificially force frequent
- // spilling. This makes sure that our tests exercise the spilling algorithm and the associated
- // logic for merging partial aggregates which otherwise would require large data sizes to
- // exercise.
- const bool forceIncreasedSpilling = allowDiskUse &&
- (kDebugBuild || internalQuerySlotBasedExecutionHashAggForceIncreasedSpilling.load());
stage.stage = sbe::makeS<sbe::HashAggStage>(std::move(stage.stage),
std::move(gbs),
std::move(aggs),
@@ -496,9 +485,7 @@ EvalStage makeHashAgg(EvalStage stage,
true /* optimized close */,
collatorSlot,
allowDiskUse,
- std::move(mergingExprs),
- planNodeId,
- forceIncreasedSpilling);
+ planNodeId);
return stage;
}
@@ -862,22 +849,6 @@ void indexKeyCorruptionCheckCallback(OperationContext* opCtx,
auto bsonKeyString = KeyString::toBson(*keyString, Ordering::make(bsonKeyPattern));
auto hydratedKey = IndexKeyEntry::rehydrateKey(bsonKeyPattern, bsonKeyString);
- HealthLogEntry entry;
- entry.setNss(nss);
- entry.setTimestamp(Date_t::now());
- entry.setSeverity(SeverityEnum::Error);
- entry.setScope(ScopeEnum::Index);
- entry.setOperation("Index scan");
- entry.setMsg("Erroneous index key found with reference to non-existent record id");
-
- BSONObjBuilder bob;
- bob.append("recordId", rid.toString());
- bob.append("indexKeyData", hydratedKey);
- bob.appendElements(getStackTrace().getBSONRepresentation());
- entry.setData(bob.obj());
-
- HealthLogInterface::get(opCtx)->log(entry);
-
LOGV2_ERROR_OPTIONS(
5113709,
{logv2::UserAssertAfterLog(ErrorCodes::DataCorruptionDetected)},
@@ -897,13 +868,12 @@ void indexKeyCorruptionCheckCallback(OperationContext* opCtx,
* or that the index keys are still part of the underlying index.
*/
bool indexKeyConsistencyCheckCallback(OperationContext* opCtx,
- StringMap<const IndexCatalogEntry*>& entryMap,
+ StringMap<const IndexAccessMethod*> iamTable,
sbe::value::SlotAccessor* snapshotIdAccessor,
sbe::value::SlotAccessor* indexIdAccessor,
sbe::value::SlotAccessor* indexKeyAccessor,
const CollectionPtr& collection,
const Record& nextRecord) {
- // The index consistency check is only performed when 'snapshotIdAccessor' is set.
if (snapshotIdAccessor) {
auto currentSnapshotId = opCtx->recoveryUnit()->getSnapshotId();
auto [snapshotIdTag, snapshotIdVal] = snapshotIdAccessor->getViewOfValue();
@@ -934,29 +904,14 @@ bool indexKeyConsistencyCheckCallback(OperationContext* opCtx,
auto indexId = sbe::value::getStringView(indexIdTag, indexIdVal);
tassert(5290712, "KeyString does not exist", keyString);
- auto it = entryMap.find(indexId);
-
- // If 'entryMap' doesn't contain an entry for 'indexId', create one.
- if (it == entryMap.end()) {
- auto indexCatalog = collection->getIndexCatalog();
- auto indexDesc = indexCatalog->findIndexByName(opCtx, indexId);
- auto entry = indexDesc ? indexDesc->getEntry() : nullptr;
+ auto it = iamTable.find(indexId);
+ tassert(5290713,
+ str::stream() << "IndexAccessMethod not found for index " << indexId,
+ it != iamTable.end());
- // Throw an error if we can't get the IndexDescriptor or the IndexCatalogEntry
- // (or if the index is dropped).
- uassert(ErrorCodes::QueryPlanKilled,
- str::stream() << "query plan killed :: index dropped: " << indexId,
- indexDesc && entry && !entry->isDropped());
-
- auto [newIt, _] = entryMap.emplace(indexId, entry);
-
- it = newIt;
- }
-
- auto entry = it->second;
- auto iam = entry->accessMethod()->asSortedData();
+ auto iam = it->second->asSortedData();
tassert(5290709,
- str::stream() << "Expected to find SortedDataIndexAccessMethod for index: "
+ str::stream() << "Expected to find SortedDataIndexAccessMethod for index "
<< indexId,
iam);
@@ -984,7 +939,6 @@ bool indexKeyConsistencyCheckCallback(OperationContext* opCtx,
return keys->count(*keyString);
}
}
-
return true;
}
@@ -996,6 +950,7 @@ makeLoopJoinForFetch(std::unique_ptr<sbe::PlanStage> inputStage,
sbe::value::SlotId indexKeySlot,
sbe::value::SlotId indexKeyPatternSlot,
const CollectionPtr& collToFetch,
+ StringMap<const IndexAccessMethod*> iamMap,
PlanNodeId planNodeId,
sbe::value::SlotVector slotsToForward,
sbe::value::SlotIdGenerator& slotIdGenerator) {
@@ -1007,7 +962,10 @@ makeLoopJoinForFetch(std::unique_ptr<sbe::PlanStage> inputStage,
auto resultSlot = slotIdGenerator.generate();
auto recordIdSlot = slotIdGenerator.generate();
- sbe::ScanCallbacks callbacks(indexKeyCorruptionCheckCallback, indexKeyConsistencyCheckCallback);
+ using namespace std::placeholders;
+ sbe::ScanCallbacks callbacks(
+ indexKeyCorruptionCheckCallback,
+ std::bind(indexKeyConsistencyCheckCallback, _1, std::move(iamMap), _2, _3, _4, _5, _6));
// Scan the collection in the range [seekKeySlot, Inf).
auto scanStage = sbe::makeS<sbe::ScanStage>(collToFetch->uuid(),
diff --git a/src/mongo/db/query/sbe_stage_builder_helpers.h b/src/mongo/db/query/sbe_stage_builder_helpers.h
index e2e203bdd55..05cf73896e0 100644
--- a/src/mongo/db/query/sbe_stage_builder_helpers.h
+++ b/src/mongo/db/query/sbe_stage_builder_helpers.h
@@ -36,7 +36,6 @@
#include "mongo/db/exec/sbe/expressions/expression.h"
#include "mongo/db/exec/sbe/stages/filter.h"
-#include "mongo/db/exec/sbe/stages/hash_agg.h"
#include "mongo/db/exec/sbe/stages/makeobj.h"
#include "mongo/db/exec/sbe/stages/project.h"
#include "mongo/db/pipeline/expression.h"
@@ -417,10 +416,9 @@ EvalStage makeUnion(std::vector<EvalStage> inputStages,
EvalStage makeHashAgg(EvalStage stage,
sbe::value::SlotVector gbs,
- sbe::SlotExprPairVector aggs,
+ sbe::value::SlotMap<std::unique_ptr<sbe::EExpression>> aggs,
boost::optional<sbe::value::SlotId> collatorSlot,
bool allowDiskUse,
- sbe::SlotExprPairVector mergingExprs,
PlanNodeId planNodeId);
EvalStage makeMkBsonObj(EvalStage stage,
@@ -539,6 +537,7 @@ makeLoopJoinForFetch(std::unique_ptr<sbe::PlanStage> inputStage,
sbe::value::SlotId indexKeySlot,
sbe::value::SlotId indexKeyPatternSlot,
const CollectionPtr& collToFetch,
+ StringMap<const IndexAccessMethod*> iamMap,
PlanNodeId planNodeId,
sbe::value::SlotVector slotsToForward,
sbe::value::SlotIdGenerator& slotIdGenerator);
diff --git a/src/mongo/db/query/sbe_stage_builder_index_scan.cpp b/src/mongo/db/query/sbe_stage_builder_index_scan.cpp
index cc63eee572a..7112e8ad56b 100644
--- a/src/mongo/db/query/sbe_stage_builder_index_scan.cpp
+++ b/src/mongo/db/query/sbe_stage_builder_index_scan.cpp
@@ -283,6 +283,10 @@ generateOptimizedMultiIntervalIndexScan(StageBuilderState& state,
makeFunction("getField"_sd, makeVariable(unwindSlot), makeConstant("l"_sd)));
projects.emplace(highKeySlot,
makeFunction("getField"_sd, makeVariable(unwindSlot), makeConstant("h"_sd)));
+ if (indexIdSlot) {
+ // Construct a copy of 'indexName' to project for use in the index consistency check.
+ projects.emplace(*indexIdSlot, makeConstant(indexName));
+ }
if (indexKeyPatternSlot) {
auto [bsonObjTag, bsonObjVal] =
@@ -296,13 +300,20 @@ generateOptimizedMultiIntervalIndexScan(StageBuilderState& state,
auto project =
sbe::makeS<sbe::ProjectStage>(std::move(unwind), std::move(projects), planNodeId);
+ // Whereas 'snapshotIdSlot' is used by the caller to inspect the snapshot id of the latest index
+ // key, 'indexSnapshotSlot' is updated by the IndexScan below during yield to obtain the latest
+ // snapshot id.
+ boost::optional<sbe::value::SlotId> indexSnapshotSlot;
+ if (snapshotIdSlot) {
+ indexSnapshotSlot = slotIdGenerator->generate();
+ }
+
auto stage = sbe::makeS<sbe::IndexScanStage>(collection->uuid(),
indexName,
forward,
recordSlot,
recordIdSlot,
- snapshotIdSlot,
- indexIdSlot,
+ indexSnapshotSlot,
indexKeysToInclude,
std::move(indexKeySlots),
lowKeySlot,
@@ -310,7 +321,19 @@ generateOptimizedMultiIntervalIndexScan(StageBuilderState& state,
yieldPolicy,
planNodeId);
+ // Add a project on top of the index scan to remember the snapshotId of the most recent index
+ // key returned by the IndexScan above. Otherwise, the index key's snapshot id would be
+ // overwritten during yield.
+ if (snapshotIdSlot) {
+ stage = sbe::makeProjectStage(
+ std::move(stage), planNodeId, *snapshotIdSlot, makeVariable(*indexSnapshotSlot));
+ }
+
auto outerSv = sbe::makeSV();
+ if (indexIdSlot) {
+ outerSv.push_back(*indexIdSlot);
+ }
+
if (indexKeyPatternSlot) {
outerSv.push_back(*indexKeyPatternSlot);
}
@@ -402,6 +425,10 @@ makeRecursiveBranchForGenericIndexScan(const CollectionPtr& collection,
// contain a value from the stack spool. See below for details.
sbe::value::SlotMap<std::unique_ptr<sbe::EExpression>> projects;
projects.emplace(lowKeySlot, makeVariable(seekKeySlot));
+ if (indexIdSlot) {
+ // Construct a copy of 'indexName' to project for use in the index consistency check.
+ projects.emplace(*indexIdSlot, makeConstant(indexName));
+ }
if (indexKeyPatternSlot) {
auto [bsonObjTag, bsonObjVal] = sbe::value::copyValue(
@@ -422,7 +449,6 @@ makeRecursiveBranchForGenericIndexScan(const CollectionPtr& collection,
resultSlot,
recordIdSlot,
snapshotIdSlot,
- indexIdSlot,
indexKeysToInclude,
std::move(savedIndexKeySlots),
lowKeySlot,
@@ -432,6 +458,10 @@ makeRecursiveBranchForGenericIndexScan(const CollectionPtr& collection,
// Get the low key from the outer side and feed it to the inner side (ixscan).
sbe::value::SlotVector outerSv = sbe::makeSV();
+ if (indexIdSlot) {
+ outerSv.push_back(*indexIdSlot);
+ }
+
if (indexKeyPatternSlot) {
outerSv.push_back(*indexKeyPatternSlot);
}
@@ -444,11 +474,19 @@ makeRecursiveBranchForGenericIndexScan(const CollectionPtr& collection,
planNodeId);
sbe::value::SlotVector correlatedSv = sbe::makeSV(seekKeySlot);
+ if (indexIdSlot) {
+ correlatedSv.push_back(*indexIdSlot);
+ }
+
if (indexKeyPatternSlot) {
correlatedSv.push_back(*indexKeyPatternSlot);
}
auto spoolValsSV = sbe::makeSV(seekKeySlot);
+ if (indexIdSlot) {
+ spoolValsSV.push_back(*indexIdSlot);
+ }
+
if (indexKeyPatternSlot) {
spoolValsSV.push_back(*indexKeyPatternSlot);
}
@@ -806,6 +844,11 @@ generateSingleIntervalIndexScan(StageBuilderState& state,
auto lowKeySlot = makeKeySlot(std::move(lowKey));
auto highKeySlot = makeKeySlot(std::move(highKey));
+ if (indexIdSlot) {
+ // Construct a copy of 'indexName' to project for use in the index consistency check.
+ projects.emplace(*indexIdSlot, makeConstant(indexName));
+ }
+
if (indexKeyPatternSlot) {
auto [bsonObjTag, bsonObjVal] =
sbe::value::copyValue(sbe::value::TypeTags::bsonObject,
@@ -840,6 +883,14 @@ generateSingleIntervalIndexScan(StageBuilderState& state,
planNodeId);
}();
+ // Whereas 'snapshotIdSlot' is used by the caller to inspect the snapshot id of the latest index
+ // key, 'indexSnapshotSlot' is updated by the IndexScan below during yield to obtain the latest
+ // snapshot id.
+ boost::optional<sbe::value::SlotId> indexSnapshotSlot;
+ if (snapshotIdSlot) {
+ indexSnapshotSlot = slotIdGenerator->generate();
+ }
+
// Scan the index in the range {'lowKeySlot', 'highKeySlot'} (subject to inclusive or
// exclusive boundaries), and produce a single field recordIdSlot that can be used to
// position into the collection.
@@ -848,8 +899,7 @@ generateSingleIntervalIndexScan(StageBuilderState& state,
forward,
recordSlot,
recordIdSlot,
- snapshotIdSlot,
- indexIdSlot,
+ indexSnapshotSlot,
indexKeysToInclude,
std::move(indexKeySlots),
lowKeySlot,
@@ -857,7 +907,19 @@ generateSingleIntervalIndexScan(StageBuilderState& state,
yieldPolicy,
planNodeId);
+ // Add a project on top of the index scan to remember the snapshotId of the most recent index
+ // key returned by the IndexScan above. Otherwise, the index key's snapshot id would be
+ // overwritten during yield.
+ if (snapshotIdSlot) {
+ stage = sbe::makeProjectStage(
+ std::move(stage), planNodeId, *snapshotIdSlot, makeVariable(*indexSnapshotSlot));
+ }
+
auto outerSv = sbe::makeSV();
+ if (indexIdSlot) {
+ outerSv.push_back(*indexIdSlot);
+ }
+
if (indexKeyPatternSlot) {
outerSv.push_back(*indexKeyPatternSlot);
}
@@ -880,7 +942,7 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> generateIndexScan(
const IndexScanNode* ixn,
const sbe::IndexKeysInclusionSet& originalIndexKeyBitset,
PlanYieldPolicy* yieldPolicy,
- bool doIndexConsistencyCheck,
+ StringMap<const IndexAccessMethod*>* iamMap,
bool needsCorruptionCheck) {
auto indexName = ixn->index.identifier.catalogName;
@@ -918,12 +980,15 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> generateIndexScan(
auto indexKeySlots = state.slotIdGenerator->generateMultiple(indexKeyBitset.count());
sbe::value::SlotVector relevantSlots;
- // Generate the relevant slots.
+ // Generate the relevant slots and add the access method corresponding to 'indexName' to
+ // 'iamMap' if a parent stage needs to execute a consistency check.
boost::optional<sbe::value::SlotId> snapshotIdSlot;
boost::optional<sbe::value::SlotId> indexIdSlot;
boost::optional<sbe::value::SlotId> indexKeySlot;
- if (doIndexConsistencyCheck) {
+ if (iamMap) {
+ iamMap->insert({indexName, accessMethod});
+
snapshotIdSlot = state.slotId();
outputs.set(PlanStageSlots::kSnapshotId, *snapshotIdSlot);
relevantSlots.push_back(*snapshotIdSlot);
@@ -1123,7 +1188,7 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> generateIndexScanWith
const IndexScanNode* ixn,
const sbe::IndexKeysInclusionSet& originalIndexKeyBitset,
PlanYieldPolicy* yieldPolicy,
- bool doIndexConsistencyCheck,
+ StringMap<const IndexAccessMethod*>* iamMap,
bool needsCorruptionCheck) {
const bool forward = ixn->direction == 1;
auto indexName = ixn->index.identifier.catalogName;
@@ -1137,6 +1202,11 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> generateIndexScanWith
// Find the IndexAccessMethod which corresponds to the 'indexName'.
auto accessMethod = descriptor->getEntry()->accessMethod()->asSortedData();
+ // Add the access method corresponding to 'indexName' to the 'iamMap' if a parent stage needs to
+ // execute a consistency check.
+ if (iamMap) {
+ iamMap->insert({indexName, accessMethod});
+ }
PlanStageSlots outputs;
sbe::value::SlotVector relevantSlots;
std::unique_ptr<sbe::PlanStage> stage;
@@ -1181,9 +1251,9 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> generateIndexScanWith
nullptr,
indexKeyBitset,
outputIndexKeySlots,
- makeSlot(doIndexConsistencyCheck, PlanStageSlots::kSnapshotId),
- makeSlot(doIndexConsistencyCheck, PlanStageSlots::kIndexId),
- makeSlot(doIndexConsistencyCheck, PlanStageSlots::kIndexKey),
+ makeSlot(iamMap, PlanStageSlots::kSnapshotId),
+ makeSlot(iamMap, PlanStageSlots::kIndexId),
+ makeSlot(iamMap, PlanStageSlots::kIndexKey),
makeSlot(needsCorruptionCheck, PlanStageSlots::kIndexKeyPattern),
yieldPolicy,
ixn->nodeId());
@@ -1221,11 +1291,11 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> generateIndexScanWith
};
auto [genericIndexScanSnapshotIdSlot, optimizedIndexScanSnapshotIdSlot] =
- makeSlotsForThenElseBranches(doIndexConsistencyCheck, PlanStageSlots::kSnapshotId);
+ makeSlotsForThenElseBranches(iamMap, PlanStageSlots::kSnapshotId);
auto [genericIndexScanIndexIdSlot, optimizedIndexScanIndexIdSlot] =
- makeSlotsForThenElseBranches(doIndexConsistencyCheck, PlanStageSlots::kIndexId);
+ makeSlotsForThenElseBranches(iamMap, PlanStageSlots::kIndexId);
auto [genericIndexScanIndexKeySlot, optimizedIndexScanIndexKeySlot] =
- makeSlotsForThenElseBranches(doIndexConsistencyCheck, PlanStageSlots::kIndexKey);
+ makeSlotsForThenElseBranches(iamMap, PlanStageSlots::kIndexKey);
// Generate a slot for an index key pattern if a parent stage needs to execute a
// corruption check.
diff --git a/src/mongo/db/query/sbe_stage_builder_index_scan.h b/src/mongo/db/query/sbe_stage_builder_index_scan.h
index 08e239c018a..340a03051eb 100644
--- a/src/mongo/db/query/sbe_stage_builder_index_scan.h
+++ b/src/mongo/db/query/sbe_stage_builder_index_scan.h
@@ -63,7 +63,7 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> generateIndexScan(
const IndexScanNode* ixn,
const sbe::IndexKeysInclusionSet& indexKeyBitset,
PlanYieldPolicy* yieldPolicy,
- bool doIndexConsistencyCheck,
+ StringMap<const IndexAccessMethod*>* iamMap,
bool needsCorruptionCheck);
/**
@@ -164,6 +164,6 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> generateIndexScanWith
const IndexScanNode* ixn,
const sbe::IndexKeysInclusionSet& indexKeyBitset,
PlanYieldPolicy* yieldPolicy,
- bool doIndexConsistencyCheck,
+ StringMap<const IndexAccessMethod*>* iamMap,
bool needsCorruptionCheck);
} // namespace mongo::stage_builder
diff --git a/src/mongo/db/query/sbe_stage_builder_lookup.cpp b/src/mongo/db/query/sbe_stage_builder_lookup.cpp
index 6037fb4eef7..4b62228edde 100644
--- a/src/mongo/db/query/sbe_stage_builder_lookup.cpp
+++ b/src/mongo/db/query/sbe_stage_builder_lookup.cpp
@@ -342,15 +342,12 @@ std::pair<SlotId /* keyValuesSetSlot */, std::unique_ptr<sbe::PlanStage>> buildK
// Re-pack the individual key values into a set. We don't cap "addToSet" here because its size
// is bounded by the size of the record.
SlotId keyValuesSetSlot = slotIdGenerator.generate();
- SlotId spillSlot = slotIdGenerator.generate();
EvalStage packedKeyValuesStage = makeHashAgg(
EvalStage{std::move(keyValuesStage), SlotVector{}},
makeSV(), /* groupBy slots - "none" means creating a single group */
- makeSlotExprPairVec(keyValuesSetSlot,
- makeFunction("addToSet"_sd, makeVariable(keyValueSlot))),
+ makeEM(keyValuesSetSlot, makeFunction("addToSet"_sd, makeVariable(keyValueSlot))),
boost::none /* we group _all_ key values into a single set, so collator is irrelevant */,
allowDiskUse,
- makeSlotExprPairVec(spillSlot, makeFunction("aggSetUnion"_sd, makeVariable(spillSlot))),
nodeId);
// The set in 'keyValuesSetSlot' might end up empty if the localField contained only missing and
@@ -406,20 +403,15 @@ std::pair<SlotId /* resultSlot */, std::unique_ptr<sbe::PlanStage>> buildForeign
// are no matches, return an empty array.
const int sizeCap = internalLookupStageIntermediateDocumentMaxSizeBytes.load();
SlotId accumulatorSlot = slotIdGenerator.generate();
- SlotId spillSlot = slotIdGenerator.generate();
innerBranch = makeHashAgg(
std::move(innerBranch),
makeSV(), /* groupBy slots */
- makeSlotExprPairVec(accumulatorSlot,
- makeFunction("addToArrayCapped"_sd,
- makeVariable(foreignRecordSlot),
- makeConstant(TypeTags::NumberInt32, sizeCap))),
+ makeEM(accumulatorSlot,
+ makeFunction("addToArrayCapped"_sd,
+ makeVariable(foreignRecordSlot),
+ makeConstant(TypeTags::NumberInt32, sizeCap))),
{} /* collatorSlot, no collation here because we want to return all matches "as is" */,
allowDiskUse,
- makeSlotExprPairVec(spillSlot,
- makeFunction("aggConcatArraysCapped",
- makeVariable(spillSlot),
- makeConstant(TypeTags::NumberInt32, sizeCap))),
nodeId);
// 'accumulatorSlot' is either Nothing or contains an array of size two, where the front element
@@ -618,6 +610,7 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
const FieldPath& foreignFieldName,
const CollectionPtr& foreignColl,
const IndexEntry& index,
+ StringMap<const IndexAccessMethod*>& iamMap,
PlanYieldPolicySBE* yieldPolicy,
boost::optional<SlotId> collatorSlot,
const PlanNodeId nodeId,
@@ -636,6 +629,7 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
foreignColl->getIndexCatalog()->getEntry(indexDescriptor)->accessMethod()->asSortedData();
const auto indexVersion = indexAccessMethod->getSortedDataInterface()->getKeyStringVersion();
const auto indexOrdering = indexAccessMethod->getSortedDataInterface()->getOrdering();
+ iamMap.insert({indexName, indexAccessMethod});
// Build the outer branch that produces the correlated local key slot.
auto [localKeysSetSlot, localKeysSetStage] = buildKeySet(JoinSide::Local,
@@ -747,10 +741,11 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
// Calculate the low key and high key of each individual local field. They are stored in
// 'lowKeySlot' and 'highKeySlot', respectively. These two slots will be made available in
- // the loop join stage to perform index seek. We also set the 'indexKeyPatternSlot' constant
- // for the seek stage later to perform consistency check.
+ // the loop join stage to perform index seek. We also set 'indexIdSlot' and
+ // 'indexKeyPatternSlot' constants for the seek stage later to perform consistency check.
auto lowKeySlot = slotIdGenerator.generate();
auto highKeySlot = slotIdGenerator.generate();
+ auto indexIdSlot = slotIdGenerator.generate();
auto indexKeyPatternSlot = slotIdGenerator.generate();
auto [_, indexKeyPatternValue] =
copyValue(TypeTags::bsonObject, bitcastFrom<const char*>(index.keyPattern.objdata()));
@@ -777,6 +772,8 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
makeNewKeyStringCall(KeyString::Discriminator::kExclusiveBefore),
highKeySlot,
makeNewKeyStringCall(KeyString::Discriminator::kExclusiveAfter),
+ indexIdSlot,
+ makeConstant(indexName),
indexKeyPatternSlot,
makeConstant(value::TypeTags::bsonObject, indexKeyPatternValue));
@@ -797,14 +794,12 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
auto foreignRecordIdSlot = slotIdGenerator.generate();
auto indexKeySlot = slotIdGenerator.generate();
auto snapshotIdSlot = slotIdGenerator.generate();
- auto indexIdSlot = slotIdGenerator.generate();
auto ixScanStage = makeS<IndexScanStage>(foreignCollUUID,
indexName,
true /* forward */,
indexKeySlot,
foreignRecordIdSlot,
snapshotIdSlot,
- indexIdSlot,
IndexKeysInclusionSet{} /* indexKeysToInclude */,
makeSV() /* vars */,
lowKeySlot,
@@ -817,7 +812,7 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
auto ixScanNljStage =
makeS<LoopJoinStage>(std::move(indexBoundKeyStage),
std::move(ixScanStage),
- makeSV(indexKeyPatternSlot) /* outerProjects */,
+ makeSV(indexIdSlot, indexKeyPatternSlot) /* outerProjects */,
makeSV(lowKeySlot, highKeySlot) /* outerCorrelated */,
nullptr /* predicate */,
nodeId);
@@ -837,7 +832,8 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
// Loop join the foreign record id produced by the index seek on the outer side with seek
// stage on the inner side to get matched foreign documents. The foreign documents are
// stored in 'foreignRecordSlot'. We also pass in 'snapshotIdSlot', 'indexIdSlot',
- // 'indexKeySlot' and 'indexKeyPatternSlot' to perform index consistency check during the seek.
+ // 'indexKeySlot' and 'indexKeyPatternSlot' to perform index consistency check during the
+ // seek.
auto [foreignRecordSlot, __, scanNljStage] = makeLoopJoinForFetch(std::move(ixScanNljStage),
foreignRecordIdSlot,
snapshotIdSlot,
@@ -845,6 +841,7 @@ std::pair<SlotId, std::unique_ptr<sbe::PlanStage>> buildIndexJoinLookupStage(
indexKeySlot,
indexKeyPatternSlot,
foreignColl,
+ iamMap,
nodeId,
makeSV() /* slotsToForward */,
slotIdGenerator);
@@ -1091,6 +1088,7 @@ std::pair<std::unique_ptr<sbe::PlanStage>, PlanStageSlots> SlotBasedStageBuilder
eqLookupNode->joinFieldForeign,
foreignColl,
*eqLookupNode->idxEntry,
+ _data.iamMap,
_yieldPolicy,
collatorSlot,
eqLookupNode->nodeId(),
diff --git a/src/mongo/db/query/sbe_stage_builder_lookup_test.cpp b/src/mongo/db/query/sbe_stage_builder_lookup_test.cpp
index bcc426984e8..b54872582eb 100644
--- a/src/mongo/db/query/sbe_stage_builder_lookup_test.cpp
+++ b/src/mongo/db/query/sbe_stage_builder_lookup_test.cpp
@@ -235,7 +235,7 @@ public:
expectedDocuments.reserve(expectedPairs.size());
for (auto& [localDocument, matchedDocuments] : expectedPairs) {
MutableDocument expectedDocument;
- expectedDocument.reset(localDocument, false /* bsonHasMetadata */);
+ expectedDocument.reset(localDocument, false /* stripMetadata */);
std::vector<mongo::Value> matchedValues{matchedDocuments.begin(),
matchedDocuments.end()};
diff --git a/src/mongo/db/query/sbe_utils.cpp b/src/mongo/db/query/sbe_utils.cpp
index 7f327848fc7..695b2904562 100644
--- a/src/mongo/db/query/sbe_utils.cpp
+++ b/src/mongo/db/query/sbe_utils.cpp
@@ -62,11 +62,9 @@ bool isQuerySbeCompatible(const CollectionPtr* collection,
const bool doesNotHaveElemMatchProject = !cq->getProj() || !cq->getProj()->containsElemMatch();
- const bool isNotInnerSideOfLookup = !(expCtx && expCtx->inLookup);
-
return allExpressionsSupported && isNotCount && doesNotContainMetadataRequirements &&
isQueryNotAgainstTimeseriesCollection && isQueryNotAgainstClusteredCollection &&
doesNotSortOnMetaOrPathWithNumericComponents && isNotOplog && doesNotRequireMatchDetails &&
- doesNotHaveElemMatchProject && isNotInnerSideOfLookup;
+ doesNotHaveElemMatchProject;
}
} // namespace mongo::sbe
diff --git a/src/mongo/db/query/sort_pattern.cpp b/src/mongo/db/query/sort_pattern.cpp
index 5a444cd31f2..fcd3cd177e1 100644
--- a/src/mongo/db/query/sort_pattern.cpp
+++ b/src/mongo/db/query/sort_pattern.cpp
@@ -112,13 +112,13 @@ QueryMetadataBitSet SortPattern::metadataDeps(QueryMetadataBitSet unavailableMet
return depsTracker.metadataDeps();
}
-Document SortPattern::serialize(SortKeySerialization serializationMode,
- const SerializationOptions& options) const {
+Document SortPattern::serialize(SortKeySerialization serializationMode) const {
MutableDocument keyObj;
const size_t n = _sortPattern.size();
for (size_t i = 0; i < n; ++i) {
if (_sortPattern[i].fieldPath) {
- keyObj.setField(options.serializeFieldPath(*_sortPattern[i].fieldPath),
+ // Append a named integer based on whether the sort is ascending/descending.
+ keyObj.setField(_sortPattern[i].fieldPath->fullPath(),
Value(_sortPattern[i].isAscending ? 1 : -1));
} else {
// Sorting by an expression, use a made up field name.
@@ -127,12 +127,7 @@ Document SortPattern::serialize(SortKeySerialization serializationMode,
case SortKeySerialization::kForExplain:
case SortKeySerialization::kForPipelineSerialization: {
const bool isExplain = (serializationMode == SortKeySerialization::kForExplain);
- auto opts = SerializationOptions{};
- if (isExplain) {
- opts.verbosity =
- boost::make_optional(ExplainOptions::Verbosity::kQueryPlanner);
- }
- keyObj[computedFieldName] = _sortPattern[i].expression->serialize(opts);
+ keyObj[computedFieldName] = _sortPattern[i].expression->serialize(isExplain);
break;
}
case SortKeySerialization::kForSortKeyMerging: {
diff --git a/src/mongo/db/query/sort_pattern.h b/src/mongo/db/query/sort_pattern.h
index c709a43eac8..9c74208ac43 100644
--- a/src/mongo/db/query/sort_pattern.h
+++ b/src/mongo/db/query/sort_pattern.h
@@ -33,7 +33,6 @@
#include "mongo/db/exec/document_value/document.h"
#include "mongo/db/pipeline/document_path_support.h"
#include "mongo/db/pipeline/expression.h"
-#include "mongo/db/query/query_shape/serialization_options.h"
namespace mongo {
class SortPattern {
@@ -73,8 +72,7 @@ public:
/**
* Write out a Document whose contents are the sort key pattern.
*/
- Document serialize(SortKeySerialization serializationMode,
- const SerializationOptions& options = {}) const;
+ Document serialize(SortKeySerialization) const;
/**
* Serializes the document to BSON, only keeping the paths specified in the sort pattern.
@@ -149,6 +147,6 @@ private:
std::vector<SortPatternPart> _sortPattern;
// The set of paths on which we're sorting.
- OrderedPathSet _paths;
+ std::set<std::string> _paths;
};
} // namespace mongo
diff --git a/src/mongo/db/query/sort_pattern_test.cpp b/src/mongo/db/query/sort_pattern_test.cpp
deleted file mode 100644
index 2abe1bffa81..00000000000
--- a/src/mongo/db/query/sort_pattern_test.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "query_shape/serialization_options.h"
-
-#include "mongo/db/exec/document_value/document.h"
-#include "mongo/db/exec/document_value/document_value_test_util.h"
-#include "mongo/db/exec/document_value/value.h"
-#include "mongo/db/pipeline/expression_context_for_test.h"
-#include "mongo/db/query/sort_pattern.h"
-#include "mongo/unittest/unittest.h"
-namespace mongo {
-namespace {
-
-auto getExpCtx() {
- auto nss = NamespaceString("db", "coll");
- return boost::intrusive_ptr<ExpressionContextForTest>{new ExpressionContextForTest(nss)};
-}
-
-TEST(SerializeSortPatternTest, SerializeAndRedactFieldName) {
- auto expCtx = getExpCtx();
- auto sortPattern = SortPattern(fromjson("{val: 1}"), expCtx);
- SerializationOptions opts = SerializationOptions::kMarkIdentifiers_FOR_TEST;
-
- // Most basic sort pattern, confirm that field name gets redacted.
- ASSERT_DOCUMENT_EQ_AUTO( // NOLINT
- R"({"HASH<val>":1})",
- sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts));
-
- // Confirm that multiple sort fields get redacted.
- sortPattern = SortPattern(fromjson("{val: 1, test: -1, third: -1}"), expCtx);
- ASSERT_DOCUMENT_EQ_AUTO( // NOLINT
- R"({"HASH<val>":1,"HASH<test>":-1,"HASH<third>":-1})",
- sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts));
-
- // Test sort pattern that contains an expression.
- sortPattern = SortPattern(fromjson("{val: 1, test: {$meta: \"randVal\"}}"), expCtx);
- ASSERT_DOCUMENT_EQ_AUTO( // NOLINT
- R"({"HASH<val>":1,"$computed1":{"$meta":"randVal"}})",
- sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts));
-
- // Sorting by only an expression results in a made up field name in serialization and therefore
- // doesn't get redacted.
- sortPattern = SortPattern(fromjson("{val: {$meta: \"textScore\"}}"), expCtx);
- ASSERT_DOCUMENT_EQ_AUTO( // NOLINT
- R"({"$computed0":{"$meta":"textScore"}})",
- sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts));
-
- sortPattern = SortPattern(fromjson("{'a.b.c': 1}"), expCtx);
- ASSERT_DOCUMENT_EQ_AUTO( // NOLINT
- R"({"HASH<a>.HASH<b>.HASH<c>":1})",
- sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts));
-}
-
-TEST(SerializeSortPatternTest, SerializeNoRedaction) {
- auto expCtx = getExpCtx();
- auto sortPattern = SortPattern(fromjson("{val: 1}"), expCtx);
- SerializationOptions opts = {};
- opts.transformIdentifiers = false;
- ASSERT_DOCUMENT_EQ_AUTO( // NOLINT
- R"({"val":1})",
- sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization, opts));
-
- // Call serialize() with no options.
- ASSERT_DOCUMENT_EQ_AUTO( // NOLINT
- R"({"val":1})",
- sortPattern.serialize(SortPattern::SortKeySerialization::kForPipelineSerialization));
-}
-
-} // namespace
-} // namespace mongo
diff --git a/src/mongo/db/query/util/deferred.h b/src/mongo/db/query/util/deferred.h
deleted file mode 100644
index a2609bb6b49..00000000000
--- a/src/mongo/db/query/util/deferred.h
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include <functional>
-
-namespace mongo {
-
-/**
- * A template class that provides a way to defer the initialization of an object until its value is
- * actually required. This is also commonly referred to as lazy initialization.
- *
- * Dangers:
- * - This implementation is currently not thread safe, and it shouldn't be used in multi-threaded
- * fashion.
- * - Be careful about using this for lazy initialization of data members and capturing the 'this'
- * variable. Code like this will result in buggy/unsafe move constructors, which would have a
- * dangling reference to the moved-from type:
- *
- * class MyType {
- * int x;
- * // !!! Dangling 'this' when moved !!!
- * Deferred<int> xSquared{[this]() { return this->x * this-> x; };
- * };
- * Instead, it is better to do something like this:
- * class MyType {
- * int xSquared() const {
- * return *_xSquared.get(_x);
- * }
- *
- * int _x;
- * Deferred<int, int> _xSquared{[](int x) { return x * x; };
- * };
- * - As a similar danger, the value is only computed once. if you initialize it with arguments like
- * the above 'xSquared()' implementation, then be cogniscent that the value will never change. If
- * '_x' changes, '_xSquared' will not.
- *
- * A Deferred class can be constructed with either an initial value (eager initialization) or a
- * function that will generate the value when needed.
- */
-template <typename T, typename... Args>
-class Deferred {
-public:
- /**
- * Instantiates a Deffered<T> with the given data - no callbacks or lazy initialization.
- */
- Deferred(T data) : _data(data) {}
-
- /**
- * Stores a function to compute a T later. Please note the warnings described in this class
- * comment.
- */
- Deferred(std::function<T(Args&&...)> initializer) : _initializer(std::move(initializer)) {}
-
- /**
- * Returns a pointer to the managed object. Initializes the object if it hasn't done so already.
- */
- T& get(Args&&... args) const {
- if (_initializer) {
- _data = _initializer(std::forward<Args>(args)...);
- _initializer = nullptr;
- }
- return _data;
- }
-
- /**
- * Dereferences the pointer to the managed object. Note this is only a valid shortcut if there
- * are no arguments to '_initializer'.
- */
- T* operator->() const {
- return &get();
- }
-
- /**
- * Returns a referenced to the managed object. Initializes the object if it hasn't done so
- * already. Note this is only a valid shortcut if there are no arguments to '_initializer'.
- */
- const T& operator*() const {
- return get();
- }
-
- bool isInitialized() const {
- return _initializer ? false : true;
- }
-
-private:
- mutable T _data;
- mutable std::function<T(Args&&...)> _initializer;
-};
-
-} // namespace mongo
diff --git a/src/mongo/db/query/util/deferred_test.cpp b/src/mongo/db/query/util/deferred_test.cpp
deleted file mode 100644
index de256394787..00000000000
--- a/src/mongo/db/query/util/deferred_test.cpp
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/util/deferred.h"
-
-#include "mongo/unittest/unittest.h"
-
-namespace mongo {
-using std::string;
-using namespace std::string_literals;
-
-
-TEST(DeferredTest, EagerInitialization) {
- Deferred<string> eager{"someString"};
- ASSERT_TRUE(eager.isInitialized());
- ASSERT_EQ(eager.get(), "someString"s);
- ASSERT_EQ(*eager, "someString"s);
-}
-
-TEST(DeferredTest, DeferredInitialization) {
- size_t initializationCount = 0;
- Deferred<string> deferred{[&]() {
- initializationCount++;
- return "someString"s;
- }};
- ASSERT_FALSE(deferred.isInitialized());
-
- // Ensure the deferred object wasn't initialized on creation.
- ASSERT_EQ(initializationCount, 0);
-
- // Ensure that the deferred object is initialized on pointer dereferences.
- ASSERT_FALSE(deferred->empty());
- ASSERT_TRUE(deferred.isInitialized());
-
- ASSERT_EQ(initializationCount, 1);
-
- // Ensure that the content of the deferred object is equal to its raw counterpart, while also
- // verifing that it is initialized at most once.
- ASSERT_EQ(deferred.get(), "someString"s);
- ASSERT_EQ(initializationCount, 1);
-}
-
-TEST(DeferredTest, DeferredInitializationWithOneArgument) {
- size_t initializationCount = 0;
- Deferred<string, const string&> deferred{[&](const string& input) {
- initializationCount++;
- return "{" + input + "}";
- }};
-
- // Ensure the deferred object wasn't initialized on creation.
- ASSERT_EQ(initializationCount, 0);
-
- // Ensure that the content of the deferred object is equal to its raw counterpart, while also
- // verifing that it is initialized at most once.
- ASSERT_EQ(deferred.get("more curlies"), "{more curlies}"s);
- ASSERT_EQ(initializationCount, 1);
-
- // Note that the value is cached, so it's not really valid to call it with a different argument.
- ASSERT_EQ(deferred.get("merganser"), "{more curlies}"s);
- ASSERT_EQ(initializationCount, 1);
-}
-
-TEST(DeferredTest, DeferredInitializationWithTwoArgs) {
- Deferred<string, const string&, const string&> deferred{
- [&](const auto& input, const auto& prefix) { return prefix + input; }};
-
- ASSERT_EQ(deferred.get("cowbell", "more "), "more cowbell"s);
- ASSERT_EQ(deferred.get("cowbell", "more "), "more cowbell"s);
- ASSERT_EQ(deferred.get("cowbell", "less?"), "more cowbell"s);
- ASSERT_EQ(deferred.get("tests", "better"), "more cowbell"s);
-}
-} // namespace mongo
diff --git a/src/mongo/db/query/util/memory_util.cpp b/src/mongo/db/query/util/memory_util.cpp
deleted file mode 100644
index 8a206deb2b7..00000000000
--- a/src/mongo/db/query/util/memory_util.cpp
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Copyright (C) 2021-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kQuery
-
-#include "mongo/db/query/util/memory_util.h"
-
-#include <cstddef>
-#include <pcrecpp.h>
-
-#include "mongo/logv2/log.h"
-#include "mongo/util/processinfo.h"
-
-
-namespace mongo::memory_util {
-
-StatusWith<MemoryUnits> parseUnitString(const std::string& strUnit) {
- if (strUnit.empty()) {
- return Status(ErrorCodes::Error{6007010}, "Unit value cannot be empty");
- }
-
- if (strUnit[0] == '%') {
- return MemoryUnits::kPercent;
- } else if (strUnit[0] == 'M' || strUnit[0] == 'm') {
- return MemoryUnits::kMB;
- } else if (strUnit[0] == 'G' || strUnit[0] == 'g') {
- return MemoryUnits::kGB;
- }
-
- return Status(ErrorCodes::Error{6007011}, "Incorrect unit value");
-}
-
-StatusWith<MemorySize> MemorySize::parse(const std::string& str) {
- pcrecpp::RE_Options opt;
- opt.set_caseless(true);
- // Looks for a floating point number with followed by a unit suffix (MB, GB, %).
- pcrecpp::RE re("\\s*(\\d+\\.?\\d*)\\s*(MB|GB|%)\\s*", opt);
-
- double size{};
- std::string strUnit{};
- if (!re.FullMatch(str, &size, &strUnit)) {
- return {ErrorCodes::Error{6007012}, "Unable to parse memory size string"};
- }
-
- auto statusWithUnit = parseUnitString(strUnit);
- if (!statusWithUnit.isOK()) {
- return statusWithUnit.getStatus();
- }
- return MemorySize{size, statusWithUnit.getValue()};
-}
-
-size_t convertToSizeInBytes(const MemorySize& memSize) {
- constexpr size_t kBytesInMB = 1024 * 1024;
- constexpr size_t kMBytesInGB = 1024;
-
- double sizeInMB = memSize.size;
-
- switch (memSize.units) {
- case MemoryUnits::kPercent:
- sizeInMB *= ProcessInfo::getMemSizeMB() / 100.0;
- break;
- case MemoryUnits::kMB:
- break;
- case MemoryUnits::kGB:
- sizeInMB *= kMBytesInGB;
- break;
- }
-
- return static_cast<size_t>(sizeInMB * kBytesInMB);
-}
-
-size_t getRequestedMemSizeInBytes(const MemorySize& memSize) {
- size_t planCacheSize = convertToSizeInBytes(memSize);
- uassert(5968001,
- "Cache size must be at least 1KB * number of cores",
- planCacheSize >= 1024 * ProcessInfo::getNumCores());
- return planCacheSize;
-}
-
-/**
- * Sets upper limit on a storage structure's size. Either that structure's maximumSize or to
- * percentage of the total system's memory (both known at call site), whichever is smaller.
- */
-size_t capMemorySize(size_t requestedSizeBytes,
- size_t maximumSizeGB,
- double percentTotalSystemMemory) {
- constexpr size_t kBytesInGB = 1024 * 1024 * 1024;
- // Express maximum size in bytes.
- const size_t maximumSizeBytes = maximumSizeGB * kBytesInGB;
- const memory_util::MemorySize limitToProcessSize{percentTotalSystemMemory,
- memory_util::MemoryUnits::kPercent};
- const size_t limitToProcessSizeInBytes = convertToSizeInBytes(limitToProcessSize);
-
- // The size will be capped by the minimum of the two values defined above.
- const size_t upperLimit = std::min(maximumSizeBytes, limitToProcessSizeInBytes);
-
- if (requestedSizeBytes > upperLimit) {
- requestedSizeBytes = upperLimit;
- }
- return requestedSizeBytes;
-}
-} // namespace mongo::memory_util
diff --git a/src/mongo/db/query/util/memory_util_test.cpp b/src/mongo/db/query/util/memory_util_test.cpp
deleted file mode 100644
index 78f7b3098d6..00000000000
--- a/src/mongo/db/query/util/memory_util_test.cpp
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Copyright (C) 2021-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/query/util/memory_util.h"
-
-#include "mongo/unittest/unittest.h"
-
-namespace mongo::memory_util {
-
-bool operator==(const MemorySize& lhs, const MemorySize& rhs) {
- constexpr double kEpsilon = 1e-10;
- return std::abs(lhs.size - rhs.size) < kEpsilon && lhs.units == rhs.units;
-}
-
-TEST(MemorySizeTest, ParseUnitStringPercent) {
- ASSERT_TRUE(MemoryUnits::kPercent == parseUnitString("%"));
-}
-
-TEST(MemorySizeTest, ParseUnitStringMB) {
- ASSERT_TRUE(MemoryUnits::kMB == parseUnitString("MB"));
- ASSERT_TRUE(MemoryUnits::kMB == parseUnitString("mb"));
- ASSERT_TRUE(MemoryUnits::kMB == parseUnitString("mB"));
- ASSERT_TRUE(MemoryUnits::kMB == parseUnitString("Mb"));
-}
-
-TEST(MemorySizeTest, ParseUnitStringGB) {
- ASSERT_TRUE(MemoryUnits::kGB == parseUnitString("GB"));
- ASSERT_TRUE(MemoryUnits::kGB == parseUnitString("gb"));
- ASSERT_TRUE(MemoryUnits::kGB == parseUnitString("gB"));
- ASSERT_TRUE(MemoryUnits::kGB == parseUnitString("Gb"));
-}
-
-TEST(MemorySizeTest, ParseUnitStringIncorrectValue) {
- ASSERT_NOT_OK(parseUnitString("").getStatus());
- ASSERT_NOT_OK(parseUnitString(" ").getStatus());
- ASSERT_NOT_OK(parseUnitString("KB").getStatus());
-}
-
-TEST(MemorySizeTest, ParseMemorySize) {
- ASSERT_TRUE((MemorySize{10.0, MemoryUnits::kPercent}) == MemorySize::parse("10%"));
- ASSERT_TRUE((MemorySize{300.0, MemoryUnits::kMB}) == MemorySize::parse("300MB"));
- ASSERT_TRUE((MemorySize{4.0, MemoryUnits::kGB}) == MemorySize::parse("4GB"));
- ASSERT_TRUE((MemorySize{5.1, MemoryUnits::kPercent}) == MemorySize::parse(" 5.1%"));
- ASSERT_TRUE((MemorySize{11.1, MemoryUnits::kMB}) == MemorySize::parse("11.1 mb"));
- ASSERT_TRUE((MemorySize{12.1, MemoryUnits::kGB}) == MemorySize::parse(" 12.1 Gb "));
-}
-} // namespace mongo::memory_util
diff --git a/src/mongo/db/query/wildcard_multikey_paths.cpp b/src/mongo/db/query/wildcard_multikey_paths.cpp
index eea11ccd4b5..fb27c8b34f5 100644
--- a/src/mongo/db/query/wildcard_multikey_paths.cpp
+++ b/src/mongo/db/query/wildcard_multikey_paths.cpp
@@ -31,7 +31,7 @@
#include "mongo/db/query/wildcard_multikey_paths.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/index/wildcard_access_method.h"
#include "mongo/db/query/index_bounds_builder.h"
#include "mongo/db/record_id_helpers.h"