diff options
Diffstat (limited to 'jstests/core/timeseries')
30 files changed, 2034 insertions, 319 deletions
diff --git a/jstests/core/timeseries/bucket_unpacking_with_sort.js b/jstests/core/timeseries/bucket_unpacking_with_sort.js index 029697be74a..695e94daec9 100644 --- a/jstests/core/timeseries/bucket_unpacking_with_sort.js +++ b/jstests/core/timeseries/bucket_unpacking_with_sort.js @@ -17,6 +17,9 @@ * requires_timeseries, * # Explain of a resolved view must be executed by mongos. * directly_against_shardsvrs_incompatible, + * # Time-series hint by index key did not exist in 5.0. + * # Also, the 5.0 version of this feature does not support reverse collscan. + * requires_fcv_60, * ] */ (function() { diff --git a/jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js b/jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js new file mode 100644 index 00000000000..e74d1c3e914 --- /dev/null +++ b/jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js @@ -0,0 +1,269 @@ +/** + * Test that sort queries work properly on dates ouside the 32 bit epoch range, + * [1970-01-01 00:00:00 UTC - 2038-01-29 03:13:07 UTC], when a collection scan is used. + * + * @tags: [ + * # Explain of a resolved view must be executed by mongos. + * directly_against_shardsvrs_incompatible, + * # This complicates aggregation extraction. + * do_not_wrap_aggregations_in_facets, + * # Refusing to run a test that issues an aggregation command with explain because it may + * # return incomplete results if interrupted by a stepdown. + * does_not_support_stepdowns, + * # We need a timeseries collection. + * requires_timeseries, + * # Cannot insert into a time-series collection in a multi-document transaction. + * does_not_support_transactions, + * # Time-series hint by index key did not exist in 5.0 + * requires_fcv_60, + * ] + */ + +(function() { +"use strict"; + +load("jstests/aggregation/extras/utils.js"); // For getExplainedPipelineFromAggregation. +load("jstests/core/timeseries/libs/timeseries.js"); +load('jstests/libs/analyze_plan.js'); + +if (!TimeseriesTest.bucketUnpackWithSortEnabled(db.getMongo())) { + jsTestLog("Skipping test because 'BucketUnpackWithSort' is disabled."); + return; +} + +const timeFieldName = "t"; + +// Create unindexed collection +const coll = db.timeseries_internal_bounded_sort_extended_range; +const buckets = db['system.buckets.' + coll.getName()]; +coll.drop(); +assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField: 't'}})); + +// Create collection indexed on time +const collIndexed = db.timeseries_internal_bounded_sort_extended_range_with_index; +const bucketsIndexed = db['system.buckets.' + collIndexed.getName()]; +collIndexed.drop(); +assert.commandWorked(db.createCollection(collIndexed.getName(), {timeseries: {timeField: 't'}})); +assert.commandWorked(collIndexed.createIndex({'t': 1})); + +jsTestLog(collIndexed.getIndexes()); +jsTestLog(bucketsIndexed.getIndexes()); + +function numShards() { + return db.getSiblingDB('config').shards.count(); +} +for (const collection of [buckets, bucketsIndexed]) { + if (FixtureHelpers.isSharded(collection) && numShards() >= 2) { + // Split and move data to create an interesting scenario: we have some data on each shard, + // but all the extended-range data is on a non-primary shard. This means view resolution is + // unaware of the extended-range data, because that happens on the primary shard. + + const shards = db.getSiblingDB('config').shards.find().toArray(); + const [shardName0, shardName1] = shards.map(doc => doc._id); + + assert.commandWorked(db.adminCommand({movePrimary: db.getName(), to: shardName0})); + const collName = collection.getFullName(); + // Our example data has documents between 2000-2003, and these dates are non-wrapping. + // So this goes on the primary shard, and everything else goes on the non-primary. + assert.commandWorked(sh.splitAt(collName, {'control.min.t': ISODate('2000-01-01')})); + assert.commandWorked(sh.splitAt(collName, {'control.min.t': ISODate('2003-01-01')})); + assert.commandWorked( + sh.moveChunk(collName, {'control.min.t': ISODate('1969-01-01')}, shardName1)); + assert.commandWorked( + sh.moveChunk(collName, {'control.min.t': ISODate('2000-01-01')}, shardName0)); + assert.commandWorked( + sh.moveChunk(collName, {'control.min.t': ISODate('2003-01-01')}, shardName1)); + } +} + +const intervalMillis = 60000; +function insertBucket(start) { + jsTestLog("Inserting bucket starting with " + Date(start).toString()); + + const batchSize = 1000; + + const batch = + Array.from({length: batchSize}, (_, j) => ({t: new Date(start + j * intervalMillis)})); + + assert.commandWorked(coll.insert(batch)); + assert.commandWorked(collIndexed.insert(batch)); +} + +// Insert some data. We'll insert 5 buckets in each range, with values < 0, +// values between 0 and FFFFFFFF(unsigned), and values > FFFFFFFF. It turns out, however, +// that Javascript's Date doesn't handle dates beyond 2039 either, so we rely on lower dates +// to test for unexpected behavior. +function insertDocuments() { + // We want to choose the underflow and overflow lower bits in such a way that we + // encourage wrong results when the upper bytes are removed. + const underflowMin = new Date("1969-01-01").getTime(); // Year before the 32 bit epoch + const normalMin = new Date("2002-01-01").getTime(); // Middle of the 32 bit epoch + + insertBucket(underflowMin); + + var numBatches = 5; + + const batchOffset = Math.floor(intervalMillis / (numBatches + 1)); + for (let i = 0; i < numBatches; ++i) { + const start = normalMin + i * batchOffset; + insertBucket(start); + } + assert.gt(buckets.aggregate([{$count: 'n'}]).next().n, 1, 'Expected more than one bucket'); +} + +insertDocuments(); + +const unpackStage = getAggPlanStages(coll.explain().aggregate(), '$_internalUnpackBucket')[0]; + +function assertSorted(result, ascending) { + let prev = ascending ? {t: -Infinity} : {t: Infinity}; + for (const doc of result) { + if (ascending) { + assert.lt(+prev.t, + +doc.t, + 'Found two docs not in ascending time order: ' + tojson({prev, doc})); + } else { + assert.gt(+prev.t, + +doc.t, + 'Found two docs not in descending time order: ' + tojson({prev, doc})); + } + + prev = doc; + } +} + +function checkAgainstReferenceBoundedSortUnexpected( + collection, reference, pipeline, hint, sortOrder) { + const options = hint ? {hint: hint} : {}; + + const bucket = db['system.buckets.' + coll.getName()]; + + const plan = collection.explain().aggregate(pipeline, options); + if (FixtureHelpers.isSharded(buckets) && numShards() >= 2) { + // With a sharded collection, some shards might not have any extended-range data, + // so they might still use $_internalBoundedSort. But we know at least one + // shard has extended-range data, so we know at least one shard has to + // use a blocking sort. + const bounded = getAggPlanStages(plan, "$_internalBoundedSort"); + const blocking = getAggPlanStages(plan, "$sort"); + assert.gt(blocking.length, 0, {bounded, blocking, plan}); + assert.lt(bounded.length, + FixtureHelpers.numberOfShardsForCollection(buckets), + {bounded, blocking, plan}); + } else { + const stages = getAggPlanStages(plan, "$_internalBoundedSort"); + assert.eq([], stages, plan); + } + + const opt = collection.aggregate(pipeline, options).toArray(); + assertSorted(opt, sortOrder); + + assert.eq(reference.length, opt.length); + for (var i = 0; i < opt.length; ++i) { + assert.docEq(reference[i], opt[i]); + } +} + +function checkAgainstReferenceBoundedSortExpected( + collection, reference, pipeline, hint, sortOrder) { + const options = hint ? {hint: hint} : {}; + + const plan = collection.explain().aggregate(pipeline, options); + const stages = getAggPlanStages(plan, "$_internalBoundedSort"); + assert.neq([], stages, plan); + + const opt = collection.aggregate(pipeline, options).toArray(); + assertSorted(opt, sortOrder); + + assert.eq(reference.length, opt.length); + for (var i = 0; i < opt.length; ++i) { + assert.docEq(reference[i], opt[i]); + } +} + +function runTest(ascending) { + const reference = buckets + .aggregate([ + unpackStage, + {$_internalInhibitOptimization: {}}, + {$sort: {t: ascending ? 1 : -1}}, + ]) + .toArray(); + assertSorted(reference, ascending); + + // Check plan using collection scan + checkAgainstReferenceBoundedSortUnexpected(coll, + reference, + [ + {$sort: {t: ascending ? 1 : -1}}, + ], + {}, + ascending); + + // Check plan using hinted collection scan + checkAgainstReferenceBoundedSortUnexpected(coll, + reference, + [ + {$sort: {t: ascending ? 1 : -1}}, + ], + {"$natural": ascending ? 1 : -1}, + ascending); + + const referenceIndexed = bucketsIndexed + .aggregate([ + unpackStage, + {$_internalInhibitOptimization: {}}, + {$sort: {t: ascending ? 1 : -1}}, + ]) + .toArray(); + assertSorted(referenceIndexed, ascending); + + // Check plan using index scan. If we've inserted a date before 1-1-1970, we round the min + // up towards 1970, rather then down, which has the effect of increasing the control.min.t. + // This means the minimum time in the bucket is likely to be lower than indicated and thus, + // actual dates may be out of order relative to what's indicated by the bucket bounds. + checkAgainstReferenceBoundedSortUnexpected(collIndexed, + referenceIndexed, + [ + {$sort: {t: ascending ? 1 : -1}}, + ], + {}, + ascending); + + // Check plan using hinted index scan + checkAgainstReferenceBoundedSortUnexpected(collIndexed, + referenceIndexed, + [ + {$sort: {t: ascending ? 1 : -1}}, + ], + {"t": 1}, + ascending); + + // Check plan using hinted collection scan + checkAgainstReferenceBoundedSortUnexpected(collIndexed, + referenceIndexed, + [ + {$sort: {t: ascending ? 1 : -1}}, + ], + {"$natural": ascending ? 1 : -1}, + ascending); + + // The workaround in all cases is to create a reverse index on the time field, though + // it's necessary to force use of the reversed index. + const reverseIdxName = "reverseIdx"; + collIndexed.createIndex({t: -1}, {name: reverseIdxName}); + + checkAgainstReferenceBoundedSortExpected(collIndexed, + referenceIndexed, + [ + {$sort: {t: ascending ? 1 : -1}}, + ], + {"t": -1}, + ascending); + + collIndexed.dropIndex(reverseIdxName); +} + +runTest(false); // descending +runTest(true); // ascending +})(); diff --git a/jstests/core/timeseries/bucket_unpacking_with_sort_plan_cache.js b/jstests/core/timeseries/bucket_unpacking_with_sort_plan_cache.js index 25e1c99312e..e69a8d7a860 100644 --- a/jstests/core/timeseries/bucket_unpacking_with_sort_plan_cache.js +++ b/jstests/core/timeseries/bucket_unpacking_with_sort_plan_cache.js @@ -21,6 +21,8 @@ * directly_against_shardsvrs_incompatible, * # We use the profiler to get info in order to force replanning. * requires_profiling, + * # The 5.0 version of this feature does not support reverse collscan. + * requires_fcv_60, * ] */ (function() { diff --git a/jstests/core/timeseries/libs/timeseries.js b/jstests/core/timeseries/libs/timeseries.js index ea8d71c87a1..d9e22926520 100644 --- a/jstests/core/timeseries/libs/timeseries.js +++ b/jstests/core/timeseries/libs/timeseries.js @@ -52,9 +52,16 @@ var TimeseriesTest = class { } static bucketUnpackWithSortEnabled(conn) { - return assert - .commandWorked(conn.adminCommand({getParameter: 1, featureFlagBucketUnpackWithSort: 1})) - .featureFlagBucketUnpackWithSort.value; + const resp50 = conn.adminCommand({getParameter: 1, featureFlagBucketUnpackWithSort50: 1}); + const resp = conn.adminCommand({getParameter: 1, featureFlagBucketUnpackWithSort: 1}); + assert(Boolean(resp50.ok) ^ Boolean(resp.ok), + `Expected exactly one of these feature flags to exist: ${tojson(resp50)} vs ${ + tojson(resp)}`); + if (resp50.ok) { + return assert.commandWorked(resp50).featureFlagBucketUnpackWithSort50.value; + } else { + return assert.commandWorked(resp).featureFlagBucketUnpackWithSort.value; + } } /** @@ -199,7 +206,7 @@ var TimeseriesTest = class { static ensureDataIsDistributedIfSharded(coll, splitPointDate) { const db = coll.getDB(); - const buckets = db["system.buckets." + coll.getName()]; + const buckets = db[this.getBucketsCollName(coll.getName())]; if (FixtureHelpers.isSharded(buckets)) { const timeFieldName = db.getCollectionInfos({name: coll.getName()})[0].options.timeseries.timeField; @@ -248,4 +255,8 @@ var TimeseriesTest = class { } } } + + static getBucketsCollName(collName) { + return `system.buckets.${collName}`; + } }; diff --git a/jstests/core/timeseries/nondefault_collation.js b/jstests/core/timeseries/nondefault_collation.js index 3ff1c63d330..3188168d23f 100644 --- a/jstests/core/timeseries/nondefault_collation.js +++ b/jstests/core/timeseries/nondefault_collation.js @@ -1,8 +1,13 @@ /** - * Test ensures that users can specify non-default collation when querying on time-series - * collections. + * Correctness tests for TS collections with collation that might not match the explicit collation, + * specified in the query. + * + * Queries on timeseries attempt various optimizations to avoid unpacking of buckets. These rely on + * the meta field and the control data (currently, min and max), computed for each bucket. + * Collection's collation might affect the computed control values. * * @tags: [ + * assumes_unsharded_collection, * requires_non_retryable_writes, * requires_pipeline_optimization, * requires_getmore, @@ -21,123 +26,191 @@ load("jstests/libs/analyze_plan.js"); const coll = db.timeseries_nondefault_collation; const bucketsColl = db.getCollection('system.buckets.' + coll.getName()); -coll.drop(); // implicitly drops bucketsColl. - -const timeFieldName = 'time'; -const metaFieldName = 'meta'; - const numericOrdering = { - collation: {locale: "en_US", numericOrdering: true} + locale: "en_US", + numericOrdering: true, + strength: 1 // case and diacritics ignored }; - const caseSensitive = { - collation: {locale: "en_US", strength: 1, caseLevel: true, numericOrdering: true} + locale: "en_US", + strength: 1, + caseLevel: true }; - const diacriticSensitive = { - collation: {locale: "en_US", strength: 2} + locale: "en_US", + strength: 2, + caseLevel: false }; - -const englishCollation = { - locale: 'en', +const insensitive = { + locale: "en_US", strength: 1 }; -const simpleCollation = { - collation: {locale: "simple"} -}; +// Find on meta field isn't different from a find on any other view, but let's check it anyway. +(function testFind_MetaField() { + coll.drop(); -assert.commandWorked(db.createCollection(coll.getName(), { - timeseries: {timeField: timeFieldName, metaField: metaFieldName}, - collation: englishCollation -})); -assert.contains(bucketsColl.getName(), db.getCollectionNames()); - -assert.commandWorked( - coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "1", name: 'A', name2: "á"})); -assert.commandWorked( - coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "2", name: 'a', name2: "á"})); -assert.commandWorked( - coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "5", name: 'A', name2: "á"})); -assert.commandWorked( - coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "10", name: 'a', name2: "á"})); -assert.commandWorked( - coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "20", name: 'A', name2: "a"})); -assert.commandWorked( - coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "50", name: 'B', name2: "a"})); -assert.commandWorked( - coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "100", name: 'b', name2: "a"})); -assert.commandWorked( - coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "200", name: 'B', name2: "a"})); -assert.commandWorked( - coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "500", name: 'b', name2: "a"})); - -// Default collation is case and diacretic insensitive. -assert.eq(2, coll.aggregate([{$sortByCount: "$name"}]).itcount()); -assert.eq(1, coll.aggregate([{$sortByCount: "$name2"}]).itcount()); - -// Test that a explicit collation different from collection's default passes for a timeseries -// collection. -let results = - coll.aggregate([{$bucket: {groupBy: "$meta", boundaries: ["1", "10", "100", "1000"]}}], - numericOrdering) - .toArray(); -assert.eq(3, results.length); -assert.eq({_id: "1", count: 3}, results[0]); -assert.eq({_id: "10", count: 3}, results[1]); -assert.eq({_id: "100", count: 3}, results[2]); - -assert.eq(4, coll.aggregate([{$sortByCount: "$name"}], caseSensitive).itcount()); -assert.eq(2, coll.aggregate([{$sortByCount: "$name2"}], diacriticSensitive).itcount()); - -coll.drop(); -const defaultCollation = { - locale: "en", - numericOrdering: true, - caseLevel: true, - strength: 2 -}; -assert.commandWorked(db.createCollection(coll.getName(), { - timeseries: {timeField: timeFieldName, metaField: metaFieldName}, - collation: defaultCollation -})); -assert.contains(bucketsColl.getName(), db.getCollectionNames()); -assert.commandWorked(coll.createIndex({[metaFieldName]: 1}, {collation: {locale: "simple"}})); - -assert.commandWorked(coll.insert( - {[timeFieldName]: ISODate(), [metaFieldName]: 1, name: 'A', name2: "á", value: "1"})); -assert.commandWorked(coll.insert( - {[timeFieldName]: ISODate(), [metaFieldName]: 2, name: 'a', name2: "á", value: "11"})); -assert.commandWorked(coll.insert( - {[timeFieldName]: ISODate(), [metaFieldName]: 1, name: 'A', name2: "á", value: "50"})); -assert.commandWorked(coll.insert( - {[timeFieldName]: ISODate(), [metaFieldName]: 1, name: 'a', name2: "á", value: "100"})); -assert.commandWorked(coll.insert( - {[timeFieldName]: ISODate(), [metaFieldName]: "2", name: 'A', name2: "a", value: "3"})); -assert.commandWorked(coll.insert( - {[timeFieldName]: ISODate(), [metaFieldName]: "5", name: 'B', name2: "a", value: "-100"})); -assert.commandWorked(coll.insert( - {[timeFieldName]: ISODate(), [metaFieldName]: "1", name: 'b', name2: "a", value: "-200"})); -assert.commandWorked(coll.insert( - {[timeFieldName]: ISODate(), [metaFieldName]: "2", name: 'B', name2: "a", value: "1000"})); -assert.commandWorked(coll.insert( - {[timeFieldName]: ISODate(), [metaFieldName]: "5", name: 'b', name2: "a", value: "4"})); - -// This collection has been created using non simple collation. The collection was then indexed on -// its metadata using simple collation. These tests confirm that queries on the indexed field using -// nondefault (simple) collation use the index. They also confirm that queries that don't involve -// strings but do use default collation, on indexed fields, also use the index. -const nonDefaultCollationQuery = coll.find({meta: 2}, {collation: englishCollation}).explain(); -assert(aggPlanHasStage(nonDefaultCollationQuery, "IXSCAN"), nonDefaultCollationQuery); - -const simpleNonDefaultCollationQuery = coll.find({meta: 2}, simpleCollation).explain(); -assert(aggPlanHasStage(simpleNonDefaultCollationQuery, "IXSCAN"), simpleNonDefaultCollationQuery); - -const defaultCollationQuery = coll.find({meta: 1}, {collation: defaultCollation}).explain(); -assert(aggPlanHasStage(defaultCollationQuery, "IXSCAN"), defaultCollationQuery); - -// This test guarantees that the bucket's min/max matches the query's min/max regardless of -// collation. -results = coll.find({value: {$gt: "4"}}, simpleCollation); -assert.eq(4, results.itcount()); + assert.commandWorked(db.createCollection( + coll.getName(), + {timeseries: {timeField: 'time', metaField: 'meta'}, collation: numericOrdering})); + assert.contains(bucketsColl.getName(), db.getCollectionNames()); + + assert.commandWorked(coll.insert({time: ISODate(), meta: "1", value: 42})); + assert.commandWorked(coll.insert({time: ISODate(), meta: "10", value: 42})); + assert.commandWorked(coll.insert({time: ISODate(), meta: "5", value: 42})); + + // Use the collection's collation with numeric ordering. + let res1 = coll.find({meta: {$gt: "4"}}); + assert.eq(2, res1.itcount(), res1.toArray()); // should match "5" and "10" + + // Use explicit collation with lexicographic ordering. + let res2 = coll.find({meta: {$gt: "4"}}).collation(insensitive); + assert.eq(1, res2.itcount(), res2.toArray()); // should match only "5" +}()); + +// For the measurement fields each bucket computes additional "control values", such as min/max and +// might use them to avoid unpacking. +(function testFind_MeasurementField() { + coll.drop(); + + assert.commandWorked(db.createCollection( + coll.getName(), + {timeseries: {timeField: 'time', metaField: 'meta'}, collation: numericOrdering})); + assert.contains(bucketsColl.getName(), db.getCollectionNames()); + + // The 'numericOrdering' on the collection means that the max of the bucket with the three docs + // below is "10" (while the lexicographic max is "5"). + assert.commandWorked(coll.insert({time: ISODate(), meta: 42, value: "1"})); + assert.commandWorked(coll.insert({time: ISODate(), meta: 42, value: "10"})); + assert.commandWorked(coll.insert({time: ISODate(), meta: 42, value: "5"})); + + // A query with default collation would use the bucket's min/max and find the matches. We are + // not checking the unpacking optimizations here as it's not a concern of collation per se. + let res1 = coll.find({value: {$gt: "4"}}); + assert.eq(2, res1.itcount(), res1.toArray()); // should match "5" and "10" + + // If a query with 'insensitive' collation, which doesn't do numeric ordering, used the bucket's + // min/max it would miss the bucket. Check, that it doesn't. + let res2 = coll.find({value: {$gt: "4"}}).collation(insensitive); + assert.eq(1, res2.itcount(), res2.toArray()); // should match only "5" +}()); + +(function testAgg_GroupByMetaField() { + coll.drop(); + + assert.commandWorked(db.createCollection( + coll.getName(), + {timeseries: {timeField: 'time', metaField: 'meta'}, collation: numericOrdering})); + assert.contains(bucketsColl.getName(), db.getCollectionNames()); + + assert.commandWorked(coll.insert({time: ISODate(), meta: "1", val: 1})); + assert.commandWorked(coll.insert({time: ISODate(), meta: "5", val: 1})); + + // Using collection's collation with numeric ordering. + let res1 = + coll.aggregate([{$bucket: {groupBy: "$meta", boundaries: ["1", "10", "50"]}}]).toArray(); + assert.eq(1, res1.length); + assert.eq({_id: "1", count: 2}, res1[0]); + + // Using explicit collation with lexicographic ordering. + let res2 = coll.aggregate([{$bucket: {groupBy: "$meta", boundaries: ["1", "10", "50"]}}], + {collation: insensitive}) + .toArray(); + assert.eq(2, res2.length); + assert.eq({_id: "1", count: 1}, res2[0]); // "1" goes here + assert.eq({_id: "10", count: 1}, res2[1]); // "5" goes here +}()); + +(function testAgg_GroupByMeasurementField() { + coll.drop(); + + assert.commandWorked(db.createCollection( + coll.getName(), + {timeseries: {timeField: 'time', metaField: 'meta'}, collation: insensitive})); + assert.contains(bucketsColl.getName(), db.getCollectionNames()); + + // Cause two different buckets with various case/diacritics in each for the measurement 'name'. + assert.commandWorked(coll.insert({time: ISODate(), meta: "a", name: 'A'})); + assert.commandWorked(coll.insert({time: ISODate(), meta: "a", name: 'a'})); + assert.commandWorked(coll.insert({time: ISODate(), meta: "a", name: 'á'})); + assert.commandWorked(coll.insert({time: ISODate(), meta: "b", name: 'A'})); + assert.commandWorked(coll.insert({time: ISODate(), meta: "b", name: 'a'})); + assert.commandWorked(coll.insert({time: ISODate(), meta: "b", name: 'ä'})); + + // Test with the collection's collation, which is case and diacritic insensitive. + assert.eq(1, coll.aggregate([{$sortByCount: "$name"}]).itcount()); + + // Test with explicit collation that is different from the collection's. + assert.eq(2, coll.aggregate([{$sortByCount: "$name"}], {collation: caseSensitive}).itcount()); + assert.eq(3, + coll.aggregate([{$sortByCount: "$name"}], {collation: diacriticSensitive}).itcount()); +}()); + +// For $group queries that would put whole buckets into the same group, it might be possible to +// avoid unpacking if the information the group is computing is exposed in the control data of each +// bucket. Currently, we only do this optimization for min/max with the meta as the group key. +(function testAgg_MinMaxOptimization() { + coll.drop(); + + assert.commandWorked(db.createCollection( + coll.getName(), + {timeseries: {timeField: 'time', metaField: 'meta'}, collation: numericOrdering})); + assert.contains(bucketsColl.getName(), db.getCollectionNames()); + + // These two docs will be placed in the same bucket, and the max for the bucket will be computed + // using collection's collation, that is, it should be "10". + assert.commandWorked(coll.insert({time: ISODate(), meta: 42, val: "10"})); + assert.commandWorked(coll.insert({time: ISODate(), meta: 42, val: "5"})); + + // Let's check our understanding of what happens with the bucketing as otherwise the tests below + // won't be testing what we think they are. + let buckets = bucketsColl.find().toArray(); + assert.eq(1, buckets.length, "All docs should be placed into the same bucket"); + assert.eq("10", buckets[0].control.max.val, "Computed max control for 'val' measurement"); + + // Use the collection's collation with numeric ordering. + let res1 = coll.aggregate([{$group: {_id: "$meta", v: {$max: "$val"}}}]).toArray(); + assert.eq("10", res1[0].v, "max val in numeric ordering per the collection's collation"); + + // Use the collection's collation with lexicographic ordering. + let res2 = + coll.aggregate([{$group: {_id: "$meta", v: {$max: "$val"}}}], {collation: insensitive}) + .toArray(); + assert.eq("5", res2[0].v, "max val in lexicographic ordering per the query collation"); +}()); + +(function testFind_IndexWithDifferentCollation() { + coll.drop(); + + assert.commandWorked(db.createCollection( + coll.getName(), + {timeseries: {timeField: 'time', metaField: 'meta'}, collation: diacriticSensitive})); + assert.contains(bucketsColl.getName(), db.getCollectionNames()); + + // Create index with a different collation. + assert.commandWorked(coll.createIndex({meta: 1}, {collation: insensitive})); + + // We only check that the correct plan is chosen so the contents of the collection don't matter + // as long as it's not empty. + assert.commandWorked(coll.insert({time: ISODate(), meta: 42})); + assert.commandWorked(coll.insert({time: ISODate(), meta: "the answer"})); + + // Queries that don't specify explicit collation should use the collection's default collation + // which isn't compatible with the index, so the index should NOT be used. + let query = coll.find({meta: "str"}).explain(); + assert(!aggPlanHasStage(query, "IXSCAN"), query); + + // Queries with an explicit collation which isn't compatible with the index, should NOT do + // index scan. + query = coll.find({meta: "str"}).collation(caseSensitive).explain(); + assert(!aggPlanHasStage(query, "IXSCAN"), query); + + // Queries with the same collation as in the index, should do index scan. + query = coll.find({meta: "str"}).collation(insensitive).explain(); + assert(aggPlanHasStage(query, "IXSCAN"), query); + + // Numeric queries that don't rely on collation should do index scan. + query = coll.find({meta: 1}).explain(); + assert(aggPlanHasStage(query, "IXSCAN"), query); +}()); }()); diff --git a/jstests/core/timeseries/partialFilterExpression_with_internalBucketGeoWithin.js b/jstests/core/timeseries/partialFilterExpression_with_internalBucketGeoWithin.js index 5be75d79e28..61521e7f86e 100644 --- a/jstests/core/timeseries/partialFilterExpression_with_internalBucketGeoWithin.js +++ b/jstests/core/timeseries/partialFilterExpression_with_internalBucketGeoWithin.js @@ -10,12 +10,11 @@ */ load("jstests/libs/analyze_plan.js"); +load("jstests/libs/feature_flag_util.js"); load('jstests/noPassthrough/libs/index_build.js'); -(function() { -const isFeatureEnabled = db.adminCommand({getParameter: 1, featureFlagTimeseriesMetricIndexes: 1}) - .featureFlagTimeseriesMetricIndexes.value; -if (isFeatureEnabled) { +(function() { +if (FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { const timeFieldName = "timestamp"; const coll = db.partialFilterExpression_with_internalBucketGeoWithin; diff --git a/jstests/core/timeseries/timeseries_bucket_rename.js b/jstests/core/timeseries/timeseries_bucket_rename.js deleted file mode 100644 index 98a0b73b810..00000000000 --- a/jstests/core/timeseries/timeseries_bucket_rename.js +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Tests that a system.buckets collection cannot be renamed. - * - * @tags: [ - * does_not_support_stepdowns, - * does_not_support_transactions, - * requires_getmore, - * ] - */ -(function() { -'use strict'; - -const coll = db.timeseries_bucket_rename; -const bucketsColl = db.getCollection('system.buckets.' + coll.getName()); - -const timeFieldName = 'time'; - -coll.drop(); -assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField: timeFieldName}})); -assert.contains(bucketsColl.getName(), db.getCollectionNames()); - -assert.commandFailedWithCode(db.adminCommand({ - renameCollection: bucketsColl.getFullName(), - to: db.getName() + ".otherColl", - dropTarget: false -}), - ErrorCodes.IllegalOperation); -})(); diff --git a/jstests/core/timeseries/timeseries_create_collection.js b/jstests/core/timeseries/timeseries_create_collection.js index feb2f443d26..99616605163 100644 --- a/jstests/core/timeseries/timeseries_create_collection.js +++ b/jstests/core/timeseries/timeseries_create_collection.js @@ -17,6 +17,15 @@ assert.commandWorked(testDB.dropDatabase()); const timeFieldName = 'time'; const coll = testDB.t; +// Fails to create a time-series collection with null-embedded timeField or metaField. +assert.commandFailedWithCode( + testDB.createCollection(coll.getName(), {timeseries: {timeField: '\0time'}}), + ErrorCodes.BadValue); +assert.commandFailedWithCode( + testDB.createCollection(coll.getName(), + {timeseries: {timeField: timeFieldName, metaField: 't\0ag'}}), + ErrorCodes.BadValue); + // Create a timeseries collection, listCollection should show view and bucket collection assert.commandWorked( testDB.createCollection(coll.getName(), {timeseries: {timeField: timeFieldName}})); diff --git a/jstests/core/timeseries/timeseries_delete_hint.js b/jstests/core/timeseries/timeseries_delete_hint.js index 0facca816e1..69fb2bc5a91 100644 --- a/jstests/core/timeseries/timeseries_delete_hint.js +++ b/jstests/core/timeseries/timeseries_delete_hint.js @@ -18,11 +18,11 @@ (function() { "use strict"; -load("jstests/core/timeseries/libs/timeseries.js"); load("jstests/libs/curop_helpers.js"); +load("jstests/libs/feature_flag_util.js"); load('jstests/libs/parallel_shell_helpers.js'); -if (!TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(db.getMongo())) { +if (!FeatureFlagUtil.isEnabled(db, "TimeseriesUpdatesAndDeletes")) { jsTestLog("Skipping test because the time-series updates and deletes feature flag is disabled"); return; } @@ -71,7 +71,7 @@ const validateDeleteIndex = (docsToInsert, : assert.commandWorked( testDB.runCommand({delete: coll.getName(), deletes: deleteQuery})); assert.eq(res["n"], expectedNRemoved); - assert.docEq(coll.find({}, {_id: 0}).toArray(), expectedRemainingDocs); + assert.sameMembers(coll.find({}, {_id: 0}).toArray(), expectedRemainingDocs); assert(coll.drop()); }, docsToInsert, diff --git a/jstests/core/timeseries/timeseries_field_parsed_as_bson.js b/jstests/core/timeseries/timeseries_field_parsed_as_bson.js new file mode 100644 index 00000000000..70210a90072 --- /dev/null +++ b/jstests/core/timeseries/timeseries_field_parsed_as_bson.js @@ -0,0 +1,70 @@ +/** + * Tests that timeseries timeField is parsed as bson. + * + * @tags: [ + * # We need a timeseries collection. + * requires_timeseries, + * # Cannot insert into a time-series collection in a multi-document transaction + * does_not_support_transactions, + * multiversion_incompatible + * ] + */ + +(function() { +'use strict'; + +const collName = "timeseries_field_parsed_as_bson"; +const coll = db.getCollection(collName); + +coll.drop(); +const timeField = "badInput']}}}}}}"; +assert.commandWorked(db.createCollection(collName, {timeseries: {timeField: timeField}})); + +const timeseriesCollInfo = db.getCollectionInfos({name: "system.buckets." + collName})[0]; +jsTestLog("Timeseries system collection info: " + tojson(timeseriesCollInfo)); +const properties = {}; +properties[timeField] = { + "bsonType": "date" +}; +const expectedValidator = { + "$jsonSchema": { + "bsonType": "object", + "required": ["_id", "control", "data"], + "properties": { + "_id": {"bsonType": "objectId"}, + "control": { + "bsonType": "object", + "required": ["version", "min", "max"], + "properties": { + "version": {"bsonType": "number"}, + "min": + {"bsonType": "object", "required": [timeField], "properties": properties}, + "max": + {"bsonType": "object", "required": [timeField], "properties": properties}, + "closed": {"bsonType": "bool"}, + "count": {"bsonType": "number", "minimum": 1} + }, + "additionalProperties": false + }, + "data": {"bsonType": "object"}, + "meta": {} + }, + "additionalProperties": false + } +}; + +assert(timeseriesCollInfo.options); +assert.eq(timeseriesCollInfo.options.validator, expectedValidator); + +const doc = { + a: 1, + [timeField]: new Date("2021-01-01") +}; +assert.commandWorked(coll.insert(doc)); +assert.docEq([doc], coll.aggregate([{$match: {}}, {$project: {_id: 0}}]).toArray()); + +coll.drop(); +assert.commandWorked(db.createCollection(collName, {timeseries: {timeField: "\\"}})); +coll.drop(); +assert.commandWorked(db.createCollection(collName, {timeseries: {timeField: "\\\\"}})); +})(); diff --git a/jstests/core/timeseries/timeseries_filter_extended_range.js b/jstests/core/timeseries/timeseries_filter_extended_range.js new file mode 100644 index 00000000000..2b86e29c943 --- /dev/null +++ b/jstests/core/timeseries/timeseries_filter_extended_range.js @@ -0,0 +1,185 @@ +/** + * Test that find/match type queries work properly on dates ouside the 32 bit epoch range, + * [1970-01-01 00:00:00 UTC - 2038-01-29 03:13:07 UTC]. + * + * @tags: [ + * # Refusing to run a test that issues an aggregation command with explain because it may + * # return incomplete results if interrupted by a stepdown. + * does_not_support_stepdowns, + * # We need a timeseries collection. + * requires_timeseries, + * # Cannot insert into a time-series collection in a multi-document transaction + * does_not_support_transactions, + * # Explain of a resolved view must be executed by mongos. + * directly_against_shardsvrs_incompatible, + * ] + */ + +(function() { +"use strict"; +const timeFieldName = "time"; + +/* + * Creates a collection, populates it, runs the `query` and ensures that the result set + * is equal to `results`. + * + * If overflow is set we create a document with dates above the 32 bit range (year 2040) + * If underflow is set, we create a document with dates below the 32 bit range (year 1965) + */ +function runTest(underflow, overflow, query, results) { + // Setup our DB & our collections. + const tsColl = db.getCollection(jsTestName()); + tsColl.drop(); + + assert.commandWorked( + db.createCollection(tsColl.getName(), {timeseries: {timeField: timeFieldName}})); + + const dates = [ + // If underflow, we want to insert a date that would fall below the epoch + // i.e. 1970-01-01 00:00:00 UTC. Otherwise we use a date within the epoch. + {[timeFieldName]: underflow ? new Date("1965-01-01") : new Date("1971-01-01")}, + {[timeFieldName]: new Date("1975-01-01")}, + {[timeFieldName]: new Date("1980-01-01")}, + {[timeFieldName]: new Date("1995-01-01")}, + // If overflow, we want to insert a date that would use more than 32 bit milliseconds after + // the epoch. This overflow will occur 2038-01-29 03:13:07 UTC. Otherwise we go slightly + // before the end of the 32 bit epoch. + {[timeFieldName]: overflow ? new Date("2040-01-01") : new Date("2030-01-01")} + ]; + assert.commandWorked(tsColl.insert(dates)); + + // Make sure the expected results are in the correct order for comparison below. + function cmpTimeFields(a, b) { + return (b[timeFieldName].getTime() - a[timeFieldName].getTime()); + } + results.sort(cmpTimeFields); + + const pipeline = [{$match: query}, {$project: {_id: 0, [timeFieldName]: 1}}]; + + const plan = tsColl.explain().aggregate(pipeline); + + // Verify agg pipeline. We don't want to go through a plan that encourages a sort order to + // avoid BUS and index selection, so we sort after gathering the results. + const aggActuals = tsColl.aggregate(pipeline).toArray(); + aggActuals.sort(cmpTimeFields); + assert.docEq(results, aggActuals, JSON.stringify(plan, null, 4)); + + // Verify the equivalent find command. We again don't want to go through a plan that + // encourages a sort order to avoid BUS and index selection, so we sort after gathering the + // results. + let findActuals = tsColl.find(query, {_id: 0, [timeFieldName]: 1}).toArray(); + findActuals.sort(cmpTimeFields); + assert.docEq(findActuals, results); +} + +runTest(false, + false, + {[timeFieldName]: {$eq: new Date("1980-01-01")}}, + [{[timeFieldName]: new Date("1980-01-01")}]); +runTest(false, + true, + {[timeFieldName]: {$eq: new Date("2040-01-01")}}, + [{[timeFieldName]: new Date("2040-01-01")}]); +runTest(true, + false, + {[timeFieldName]: {$eq: new Date("1965-01-01")}}, + [{[timeFieldName]: new Date("1965-01-01")}]); + +runTest(false, + false, + {[timeFieldName]: {$lt: new Date("1980-01-01")}}, + [{[timeFieldName]: new Date("1971-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); +runTest(false, + true, + {[timeFieldName]: {$lt: new Date("1980-01-01")}}, + [{[timeFieldName]: new Date("1971-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); +runTest(true, + false, + {[timeFieldName]: {$lt: new Date("1980-01-01")}}, + [{[timeFieldName]: new Date("1965-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); +runTest(true, + true, + {[timeFieldName]: {$lt: new Date("1980-01-01")}}, + [{[timeFieldName]: new Date("1965-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); + +runTest(false, + false, + {[timeFieldName]: {$gt: new Date("1980-01-01")}}, + [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2030-01-01")}]); +runTest(false, + true, + {[timeFieldName]: {$gt: new Date("1980-01-01")}}, + [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2040-01-01")}]); +runTest(true, + false, + {[timeFieldName]: {$gt: new Date("1980-01-01")}}, + [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2030-01-01")}]); +runTest(true, + true, + {[timeFieldName]: {$gt: new Date("1980-01-01")}}, + [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2040-01-01")}]); + +runTest(false, false, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1971-01-01")}, + {[timeFieldName]: new Date("1975-01-01")}, + {[timeFieldName]: new Date("1980-01-01")} +]); +runTest(false, true, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1971-01-01")}, + {[timeFieldName]: new Date("1975-01-01")}, + {[timeFieldName]: new Date("1980-01-01")} +]); +runTest(true, false, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1965-01-01")}, + {[timeFieldName]: new Date("1975-01-01")}, + {[timeFieldName]: new Date("1980-01-01")} +]); +runTest(true, true, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1965-01-01")}, + {[timeFieldName]: new Date("1975-01-01")}, + {[timeFieldName]: new Date("1980-01-01")} +]); + +runTest(false, false, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1980-01-01")}, + {[timeFieldName]: new Date("1995-01-01")}, + {[timeFieldName]: new Date("2030-01-01")} +]); +runTest(false, true, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1980-01-01")}, + {[timeFieldName]: new Date("1995-01-01")}, + {[timeFieldName]: new Date("2040-01-01")} +]); +runTest(true, false, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1980-01-01")}, + {[timeFieldName]: new Date("1995-01-01")}, + {[timeFieldName]: new Date("2030-01-01")} +]); +runTest(true, true, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1980-01-01")}, + {[timeFieldName]: new Date("1995-01-01")}, + {[timeFieldName]: new Date("2040-01-01")} +]); + +// Verify ranges that straddle the lower and upper epoch boundaries work properly. +runTest( + true, false, {[timeFieldName]: {$gt: new Date("1920-01-01"), $lt: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1965-01-01")}, + {[timeFieldName]: new Date("1975-01-01")}, + ]); +runTest( + false, true, {[timeFieldName]: {$gt: new Date("1980-01-01"), $lt: new Date("2050-01-01")}}, [ + {[timeFieldName]: new Date("1995-01-01")}, + {[timeFieldName]: new Date("2040-01-01")}, + ]); +runTest( + false, false, {[timeFieldName]: {$gt: new Date("1920-01-01"), $lt: new Date("1980-01-01")}}, [ + {[timeFieldName]: new Date("1971-01-01")}, + {[timeFieldName]: new Date("1975-01-01")}, + ]); +runTest( + false, false, {[timeFieldName]: {$gt: new Date("1980-01-01"), $lt: new Date("2050-01-01")}}, [ + {[timeFieldName]: new Date("1995-01-01")}, + {[timeFieldName]: new Date("2030-01-01")}, + ]); +})(); diff --git a/jstests/core/timeseries/timeseries_geonear_measurements.js b/jstests/core/timeseries/timeseries_geonear_measurements.js index 5c5427a02d1..abbf7db9596 100644 --- a/jstests/core/timeseries/timeseries_geonear_measurements.js +++ b/jstests/core/timeseries/timeseries_geonear_measurements.js @@ -23,10 +23,10 @@ (function() { "use strict"; -load("jstests/core/timeseries/libs/timeseries.js"); load("jstests/libs/analyze_plan.js"); +load("jstests/libs/feature_flag_util.js"); -if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { +if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; diff --git a/jstests/core/timeseries/timeseries_groupby_reorder.js b/jstests/core/timeseries/timeseries_groupby_reorder.js new file mode 100644 index 00000000000..ade24a59470 --- /dev/null +++ b/jstests/core/timeseries/timeseries_groupby_reorder.js @@ -0,0 +1,217 @@ +/** + * Test the behavior of $group on time-series collections. Specifically, we are targeting rewrites + * that replace bucket unpacking with $group over the buckets collection. Currently, only $min/$max + * are supported for the rewrites. + * + * @tags: [ + * directly_against_shardsvrs_incompatible, + * does_not_support_stepdowns, + * does_not_support_transactions, + * requires_fcv_61, + * ] + */ +(function() { +"use strict"; + +load("jstests/libs/fixture_helpers.js"); +load("jstests/core/timeseries/libs/timeseries.js"); + +const coll = db.timeseries_groupby_reorder; +coll.drop(); + +// We will only check correctness of the results here as checking the plan in JSTests is brittle and +// is better done in document_source_internal_unpack_bucket_test/group_reorder_test.cpp. For the +// cases when the re-write isn't applicable, the used datasets should yield wrong result if the +// re-write is applied. +function runGroupRewriteTest(docs, pipeline, expectedResults) { + coll.drop(); + db.createCollection(coll.getName(), {timeseries: {metaField: "meta", timeField: "time"}}); + coll.insertMany(docs); + assert.docEq(expectedResults, coll.aggregate(pipeline).toArray(), () => { + return `Pipeline: ${tojson(pipeline)}. Explain: ${ + tojson(coll.explain().aggregate(pipeline))}`; + }); +} + +// Test with measurement group key -- a rewrite in this situation would be wrong. +(function testNonMetaGroupKey() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, key: 2, val: 1}, // global min + {time: t, meta: 1, key: 1, val: 3}, // min for key = 1 + {time: t, meta: 1, key: 1, val: 5}, // max for key = 1 + {time: t, meta: 1, key: 2, val: 7}, // global max + ]; + runGroupRewriteTest(docs, + [{$group: {_id: "$key", min: {$min: "$val"}}}, {$match: {_id: 1}}], + [{"_id": 1, "min": 3}]); + runGroupRewriteTest(docs, + [{$group: {_id: "$key", max: {$max: "$val"}}}, {$match: {_id: 1}}], + [{"_id": 1, "max": 5}]); +})(); + +// While a group with const group key can be re-written in terms of a group on the buckets, we don't +// currently do it. However, when/if we start doing it, it should work. +(function testConstGroupKey_NoFilter() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, val: 0}, + {time: t, meta: 1, val: 1}, + {time: t, meta: 1, val: 5}, + ]; + runGroupRewriteTest( + docs, [{$group: {_id: null, min: {$min: "$val"}}}], [{"_id": null, "min": 0}]); + runGroupRewriteTest( + docs, [{$group: {_id: null, max: {$max: "$val"}}}], [{"_id": null, "max": 5}]); +})(); + +// With a filter on meta the group re-write would still apply if the group key is const. +(function testConstGroupKey_WithFilterOnMeta() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, val: 0}, + {time: t, meta: 2, val: 1}, + {time: t, meta: 2, val: 3}, + {time: t, meta: 1, val: 5}, + ]; + runGroupRewriteTest(docs, + [{$match: {meta: 2}}, {$group: {_id: null, min: {$min: "$val"}}}], + [{"_id": null, "min": 1}]); + runGroupRewriteTest(docs, + [{$match: {meta: 2}}, {$group: {_id: null, max: {$max: "$val"}}}], + [{"_id": null, "max": 3}]); +})(); + +// In presense of a non-meta filter the group re-write doesn't apply even if the group key is const. +(function testConstGroupKey_WithFilterOnMeasurement() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, val: 0, include: false}, + {time: t, meta: 1, val: 1, include: true}, + {time: t, meta: 1, val: 5, include: false}, + ]; + runGroupRewriteTest(docs, + [{$match: {include: true}}, {$group: {_id: null, min: {$min: "$val"}}}], + [{"_id": null, "min": 1}]); + runGroupRewriteTest(docs, + [{$match: {include: true}}, {$group: {_id: null, max: {$max: "$val"}}}], + [{"_id": null, "max": 1}]); +})(); + +// Test with meta group key. The group re-write applies. +(function testMetaGroupKey_NoFilter() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, val: 5}, + {time: t, meta: 2, val: 4}, + {time: t, meta: 2, val: 3}, + {time: t, meta: 1, val: 1}, + ]; + runGroupRewriteTest(docs, + [{$group: {_id: "$meta", min: {$min: "$val"}}}, {$match: {_id: 2}}], + [{"_id": 2, "min": 3}]); + runGroupRewriteTest(docs, + [{$group: {_id: "$meta", max: {$max: "$val"}}}, {$match: {_id: 2}}], + [{"_id": 2, "max": 4}]); +})(); + +// Test with meta group key preceeded by a filter on the meta key. The re-write still applies. +(function testMetaGroupKey_WithFilterOnMeta() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, val: 5}, + {time: t, meta: 2, val: 4}, + {time: t, meta: 2, val: 3}, + {time: t, meta: 1, val: 1}, + ]; + runGroupRewriteTest(docs, + [{$match: {meta: 2}}, {$group: {_id: "$meta", min: {$min: "$val"}}}], + [{"_id": 2, "min": 3}]); + runGroupRewriteTest(docs, + [{$match: {meta: 2}}, {$group: {_id: "$meta", max: {$max: "$val"}}}], + [{"_id": 2, "max": 4}]); +})(); + +// Test with meta group key preceeded by a filter on a measurement key. The re-write doesn't apply. +(function testMetaGroupKey_WithFilterOnMeasurement() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, val: 3, include: false}, + {time: t, meta: 1, val: 4, include: true}, + {time: t, meta: 1, val: 5, include: false}, + ]; + runGroupRewriteTest(docs, + [{$match: {include: true}}, {$group: {_id: "$meta", min: {$min: "$val"}}}], + [{"_id": 1, "min": 4}]); + runGroupRewriteTest(docs, + [{$match: {include: true}}, {$group: {_id: "$meta", max: {$max: "$val"}}}], + [{"_id": 1, "max": 4}]); +})(); + +// Test SERVER-73822 fix: complex $min and $max (i.e. not just straight field refs) work correctly. +(function testMetaGroupKey_WithNonPathExpressionUnderMinMax() { + const t = new Date(); + // min(a+b) != min(a) + min(b) and max(a+b) != max(a) + max(b) + const docs = [ + {time: t, meta: 1, a: 1, b: 20}, // max(a + b) + {time: t, meta: 1, a: 2, b: 10}, + {time: t, meta: 1, a: 3, b: 1}, // min(a + b) + ]; + runGroupRewriteTest(docs, + [{$group: {_id: "$meta", min: {$min: {$add: ["$a", "$b"]}}}}], + [{"_id": 1, "min": 4}]); + runGroupRewriteTest(docs, + [{$group: {_id: "$meta", max: {$max: {$add: ["$a", "$b"]}}}}], + [{"_id": 1, "max": 21}]); +})(); + +// Test with meta group key and a non-min/max accumulator that doesn't use any fields. The buckets +// still have to be unpacked because we don't know the number of events in uncompressed buckets. +(function testMetaGroupKey_WithAccumulatorNotUsingAnyFields() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, val: 3}, + {time: t, meta: 3, val: 4}, + {time: t, meta: 1, val: 5}, + ]; + runGroupRewriteTest( + docs, [{$group: {_id: "$meta", x: {$sum: 1}}}, {$match: {_id: 1}}], [{"_id": 1, "x": 2}]); +})(); + +// Test with meta group key and a non-min/max accumulator that uses only the meta field. This query +// is _not_ eligible for the re-write w/o bucket unpacking because while it doesn't depend on any +// fields of the individual events it still depends on the number of events in a bucket. +(function testMetaGroupKey_WithNonMinMaxAccumulatorOnMeta() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, val: 3}, + {time: t, meta: 3, val: 4}, + {time: t, meta: 1, val: 5}, + ]; + runGroupRewriteTest(docs, + [{$group: {_id: "$meta", x: {$sum: "$meta"}}}, {$match: {_id: 1}}], + [{"_id": 1, "x": 2}]); +})(); + +// In presence of a filter $min and $max on the meta field cannot be re-written because a filter +// might end up selecting nothing in buckets with a particular meta. +(function testMetaGroupKey_WithAccumulatorOnMeta_WithFilterOnMeasurement() { + const t = new Date(); + const docs = [ + {time: t, meta: 1, include: false}, + ]; + runGroupRewriteTest( + docs, [{$match: {include: true}}, {$group: {_id: "$meta", x: {$min: "$meta"}}}], []); +})(); + +// Test min/max on the time field (cannot rewrite $min because the control.time.min is rounded +// down). +(function testMetaGroupKey_WithMinMaxOnTime() { + const docs = [ + {time: ISODate("2023-07-20T23:16:47.683Z"), meta: 1}, + ]; + runGroupRewriteTest(docs, + [{$group: {_id: "$meta", min: {$min: "$time"}}}], + [{_id: 1, min: ISODate("2023-07-20T23:16:47.683Z")}]); +})(); +})(); diff --git a/jstests/core/timeseries/timeseries_index.js b/jstests/core/timeseries/timeseries_index.js index bef86453999..9547362d105 100644 --- a/jstests/core/timeseries/timeseries_index.js +++ b/jstests/core/timeseries/timeseries_index.js @@ -11,8 +11,9 @@ (function() { "use strict"; -load("jstests/libs/fixture_helpers.js"); load("jstests/core/timeseries/libs/timeseries.js"); +load("jstests/libs/feature_flag_util.js"); +load("jstests/libs/fixture_helpers.js"); TimeseriesTest.run((insert) => { const collNamePrefix = 'timeseries_index_'; @@ -225,7 +226,7 @@ TimeseriesTest.run((insert) => { runTest({[metaFieldName + '.location']: "2d", [metaFieldName + '.tag1']: -1}, {'meta.location': "2d", 'meta.tag1': -1}); - if (TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { + if (FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { // Measurement 2dsphere index runTest({'loc': '2dsphere'}, {'data.loc': '2dsphere_bucket'}); } @@ -242,7 +243,7 @@ TimeseriesTest.run((insert) => { coll.getName(), {timeseries: {timeField: timeFieldName, metaField: metaFieldName}})); assert.commandWorked(insert(coll, doc), 'failed to insert doc: ' + tojson(doc)); - if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { + if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { // Reject index keys that do not include the metadata field. assert.commandFailedWithCode(coll.createIndex({not_metadata: 1}), ErrorCodes.CannotCreateIndex); @@ -265,7 +266,7 @@ TimeseriesTest.run((insert) => { [ErrorCodes.CannotCreateIndex, ErrorCodes.InvalidOptions]); }; - if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { + if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { // Partial indexes are not supported on time-series collections if the time-series metric // feature flag is disabled. testCreateIndexFailed({[metaFieldName]: 1}, {partialFilterExpression: {meta: {$gt: 5}}}); diff --git a/jstests/core/timeseries/timeseries_index_partial.js b/jstests/core/timeseries/timeseries_index_partial.js index eb2512b1bd7..014158c3542 100644 --- a/jstests/core/timeseries/timeseries_index_partial.js +++ b/jstests/core/timeseries/timeseries_index_partial.js @@ -2,6 +2,8 @@ * Test creating and using partial indexes, on a time-series collection. * * @tags: [ + * # TODO (SERVER-73316): remove + * assumes_against_mongod_not_mongos, * does_not_support_stepdowns, * does_not_support_transactions, * requires_fcv_52, @@ -12,10 +14,10 @@ (function() { "use strict"; -load("jstests/core/timeseries/libs/timeseries.js"); load("jstests/libs/analyze_plan.js"); +load("jstests/libs/feature_flag_util.js"); -if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { +if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; @@ -78,10 +80,24 @@ assert.commandFailedWithCode(coll.createIndex({a: 1}, {partialFilterExpression: // Test creating and using a partial index. { - // Make sure the query uses the {a: 1} index. + let ixscanInWinningPlan = 0; + + // Make sure the {a: 1} index was considered for this query. function checkPlan(predicate) { const explain = coll.find(predicate).explain(); - const scan = getAggPlanStage(explain, 'IXSCAN'); + let scan = getAggPlanStage(explain, 'IXSCAN'); + // If scan is not present, check rejected plans + if (scan === null) { + const rejectedPlans = getRejectedPlans(getAggPlanStage(explain, "$cursor")["$cursor"]); + if (rejectedPlans.length === 1) { + const scans = getPlanStages(getRejectedPlan(rejectedPlans[0]), "IXSCAN"); + if (scans.length === 1) { + scan = scans[0]; + } + } + } else { + ixscanInWinningPlan++; + } const indexes = buckets.getIndexes(); assert(scan, "Expected an index scan for predicate: " + tojson(predicate) + @@ -90,10 +106,10 @@ assert.commandFailedWithCode(coll.createIndex({a: 1}, {partialFilterExpression: } // Make sure the query results match a collection-scan plan. function checkResults(predicate) { - const result = coll.aggregate({$match: predicate}).toArray(); + const result = coll.aggregate([{$match: predicate}], {hint: {a: 1}}).toArray(); const unindexed = coll.aggregate([{$_internalInhibitOptimization: {}}, {$match: predicate}]).toArray(); - assert.docEq(result, unindexed); + assert.sameMembers(result, unindexed); } function checkPlanAndResults(predicate) { checkPlan(predicate); @@ -133,10 +149,6 @@ assert.commandFailedWithCode(coll.createIndex({a: 1}, {partialFilterExpression: const t1 = ISODate('2000-01-01T00:00:01Z'); const t2 = ISODate('2000-01-01T00:00:02Z'); - // When the collection is sharded, there is an index on time that can win, instead of the - // partial index. So only check the results in that case, not the plan. - const check = FixtureHelpers.isSharded(buckets) ? checkResults : checkPlanAndResults; - assert.commandWorked(coll.dropIndex({a: 1})); assert.commandWorked( coll.createIndex({a: 1}, {partialFilterExpression: {[timeField]: {$lt: t1}}})); @@ -165,6 +177,7 @@ assert.commandFailedWithCode(coll.createIndex({a: 1}, {partialFilterExpression: assert.commandWorked(coll.dropIndex({a: 1})); assert.sameMembers(coll.getIndexes(), extraIndexes); assert.sameMembers(buckets.getIndexes(), extraBucketIndexes); + assert.gt(ixscanInWinningPlan, 0); } // Check that partialFilterExpression can use a mixture of metadata, time, and measurement fields, diff --git a/jstests/core/timeseries/timeseries_index_spec.js b/jstests/core/timeseries/timeseries_index_spec.js index f13c64dd80b..d3822dc642b 100644 --- a/jstests/core/timeseries/timeseries_index_spec.js +++ b/jstests/core/timeseries/timeseries_index_spec.js @@ -16,6 +16,7 @@ "use strict"; load("jstests/core/timeseries/libs/timeseries.js"); +load("jstests/libs/feature_flag_util.js"); TimeseriesTest.run(() => { const collName = "timeseries_index_spec"; @@ -85,7 +86,7 @@ TimeseriesTest.run(() => { {name: "time_meta_field_downgradable"})); verifyAndDropIndex(/*isDowngradeCompatible=*/true, "time_meta_field_downgradable"); - if (TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { + if (FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { assert.commandWorked(coll.createIndex({x: 1}, {name: "x_1"})); verifyAndDropIndex(/*isDowngradeCompatible=*/false, "x_1"); @@ -110,7 +111,7 @@ TimeseriesTest.run(() => { // Creating an index directly on the buckets collection is permitted. However, these types of // index creations will not have an "originalSpec" field and rely on the reverse mapping // mechanism. - if (TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { + if (FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { assert.commandWorked( bucketsColl.createIndex({"control.min.y": 1, "control.max.y": 1}, {name: "y"})); diff --git a/jstests/core/timeseries/timeseries_index_use.js b/jstests/core/timeseries/timeseries_index_use.js index 8770047f4d2..6ce128ecbbd 100644 --- a/jstests/core/timeseries/timeseries_index_use.js +++ b/jstests/core/timeseries/timeseries_index_use.js @@ -50,8 +50,9 @@ const generateTest = (useHint) => { /** * Creates the index specified by the spec and options, then explains the query to ensure - * that the created index is used. Runs the query and verifies that the expected number of - * documents are matched. Finally, deletes the created index. + * that the created index is used or was considered by multi-planner. + * Runs the query and verifies that the expected number of documents are matched. + * Finally, deletes the created index. */ const testQueryUsesIndex = function( filter, numMatches, indexSpec, indexOpts = {}, queryOpts = {}) { @@ -67,9 +68,23 @@ const generateTest = (useHint) => { assert.eq(numMatches, query.itcount()); const explain = query.explain(); - const ixscan = getAggPlanStage(explain, "IXSCAN"); - assert.neq(null, ixscan, tojson(explain)); - assert.eq("testIndexName", ixscan.indexName, tojson(ixscan)); + if (useHint) { + const ixscan = getAggPlanStage(explain, "IXSCAN"); + assert.neq(null, ixscan, tojson(explain)); + assert.eq("testIndexName", ixscan.indexName, tojson(ixscan)); + } else { + let ixscan = getAggPlanStage(explain, "IXSCAN"); + // If ixscan is not present, check rejected plans + if (ixscan === null) { + const rejectedPlans = + getRejectedPlans(getAggPlanStage(explain, "$cursor")["$cursor"]); + assert.eq(1, rejectedPlans.length); + const ixscans = getPlanStages(getRejectedPlan(rejectedPlans[0]), "IXSCAN"); + assert.eq(1, ixscans.length); + ixscan = ixscans[0]; + } + assert.eq("testIndexName", ixscan.indexName, tojson(ixscan)); + } assert.commandWorked(coll.dropIndex("testIndexName")); }; diff --git a/jstests/core/timeseries/timeseries_internal_bucket_geo_within.js b/jstests/core/timeseries/timeseries_internal_bucket_geo_within.js index 9c237d8dd77..2e350ebdbb0 100644 --- a/jstests/core/timeseries/timeseries_internal_bucket_geo_within.js +++ b/jstests/core/timeseries/timeseries_internal_bucket_geo_within.js @@ -4,7 +4,7 @@ * collection. * * @tags: [ - * requires_fcv_51, + * requires_fcv_60, * requires_pipeline_optimization, * requires_timeseries, * does_not_support_stepdowns, @@ -47,10 +47,6 @@ for (let collScanStage of collScanStages) { "field": "loc" } }; - // TODO SERVER-60373 Fix duplicate predicates for sharded time-series collection - if (FixtureHelpers.isSharded(bucketsColl)) { - expectedPredicate = {$and: [expectedPredicate, expectedPredicate]}; - } assert.docEq(expectedPredicate, collScanStage.filter, collScanStages); } diff --git a/jstests/core/timeseries/timeseries_lastpoint_top.js b/jstests/core/timeseries/timeseries_lastpoint_top.js index 2efd352b574..c85f48fb076 100644 --- a/jstests/core/timeseries/timeseries_lastpoint_top.js +++ b/jstests/core/timeseries/timeseries_lastpoint_top.js @@ -27,6 +27,7 @@ assert.commandWorked(testDB.dropDatabase()); // Do not run the rest of the tests if the lastpoint optimization is disabled. if (!FeatureFlagUtil.isEnabled(db, "LastPointQuery")) { + jsTestLog("Skipping the test."); return; } diff --git a/jstests/core/timeseries/timeseries_lookup.js b/jstests/core/timeseries/timeseries_lookup.js index c173764a591..ba804c4ec92 100644 --- a/jstests/core/timeseries/timeseries_lookup.js +++ b/jstests/core/timeseries/timeseries_lookup.js @@ -24,43 +24,44 @@ TimeseriesTest.run((insert) => { Random.setRandomSeed(); const hosts = TimeseriesTest.generateHosts(numHosts); - let testFunc = function(collAOption, collBOption) { - // Prepare two time-series collections. - const collA = testDB.getCollection("a"); - const collB = testDB.getCollection("b"); - collA.drop(); - collB.drop(); - assert.commandWorked(testDB.createCollection(collA.getName(), collAOption)); - assert.commandWorked(testDB.createCollection(collB.getName(), collBOption)); + let + testFunc = function(collAOption, collBOption) { + // Prepare two time-series collections. + const collA = testDB.getCollection("a"); + const collB = testDB.getCollection("b"); + collA.drop(); + collB.drop(); + assert.commandWorked(testDB.createCollection(collA.getName(), collAOption)); + assert.commandWorked(testDB.createCollection(collB.getName(), collBOption)); - let entryCountPerHost = new Array(numHosts).fill(0); + let entryCountPerHost = new Array(numHosts).fill(0); - // Insert into collA, one entry per host. - for (let i = 0; i < numHosts; i++) { - let host = hosts[i]; - assert.commandWorked(insert(collA, { - measurement: "cpu", - time: ISODate(), - tags: host.tags, - })); - } + // Insert into collA, one entry per host. + for (let i = 0; i < numHosts; i++) { + let host = hosts[i]; + assert.commandWorked(insert(collA, { + measurement: "cpu", + time: ISODate(), + tags: host.tags, + })); + } - // Insert some random documents to collB. - for (let i = 0; i < numDocs; i++) { - let host = TimeseriesTest.getRandomElem(hosts); - assert.commandWorked(insert(collB, { - measurement: "cpu", - time: ISODate(), - tags: host.tags, - })); - // Here we extract the hostId from "host.tags.hostname". It is expected that the - // "host.tags.hostname" is in the form of 'host_<hostNum>'. - entryCountPerHost[parseInt( - host.tags.hostname.substring(5, host.tags.hostname.length))]++; - } + // Insert some random documents to collB. + for (let i = 0; i < numDocs; i++) { + let host = TimeseriesTest.getRandomElem(hosts); + assert.commandWorked(insert(collB, { + measurement: "cpu", + time: ISODate(), + tags: host.tags, + })); + // Here we extract the hostId from "host.tags.hostname". It is expected that the + // "host.tags.hostname" is in the form of 'host_<hostNum>'. + entryCountPerHost[parseInt( + host.tags.hostname.substring(5, host.tags.hostname.length))]++; + } - // Equality Match - let results = collA.aggregate([ + // Equality Match + let results = collA.aggregate([ { $lookup: { from: collB.getName(), @@ -79,13 +80,106 @@ TimeseriesTest.run((insert) => { }, {$sort: {host: 1}} ]).toArray(); - assert.eq(numHosts, results.length, results); - for (let i = 0; i < numHosts; i++) { - assert.eq({host: "host_" + i, matchedB: entryCountPerHost[i]}, results[i], results); - } + assert.eq(numHosts, results.length, results); + for (let i = 0; i < numHosts; i++) { + assert.eq({host: "host_" + i, matchedB: entryCountPerHost[i]}, results[i], results); + } - // Unequal joins - results = collA.aggregate([ + // Equality Match With Let (uncorrelated) + // Make sure injected $sequentialDocumentCache (right after unpack bucket stage) + // in the inner pipeline is removed. + results = collA.aggregate([ + { + $lookup: { + from: collB.getName(), + let: {"outer_hostname": "$tags.hostname"}, + pipeline: [ + // $match will be pushed before unpack bucket stage + {$match: {$expr: {$eq: ["$$outer_hostname", hosts[0].tags.hostname]}}}, + ], + as: "matchedB" + } + }, { + $project: { + _id: 0, + host: "$tags.hostname", + matchedB: { + $size: "$matchedB" + } + } + }, + {$sort: {host: 1}} + ]).toArray(); + assert.eq(numHosts, results.length, results); + for (let i = 0; i < numHosts; i++) { + const matched = i === 0 ? numDocs : 0; + assert.eq({host: "host_" + i, matchedB: matched}, results[i], results); + } + + // Equality Match With Let (uncorrelated) + // Make sure injected $sequentialDocumentCache in the inner pipeline is removed. + // $sequentialDocumentCache is not located right after unpack bucket stage. + results = collA.aggregate([ + { + $lookup: { + from: collB.getName(), + let: {"outer_hostname": "$tags.hostname"}, + pipeline: [ + {$match: {$expr: {$eq: ["$$outer_hostname", hosts[0].tags.hostname]}}}, + {$set: {foo: {$const: 123}}}, // uncorrelated + ], + as: "matchedB" + } + }, { + $project: { + _id: 0, + host: "$tags.hostname", + matchedB: { + $size: "$matchedB" + } + } + }, + {$sort: {host: 1}} + ]).toArray(); + assert.eq(numHosts, results.length, results); + for (let i = 0; i < numHosts; i++) { + const matched = i === 0 ? numDocs : 0; + assert.eq({host: "host_" + i, matchedB: matched}, results[i], results); + } + + // Equality Match With Let (correlated, no $match re-order) + // Make sure injected $sequentialDocumentCache in the inner pipeline is removed. + // $sequentialDocumentCache is located at the very end of pipeline. + results = collA.aggregate([ + { + $lookup: { + from: collB.getName(), + let: {"outer_hostname": "$tags.hostname"}, + pipeline: [ + {$match: {$expr: {$eq: ["$$outer_hostname", hosts[0].tags.hostname]}}}, + {$set: {foo: "$$outer_hostname"}}, // correlated + ], + as: "matchedB" + } + }, { + $project: { + _id: 0, + host: "$tags.hostname", + matchedB: { + $size: "$matchedB" + } + } + }, + {$sort: {host: 1}} + ]).toArray(); + assert.eq(numHosts, results.length, results); + for (let i = 0; i < numHosts; i++) { + const matched = i === 0 ? numDocs : 0; + assert.eq({host: "host_" + i, matchedB: matched}, results[i], results); + } + + // Unequal joins + results = collA.aggregate([ { $lookup: { from: collB.getName(), @@ -114,13 +208,28 @@ TimeseriesTest.run((insert) => { }, {$sort: {host: 1}} ]).toArray(); - assert.eq(numHosts, results.length, results); - let expectedCount = 0; - for (let i = 0; i < numHosts; i++) { - expectedCount += entryCountPerHost[i]; - assert.eq({host: "host_" + i, matchedB: expectedCount}, results[i], entryCountPerHost); - } - }; + assert.eq(numHosts, results.length, results); + let expectedCount = 0; + for (let i = 0; i < numHosts; i++) { + expectedCount += entryCountPerHost[i]; + assert.eq( + {host: "host_" + i, matchedB: expectedCount}, results[i], entryCountPerHost); + } + + // $sequenceDocumentsCache might optimize out $internalUnpackBucket and cause a + // crash on such query. + results = collA.aggregate([ + {$lookup: { + from: collB.getName(), + as: 'docs', + let: {}, + pipeline: [ + {$sort: {_id: 1}} + ] + }} + ]).toArray(); + assert.eq(numHosts, results.length, results); + }; // Exhaust the combinations of non-time-series and time-series collections for inner and outer // $lookup collections. diff --git a/jstests/core/timeseries/timeseries_match_pushdown.js b/jstests/core/timeseries/timeseries_match_pushdown.js new file mode 100644 index 00000000000..e00dcd4341b --- /dev/null +++ b/jstests/core/timeseries/timeseries_match_pushdown.js @@ -0,0 +1,414 @@ +/** + * Tests that the $match stage followed by unpacking stage has been pushed down with correct + * predicates. + * + * @tags: [ + * requires_timeseries, + * requires_fcv_60, + * does_not_support_stepdowns, + * does_not_support_transactions, + * directly_against_shardsvrs_incompatible, + * ] + */ +(function() { +"use strict"; + +load("jstests/libs/analyze_plan.js"); // For getAggPlanStages + +const coll = db.timeseries_match_pushdown; +coll.drop(); + +const timeField = 'time'; +const metaField = 'meta'; +const measureField = 'a'; +assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField, metaField}})); + +// Insert documents into the collection. The bucketing is designed so that some buckets match the +// query entirely, some buckets match the query partially, and some with no matches. +assert.commandWorked(coll.insert([ + {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, + {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:09'), [measureField]: 9, [metaField]: 4}, +])); +const aTime = ISODate('2022-01-01T00:00:03'); +const bTime = ISODate('2022-01-01T00:00:07'); +const bMeta = 3; +const aMeasure = 3; + +/** + * Runs a $match query with the specified 'eventFilter' or a 'pipeline'. + * Assert the 'wholeBucketFilter' is attached correctly to the unpacking stage, and has the expected + * result 'expectedDocs'. + */ +const runTest = function({pipeline, eventFilter, wholeBucketFilter, expectedDocs}) { + if (!pipeline) { + pipeline = [{$match: eventFilter}]; + } + const explain = assert.commandWorked(coll.explain().aggregate(pipeline)); + const unpackStages = getAggPlanStages(explain, '$_internalUnpackBucket'); + assert.eq(1, + unpackStages.length, + "Should only have a single $_internalUnpackBucket stage: " + tojson(explain)); + const unpackStage = unpackStages[0].$_internalUnpackBucket; + assert.docEq(unpackStage.eventFilter, eventFilter, "Incorrect eventFilter: " + tojson(explain)); + if (wholeBucketFilter) { + assert.docEq(unpackStage.wholeBucketFilter, + wholeBucketFilter, + "Incorrect wholeBucketFilter: " + tojson(explain)); + } else { + assert(!unpackStage.wholeBucketFilter, "Incorrect wholeBucketFilter: " + tojson(explain)); + } + + const docs = coll.aggregate([...pipeline, {$sort: {time: 1}}]).toArray(); + assert.eq(docs.length, expectedDocs.length, "Incorrect docs: " + tojson(docs)); + docs.forEach((doc, i) => { + // Do not need to check document _id, since checking time is already unique. + delete doc._id; + assert.docEq(doc, expectedDocs[i], "Incorrect docs: " + tojson(docs)); + }); +}; + +const minTimeField = `control.min.${timeField}`; +const maxTimeField = `control.max.${timeField}`; + +// $gt on time +runTest({ + eventFilter: {[timeField]: {$gt: aTime}}, + wholeBucketFilter: {[minTimeField]: {$gt: aTime}}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:09'), [measureField]: 9, [metaField]: 4}, + ], +}); + +// $gt on measurement +runTest({ + eventFilter: {[measureField]: {$gt: aMeasure}}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:09'), [measureField]: 9, [metaField]: 4}, + ], +}); + +// $gt in $expr on time +runTest({ + pipeline: [{$match: {$expr: {$gt: [`$${timeField}`, {$const: aTime}]}}}], + eventFilter: { + $and: [ + {[timeField]: {$_internalExprGt: aTime}}, + {$expr: {$gt: [`$${timeField}`, {$const: aTime}]}}, + ] + }, + wholeBucketFilter: { + $and: [ + {[minTimeField]: {$_internalExprGt: aTime}}, + {[minTimeField]: {$_internalExprGt: aTime}}, + ] + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:09'), [measureField]: 9, [metaField]: 4}, + ], +}); + +// $gte on time +runTest({ + eventFilter: {[timeField]: {$gte: aTime}}, + wholeBucketFilter: {[minTimeField]: {$gte: aTime}}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:09'), [measureField]: 9, [metaField]: 4}, + ], +}); + +// $gte on measurement +runTest({ + eventFilter: {[measureField]: {$gte: aMeasure}}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:09'), [measureField]: 9, [metaField]: 4}, + ], +}); + +// $gte in $expr on time +runTest({ + pipeline: [{$match: {$expr: {$gte: [`$${timeField}`, {$const: aTime}]}}}], + eventFilter: { + $and: [ + {[timeField]: {$_internalExprGte: aTime}}, + {$expr: {$gte: [`$${timeField}`, {$const: aTime}]}}, + ] + }, + wholeBucketFilter: { + $and: [ + {[minTimeField]: {$_internalExprGte: aTime}}, + {[minTimeField]: {$_internalExprGte: aTime}}, + ] + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:09'), [measureField]: 9, [metaField]: 4}, + ], +}); + +// $lt on time +runTest({ + eventFilter: {[timeField]: {$lt: aTime}}, + wholeBucketFilter: {[maxTimeField]: {$lt: aTime}}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, + {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, + ], +}); + +// $lt on measurement +runTest({ + eventFilter: {[measureField]: {$lt: aMeasure}}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, + {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, + ], +}); + +// $lt in $expr on time +runTest({ + pipeline: [{$match: {$expr: {$lt: [`$${timeField}`, {$const: aTime}]}}}], + eventFilter: { + $and: [ + {[timeField]: {$_internalExprLt: aTime}}, + {$expr: {$lt: [`$${timeField}`, {$const: aTime}]}}, + ] + }, + wholeBucketFilter: { + $and: [ + {[maxTimeField]: {$_internalExprLt: aTime}}, + {[maxTimeField]: {$_internalExprLt: aTime}}, + ] + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, + {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, + ], +}); + +// $lte on time +runTest({ + eventFilter: {[timeField]: {$lte: aTime}}, + wholeBucketFilter: {[maxTimeField]: {$lte: aTime}}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, + {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + ], +}); + +// $lte in $expr on time +runTest({ + pipeline: [{$match: {$expr: {$lte: [`$${timeField}`, {$const: aTime}]}}}], + eventFilter: { + $and: [ + {[timeField]: {$_internalExprLte: aTime}}, + {$expr: {$lte: [`$${timeField}`, {$const: aTime}]}}, + ] + }, + wholeBucketFilter: { + $and: [ + {[maxTimeField]: {$_internalExprLte: aTime}}, + {[maxTimeField]: {$_internalExprLte: aTime}}, + ] + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, + {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + ], +}); + +// $lte on measurement +runTest({ + eventFilter: {[measureField]: {$lte: aMeasure}}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, + {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + ], +}); + +// $eq on time +runTest({ + eventFilter: {[timeField]: {$eq: aTime}}, + wholeBucketFilter: {$and: [{[minTimeField]: {$eq: aTime}}, {[maxTimeField]: {$eq: aTime}}]}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + ], +}); + +// $eq in $expr on time +runTest({ + pipeline: [{$match: {$expr: {$eq: [`$${timeField}`, {$const: aTime}]}}}], + eventFilter: { + $and: [ + {[timeField]: {$_internalExprEq: aTime}}, + {$expr: {$eq: [`$${timeField}`, {$const: aTime}]}}, + ] + }, + wholeBucketFilter: { + $and: [ + {[minTimeField]: {$_internalExprEq: aTime}}, + {[maxTimeField]: {$_internalExprEq: aTime}}, + {[minTimeField]: {$_internalExprEq: aTime}}, + {[maxTimeField]: {$_internalExprEq: aTime}}, + ] + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + ], +}); + +// $eq on measurement +runTest({ + eventFilter: {[measureField]: {$eq: aMeasure}}, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + ], +}); + +// $and on time +runTest({ + eventFilter: {$and: [{[timeField]: {$gt: aTime}}, {[timeField]: {$lt: bTime}}]}, + wholeBucketFilter: { + $and: [ + {[minTimeField]: {$gt: aTime}}, + {[maxTimeField]: {$lt: bTime}}, + ] + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + ], +}); + +// $or on time +runTest({ + eventFilter: {$or: [{[timeField]: {$lte: aTime}}, {[timeField]: {$gte: bTime}}]}, + wholeBucketFilter: { + $or: [ + {[maxTimeField]: {$lte: aTime}}, + {[minTimeField]: {$gte: bTime}}, + ] + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, + {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:09'), [measureField]: 9, [metaField]: 4}, + ], +}); + +// $match on time and meta +runTest({ + pipeline: [{$match: {$and: [{[timeField]: {$gt: aTime}}, {[metaField]: {$lte: bMeta}}]}}], + eventFilter: {[timeField]: {$gt: aTime}}, + wholeBucketFilter: { + [minTimeField]: {$gt: aTime}, + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, + {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, + ], +}); + +// $match on time or meta +runTest({ + eventFilter: {$or: [{[timeField]: {$lte: aTime}}, {[metaField]: {$gt: bMeta}}]}, + wholeBucketFilter: { + $or: [ + {[maxTimeField]: {$lte: aTime}}, + {[metaField]: {$gt: bMeta}}, + ] + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, + {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:09'), [measureField]: 9, [metaField]: 4}, + ], +}); + +// double $match +runTest({ + pipeline: [{$match: {[timeField]: {$gt: aTime}}}, {$match: {[timeField]: {$lt: bTime}}}], + eventFilter: {$and: [{[timeField]: {$gt: aTime}}, {[timeField]: {$lt: bTime}}]}, + wholeBucketFilter: { + $and: [ + {[minTimeField]: {$gt: aTime}}, + {[maxTimeField]: {$lt: bTime}}, + ] + }, + expectedDocs: [ + {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, + {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, + {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, + ], +}); + +// triple $match +runTest({ + pipeline: [ + {$match: {[timeField]: {$gt: aTime}}}, + {$match: {[timeField]: {$lt: bTime}}}, + {$match: {[timeField]: {$lt: aTime}}}, + ], + eventFilter: { + $and: + [{[timeField]: {$gt: aTime}}, {[timeField]: {$lt: bTime}}, {[timeField]: {$lt: aTime}}] + }, + wholeBucketFilter: { + $and: [ + {[minTimeField]: {$gt: aTime}}, + {[maxTimeField]: {$lt: bTime}}, + {[maxTimeField]: {$lt: aTime}}, + ] + }, + expectedDocs: [], +}); +})(); diff --git a/jstests/core/timeseries/timeseries_match_pushdown_with_project.js b/jstests/core/timeseries/timeseries_match_pushdown_with_project.js new file mode 100644 index 00000000000..2d8e872c33a --- /dev/null +++ b/jstests/core/timeseries/timeseries_match_pushdown_with_project.js @@ -0,0 +1,131 @@ +/** + * Tests that the unpacking stage has correct unpacking behaviour when $match is pushed into it. + * + * @tags: [ + * requires_timeseries, + * requires_fcv_60, + * does_not_support_stepdowns, + * does_not_support_transactions, + * directly_against_shardsvrs_incompatible, + * ] + */ +(function() { +"use strict"; + +load("jstests/libs/analyze_plan.js"); // For getAggPlanStages + +const coll = db.timeseries_match_pushdown_with_project; +coll.drop(); + +const timeField = 'time'; +const metaField = 'meta'; +assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField, metaField}})); + +const aTime = ISODate('2022-01-01T00:00:00'); +assert.commandWorked(coll.insert([ + {[timeField]: aTime, a: 1, b: 1, _id: 1}, + {[timeField]: aTime, a: 2, b: 2, _id: 2}, + {[timeField]: aTime, a: 3, b: 3, _id: 3}, + {[timeField]: aTime, a: 4, b: 4, _id: 4}, + {[timeField]: aTime, a: 5, b: 5, _id: 5}, + {[timeField]: aTime, a: 6, b: 6, _id: 6}, + {[timeField]: aTime, a: 7, b: 7, _id: 7}, + {[timeField]: aTime, a: 8, b: 8, _id: 8}, + {[timeField]: aTime, a: 9, b: 9, _id: 9}, +])); + +/** + * Runs a 'pipeline', asserts the bucket unpacking 'behaviour' (either include or exclude) is + * expected. + */ +const runTest = function({pipeline, behaviour, expectedDocs}) { + const explain = assert.commandWorked(coll.explain().aggregate(pipeline)); + const unpackStages = getAggPlanStages(explain, '$_internalUnpackBucket'); + assert.eq(1, + unpackStages.length, + "Should only have a single $_internalUnpackBucket stage: " + tojson(explain)); + const unpackStage = unpackStages[0].$_internalUnpackBucket; + if (behaviour.include) { + assert(unpackStage.include, + "Unpacking stage must have 'include' behaviour: " + tojson(explain)); + assert.sameMembers(behaviour.include, unpackStage.include); + } + if (behaviour.exclude) { + assert(unpackStage.exclude, + "Unpacking stage must have 'exclude' behaviour: " + tojson(explain)); + assert.sameMembers(behaviour.exclude, unpackStage.exclude); + } + + const docs = coll.aggregate([...pipeline, {$sort: {a: 1, b: 1, _id: 1}}]).toArray(); + assert.eq(docs.length, expectedDocs.length, "Incorrect docs: " + tojson(docs)); + docs.forEach((doc, i) => { + assert.docEq(doc, expectedDocs[i], "Incorrect docs: " + tojson(docs)); + }); +}; + +runTest({ + pipeline: [{$match: {a: {$gt: 5}}}, {$project: {b: 1}}], + behaviour: {include: ['_id', 'a', 'b']}, + expectedDocs: [ + {b: 6, _id: 6}, + {b: 7, _id: 7}, + {b: 8, _id: 8}, + {b: 9, _id: 9}, + ], +}); + +runTest({ + pipeline: [{$match: {a: {$gt: 5}}}, {$project: {_id: 0, b: 1}}], + behaviour: {include: ['a', 'b']}, + expectedDocs: [ + {b: 6}, + {b: 7}, + {b: 8}, + {b: 9}, + ], +}); + +runTest({ + pipeline: [{$match: {a: {$gt: 5}}}, {$project: {a: 1}}], + behaviour: {include: ['_id', 'a']}, + expectedDocs: [ + {a: 6, _id: 6}, + {a: 7, _id: 7}, + {a: 8, _id: 8}, + {a: 9, _id: 9}, + ], +}); + +runTest({ + pipeline: [{$match: {a: {$gt: 5}}}, {$project: {_id: 0, a: 1}}], + behaviour: {include: ['a']}, + expectedDocs: [ + {a: 6}, + {a: 7}, + {a: 8}, + {a: 9}, + ], +}); + +runTest({ + pipeline: [{$match: {a: {$gt: 5}}}, {$project: {a: 0}}], + behaviour: {exclude: []}, + expectedDocs: [ + {[timeField]: aTime, b: 6, _id: 6}, + {[timeField]: aTime, b: 7, _id: 7}, + {[timeField]: aTime, b: 8, _id: 8}, + {[timeField]: aTime, b: 9, _id: 9}, + ], +}); + +runTest({ + pipeline: [{$match: {a: {$gt: 5}}}, {$project: {b: 0}}], + behaviour: {exclude: []}, + expectedDocs: [ + {[timeField]: aTime, a: 6, _id: 6}, + {[timeField]: aTime, a: 7, _id: 7}, + {[timeField]: aTime, a: 8, _id: 8}, + {[timeField]: aTime, a: 9, _id: 9}, + ], +}); +})(); diff --git a/jstests/core/timeseries/timeseries_metric_index_2dsphere.js b/jstests/core/timeseries/timeseries_metric_index_2dsphere.js index e47119b5bbd..7096ace9513 100644 --- a/jstests/core/timeseries/timeseries_metric_index_2dsphere.js +++ b/jstests/core/timeseries/timeseries_metric_index_2dsphere.js @@ -18,8 +18,9 @@ load("jstests/core/timeseries/libs/timeseries.js"); load("jstests/libs/analyze_plan.js"); +load("jstests/libs/feature_flag_util.js"); -if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { +if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { return; } diff --git a/jstests/core/timeseries/timeseries_metric_index_ascending_descending.js b/jstests/core/timeseries/timeseries_metric_index_ascending_descending.js index bf1dc10e625..8a827b9d07b 100644 --- a/jstests/core/timeseries/timeseries_metric_index_ascending_descending.js +++ b/jstests/core/timeseries/timeseries_metric_index_ascending_descending.js @@ -14,9 +14,10 @@ "use strict"; load("jstests/core/timeseries/libs/timeseries.js"); +load("jstests/libs/feature_flag_util.js"); load("jstests/libs/fixture_helpers.js"); -if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { +if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; diff --git a/jstests/core/timeseries/timeseries_metric_index_compound.js b/jstests/core/timeseries/timeseries_metric_index_compound.js index 2ecc4732b70..c81f9a4c53c 100644 --- a/jstests/core/timeseries/timeseries_metric_index_compound.js +++ b/jstests/core/timeseries/timeseries_metric_index_compound.js @@ -13,8 +13,9 @@ "use strict"; load("jstests/core/timeseries/libs/timeseries.js"); +load("jstests/libs/feature_flag_util.js"); -if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { +if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; diff --git a/jstests/core/timeseries/timeseries_predicates.js b/jstests/core/timeseries/timeseries_predicates.js index a273cc8b128..ca78464cb31 100644 --- a/jstests/core/timeseries/timeseries_predicates.js +++ b/jstests/core/timeseries/timeseries_predicates.js @@ -17,7 +17,7 @@ const tsColl = db.timeseries_predicates_timeseries; coll.drop(); tsColl.drop(); assert.commandWorked( - db.createCollection(tsColl.getName(), {timeseries: {timeField: 'time', metaField: 'meta'}})); + db.createCollection(tsColl.getName(), {timeseries: {timeField: 'time', metaField: 'mt'}})); const bucketsColl = db.getCollection('system.buckets.' + tsColl.getName()); // Test that 'predicate' behaves correctly on the example documents, @@ -86,18 +86,18 @@ checkAllBucketings({x: {$exists: true}}, [ // Test $or... { - // ... on metric + meta. + // ... on metric + mt. checkAllBucketings({ $or: [ {x: {$lt: 0}}, - {'meta.y': {$gt: 0}}, + {'mt.y': {$gt: 0}}, ] }, [ - {x: +1, meta: {y: -1}}, - {x: +1, meta: {y: +1}}, - {x: -1, meta: {y: -1}}, - {x: -1, meta: {y: +1}}, + {x: +1, mt: {y: -1}}, + {x: +1, mt: {y: +1}}, + {x: -1, mt: {y: -1}}, + {x: -1, mt: {y: +1}}, ]); // ... when one argument can't be pushed down. @@ -131,18 +131,18 @@ checkAllBucketings({x: {$exists: true}}, [ // Test $and... { - // ... on metric + meta. + // ... on metric + mt. checkAllBucketings({ $and: [ {x: {$lt: 0}}, - {'meta.y': {$gt: 0}}, + {'mt.y': {$gt: 0}}, ] }, [ - {x: +1, meta: {y: -1}}, - {x: +1, meta: {y: +1}}, - {x: -1, meta: {y: -1}}, - {x: -1, meta: {y: +1}}, + {x: +1, mt: {y: -1}}, + {x: +1, mt: {y: +1}}, + {x: -1, mt: {y: -1}}, + {x: -1, mt: {y: +1}}, ]); // ... when one argument can't be pushed down. @@ -180,28 +180,28 @@ checkAllBucketings({ $or: [ { $and: [ - {'meta.a': {$gt: 0}}, + {'mt.a': {$gt: 0}}, {'x': {$lt: 0}}, ] }, { $and: [ - {'meta.b': {$gte: 0}}, + {'mt.b': {$gte: 0}}, {time: {$gt: ISODate('2020-01-01')}}, ] }, ] }, [ - {meta: {a: -1, b: -1}, x: -1, time: ISODate('2020-02-01')}, - {meta: {a: -1, b: -1}, x: -1, time: ISODate('2019-12-31')}, - {meta: {a: -1, b: -1}, x: +1, time: ISODate('2020-02-01')}, - {meta: {a: -1, b: -1}, x: +1, time: ISODate('2019-12-31')}, + {mt: {a: -1, b: -1}, x: -1, time: ISODate('2020-02-01')}, + {mt: {a: -1, b: -1}, x: -1, time: ISODate('2019-12-31')}, + {mt: {a: -1, b: -1}, x: +1, time: ISODate('2020-02-01')}, + {mt: {a: -1, b: -1}, x: +1, time: ISODate('2019-12-31')}, - {meta: {a: +1, b: -1}, x: -1, time: ISODate('2020-02-01')}, - {meta: {a: +1, b: -1}, x: -1, time: ISODate('2019-12-31')}, - {meta: {a: +1, b: -1}, x: +1, time: ISODate('2020-02-01')}, - {meta: {a: +1, b: -1}, x: +1, time: ISODate('2019-12-31')}, + {mt: {a: +1, b: -1}, x: -1, time: ISODate('2020-02-01')}, + {mt: {a: +1, b: -1}, x: -1, time: ISODate('2019-12-31')}, + {mt: {a: +1, b: -1}, x: +1, time: ISODate('2020-02-01')}, + {mt: {a: +1, b: -1}, x: +1, time: ISODate('2019-12-31')}, ]); // Test nested $and / $or where some leaf predicates cannot be pushed down. @@ -209,72 +209,72 @@ checkAllBucketings({ $or: [ { $and: [ - {'meta.a': {$gt: 0}}, + {'mt.a': {$gt: 0}}, {'x': {$exists: false}}, ] }, { $and: [ - {'meta.b': {$gte: 0}}, + {'mt.b': {$gte: 0}}, {time: {$gt: ISODate('2020-01-01')}}, ] }, ] }, [ - {meta: {a: -1, b: -1}, time: ISODate('2020-02-01')}, - {meta: {a: -1, b: -1}, time: ISODate('2019-12-31')}, - {meta: {a: -1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')}, - {meta: {a: -1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')}, + {mt: {a: -1, b: -1}, time: ISODate('2020-02-01')}, + {mt: {a: -1, b: -1}, time: ISODate('2019-12-31')}, + {mt: {a: -1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')}, + {mt: {a: -1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')}, - {meta: {a: +1, b: -1}, time: ISODate('2020-02-01')}, - {meta: {a: +1, b: -1}, time: ISODate('2019-12-31')}, - {meta: {a: +1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')}, - {meta: {a: +1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')}, + {mt: {a: +1, b: -1}, time: ISODate('2020-02-01')}, + {mt: {a: +1, b: -1}, time: ISODate('2019-12-31')}, + {mt: {a: +1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')}, + {mt: {a: +1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')}, ]); -// Test $exists on meta, inside $or. +// Test $exists on mt, inside $or. checkAllBucketings({ $or: [ - {"meta.a": {$exists: true}}, + {"mt.a": {$exists: true}}, {"x": {$gt: 2}}, ] }, [ - {meta: {a: 1}, x: 1}, - {meta: {a: 2}, x: 2}, - {meta: {a: 3}, x: 3}, - {meta: {a: 4}, x: 4}, - {meta: {}, x: 1}, - {meta: {}, x: 2}, - {meta: {}, x: 3}, - {meta: {}, x: 4}, + {mt: {a: 1}, x: 1}, + {mt: {a: 2}, x: 2}, + {mt: {a: 3}, x: 3}, + {mt: {a: 4}, x: 4}, + {mt: {}, x: 1}, + {mt: {}, x: 2}, + {mt: {}, x: 3}, + {mt: {}, x: 4}, ]); -// Test $in on meta, inside $or. +// Test $in on mt, inside $or. checkAllBucketings({ $or: [ - {"meta.a": {$in: [1, 3]}}, + {"mt.a": {$in: [1, 3]}}, {"x": {$gt: 2}}, ] }, [ - {meta: {a: 1}, x: 1}, - {meta: {a: 2}, x: 2}, - {meta: {a: 3}, x: 3}, - {meta: {a: 4}, x: 4}, - {meta: {}, x: 1}, - {meta: {}, x: 2}, - {meta: {}, x: 3}, - {meta: {}, x: 4}, + {mt: {a: 1}, x: 1}, + {mt: {a: 2}, x: 2}, + {mt: {a: 3}, x: 3}, + {mt: {a: 4}, x: 4}, + {mt: {}, x: 1}, + {mt: {}, x: 2}, + {mt: {}, x: 3}, + {mt: {}, x: 4}, ]); -// Test geo predicates on meta, inside $or. +// Test geo predicates on mt, inside $or. for (const pred of ['$geoWithin', '$geoIntersects']) { checkAllBucketings({ $or: [ { - "meta.location": { + "mt.location": { [pred]: { $geometry: { type: "Polygon", @@ -293,66 +293,90 @@ for (const pred of ['$geoWithin', '$geoIntersects']) { ] }, [ - {meta: {location: [1, 1]}, x: 1}, - {meta: {location: [1, 1]}, x: 2}, - {meta: {location: [1, 1]}, x: 3}, - {meta: {location: [1, 1]}, x: 4}, - {meta: {location: [5, 5]}, x: 1}, - {meta: {location: [5, 5]}, x: 2}, - {meta: {location: [5, 5]}, x: 3}, - {meta: {location: [5, 5]}, x: 4}, + {mt: {location: [1, 1]}, x: 1}, + {mt: {location: [1, 1]}, x: 2}, + {mt: {location: [1, 1]}, x: 3}, + {mt: {location: [1, 1]}, x: 4}, + {mt: {location: [5, 5]}, x: 1}, + {mt: {location: [5, 5]}, x: 2}, + {mt: {location: [5, 5]}, x: 3}, + {mt: {location: [5, 5]}, x: 4}, ]); } -// Test $mod on meta, inside $or. +// Test $mod on mt, inside $or. // $mod is an example of a predicate that we don't handle specially in time-series optimizations: // it can be pushed down if and only if it's on a metadata field. checkAllBucketings({ $or: [ - {"meta.a": {$mod: [2, 0]}}, + {"mt.a": {$mod: [2, 0]}}, {"x": {$gt: 4}}, ] }, [ - {meta: {a: 1}, x: 1}, - {meta: {a: 2}, x: 2}, - {meta: {a: 3}, x: 3}, - {meta: {a: 4}, x: 4}, - {meta: {a: 5}, x: 5}, - {meta: {a: 6}, x: 6}, - {meta: {a: 7}, x: 7}, - {meta: {a: 8}, x: 8}, + {mt: {a: 1}, x: 1}, + {mt: {a: 2}, x: 2}, + {mt: {a: 3}, x: 3}, + {mt: {a: 4}, x: 4}, + {mt: {a: 5}, x: 5}, + {mt: {a: 6}, x: 6}, + {mt: {a: 7}, x: 7}, + {mt: {a: 8}, x: 8}, ]); -// Test $elemMatch on meta, inside $or. +// Test $elemMatch on mt, inside $or. checkAllBucketings({ $or: [ - {"meta.a": {$elemMatch: {b: 3}}}, + {"mt.a": {$elemMatch: {b: 3}}}, {"x": {$gt: 4}}, ] }, [ - {x: 1, meta: {a: []}}, - {x: 2, meta: {a: [{b: 2}]}}, - {x: 3, meta: {a: [{b: 3}]}}, - {x: 4, meta: {a: [{b: 2}, {b: 3}]}}, - {x: 5, meta: {a: []}}, - {x: 6, meta: {a: [{b: 2}]}}, - {x: 7, meta: {a: [{b: 3}]}}, - {x: 8, meta: {a: [{b: 2}, {b: 3}]}}, + {x: 1, mt: {a: []}}, + {x: 2, mt: {a: [{b: 2}]}}, + {x: 3, mt: {a: [{b: 3}]}}, + {x: 4, mt: {a: [{b: 2}, {b: 3}]}}, + {x: 5, mt: {a: []}}, + {x: 6, mt: {a: [{b: 2}]}}, + {x: 7, mt: {a: [{b: 3}]}}, + {x: 8, mt: {a: [{b: 2}, {b: 3}]}}, ]); checkAllBucketings({ $or: [ - {"meta.a": {$elemMatch: {b: 2, c: 3}}}, + {"mt.a": {$elemMatch: {b: 2, c: 3}}}, {"x": {$gt: 3}}, ] }, [ - {x: 1, meta: {a: []}}, - {x: 2, meta: {a: [{b: 2, c: 3}]}}, - {x: 3, meta: {a: [{b: 2}, {c: 3}]}}, - {x: 4, meta: {a: []}}, - {x: 5, meta: {a: [{b: 2, c: 3}]}}, - {x: 6, meta: {a: [{b: 2}, {c: 3}]}}, + {x: 1, mt: {a: []}}, + {x: 2, mt: {a: [{b: 2, c: 3}]}}, + {x: 3, mt: {a: [{b: 2}, {c: 3}]}}, + {x: 4, mt: {a: []}}, + {x: 5, mt: {a: [{b: 2, c: 3}]}}, + {x: 6, mt: {a: [{b: 2}, {c: 3}]}}, ]); + +// Test a standalone $elemMatch on mt. +checkAllBucketings({"mt.a": {$elemMatch: {b: 3}}}, [ + {mt: {a: []}}, + {mt: {a: [{b: 2}]}}, + {mt: {a: [{b: 3}]}}, + {mt: {a: [{b: 2}, {b: 3}]}}, + {mt: {a: []}}, + {mt: {a: [{b: 2}]}}, + {mt: {a: [{b: 3}]}}, + {mt: {a: [{b: 2}, {b: 3}]}}, +]); + +// Test a standalone $size on mt. +checkAllBucketings({"mt.a": {$size: 1}}, [ + {mt: {a: []}}, + {mt: {a: [{b: 2}]}}, + {mt: {a: [{b: 3}]}}, + {mt: {a: [{b: 2}, {b: 3}]}}, + {mt: {a: []}}, + {mt: {a: [{b: 2}]}}, + {mt: {a: [{b: 3}]}}, + {mt: {a: [{b: 2}, {b: 3}]}}, +]); })(); diff --git a/jstests/core/timeseries/timeseries_project.js b/jstests/core/timeseries/timeseries_project.js index 9375f06240d..11ca90f2efe 100644 --- a/jstests/core/timeseries/timeseries_project.js +++ b/jstests/core/timeseries/timeseries_project.js @@ -4,7 +4,7 @@ * @tags: [ * does_not_support_stepdowns, * does_not_support_transactions, - * requires_fcv_53, + * requires_fcv_60, * ] */ (function() { @@ -97,6 +97,7 @@ const doc = { time: new Date("2019-10-11T14:39:18.670Z"), x: 5, a: 3, + obj: {a: 3}, }; assert.commandWorked(tsColl.insert(doc)); assert.commandWorked(regColl.insert(doc)); @@ -107,10 +108,25 @@ let tsDoc = tsColl.aggregate(pipeline).toArray(); let regDoc = regColl.aggregate(pipeline).toArray(); assert.docEq(tsDoc, regDoc); +pipeline = [{$project: {_id: 0, obj: "$x", b: {$add: ["$obj.a", 1]}}}]; +tsDoc = tsColl.aggregate(pipeline).toArray(); +regDoc = regColl.aggregate(pipeline).toArray(); +assert.docEq(tsDoc, regDoc); + // Test $addFields. pipeline = [{$addFields: {a: "$x", b: "$a"}}, {$project: {_id: 0}}]; tsDoc = tsColl.aggregate(pipeline).toArray(); regDoc = regColl.aggregate(pipeline).toArray(); assert.docEq(tsDoc, regDoc); + +pipeline = [{$addFields: {obj: "$x", b: {$add: ["$obj.a", 1]}}}, {$project: {_id: 0}}]; +tsDoc = tsColl.aggregate(pipeline).toArray(); +regDoc = regColl.aggregate(pipeline).toArray(); +assert.docEq(tsDoc, regDoc); + +pipeline = [{$project: {a: 1, _id: 0}}, {$project: {newMeta: "$x"}}]; +tsDoc = tsColl.aggregate(pipeline).toArray(); +regDoc = regColl.aggregate(pipeline).toArray(); +assert.docEq(tsDoc, regDoc); })(); })(); diff --git a/jstests/core/timeseries/timeseries_resume_after.js b/jstests/core/timeseries/timeseries_resume_after.js index a2b31972c85..66d020d43b4 100644 --- a/jstests/core/timeseries/timeseries_resume_after.js +++ b/jstests/core/timeseries/timeseries_resume_after.js @@ -8,6 +8,7 @@ * does_not_support_stepdowns, * does_not_support_transactions, * requires_getmore, + * requires_fcv_60 * ] */ (function() { @@ -91,5 +92,25 @@ TimeseriesTest.run((insert) => { resumeToken = res.cursor.postBatchResumeToken; jsTestLog("Got resume token " + tojson(resumeToken)); + + // Test that '$_resumeAfter' fails if the recordId is Long. + assert.commandFailedWithCode(db.runCommand({ + find: bucketsColl.getName(), + filter: {}, + $_requestResumeToken: true, + $_resumeAfter: {'$recordId': NumberLong(10)}, + hint: {$natural: 1} + }), + 7738600); + + // Test that '$_resumeAfter' fails if querying the time-series view. + assert.commandFailedWithCode(db.runCommand({ + find: coll.getName(), + filter: {}, + $_requestResumeToken: true, + $_resumeAfter: {'$recordId': BinData(5, '1234')}, + hint: {$natural: 1} + }), + ErrorCodes.InvalidPipelineOperator); }); })(); diff --git a/jstests/core/timeseries/timeseries_sparse_index.js b/jstests/core/timeseries/timeseries_sparse_index.js index 9ffd008ff62..7c9f7556e38 100644 --- a/jstests/core/timeseries/timeseries_sparse_index.js +++ b/jstests/core/timeseries/timeseries_sparse_index.js @@ -13,8 +13,9 @@ "use strict"; load("jstests/core/timeseries/libs/timeseries.js"); +load("jstests/libs/feature_flag_util.js"); -if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { +if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; diff --git a/jstests/core/timeseries/timeseries_streaming_group.js b/jstests/core/timeseries/timeseries_streaming_group.js new file mode 100644 index 00000000000..6d97eb51621 --- /dev/null +++ b/jstests/core/timeseries/timeseries_streaming_group.js @@ -0,0 +1,159 @@ +/** + * Tests that the $group stage will be replaced with $_internalStreamingGroup when group id is + * monotonic on time and documents are sorted on time. + * + * @tags: [ + * # Explain of a resolved view must be executed by mongos. + * directly_against_shardsvrs_incompatible, + * does_not_support_stepdowns, + * does_not_support_transactions, + * requires_fcv_60, + * requires_timeseries, + * ] + */ +(function() { +"use strict"; + +load("jstests/libs/analyze_plan.js"); // For getAggPlanStages +load("jstests/libs/fail_point_util.js"); // For configureFailPoint + +const ts = db.timeseries_streaming_group; +ts.drop(); + +const coll = db.timeseires_streaming_group_regular_collection; +coll.drop(); + +assert.commandWorked( + db.createCollection(ts.getName(), {timeseries: {timeField: "time", metaField: "meta"}})); + +const numTimes = 100; +const numSymbols = 10; +const minPrice = 100; +const maxPrice = 200; +const minAmount = 1; +const maxAmount = 20; + +Random.setRandomSeed(1); + +const randRange = function(min, max) { + return min + Random.randInt(max - min); +}; + +const symbols = []; +for (let i = 0; i < numSymbols; i++) { + let randomName = ""; + const randomStrLen = 5; + for (let j = 0; j < randomStrLen; j++) { + randomName += String.fromCharCode("A".charCodeAt(0) + Random.randInt(26)); + } + symbols.push(randomName); +} + +const documents = []; +const startTime = 1641027600000; +for (let i = 0; i < numTimes; i++) { + for (const symbol of symbols) { + documents.push({ + time: new Date(startTime + i * 1000), + price: randRange(minPrice, maxPrice), + amount: randRange(minAmount, maxAmount), + meta: {"symbol": symbol} + }); + } +} + +assert.commandWorked(ts.insert(documents)); +assert.commandWorked(coll.insert(documents)); + +// Incorrect use of $_internalStreamingGroup should return error +assert.commandFailedWithCode(db.runCommand({ + aggregate: "timeseires_streaming_group_regular_collection", + pipeline: [{ + $_internalStreamingGroup: { + _id: {symbol: "$symbol", time: "$time"}, + count: {$sum: 1}, + $monotonicIdFields: ["price"] + } + }], + cursor: {}, +}), + 7026705); +assert.commandFailedWithCode(db.runCommand({ + aggregate: "timeseires_streaming_group_regular_collection", + pipeline: [{$_internalStreamingGroup: {_id: null, count: {$sum: 1}}}], + cursor: {}, +}), + 7026702); +assert.commandFailedWithCode(db.runCommand({ + aggregate: "timeseires_streaming_group_regular_collection", + pipeline: + [{$_internalStreamingGroup: {_id: null, count: {$sum: 1}, $monotonicIdFields: ["_id"]}}], + cursor: {}, +}), + 7026708); + +const runTest = function(pipeline, expectedMonotonicIdFields) { + const explain = assert.commandWorked(ts.explain().aggregate(pipeline)); + const streamingGroupStage = getAggPlanStage(explain, "$_internalStreamingGroup"); + assert.neq(streamingGroupStage, null); + assert.eq(streamingGroupStage.$_internalStreamingGroup.$monotonicIdFields, + expectedMonotonicIdFields); + + const found = ts.aggregate(pipeline).toArray(); + const expected = coll.aggregate(pipeline).toArray(); + assert.eq(expected, found); +}; + +runTest( + [ + {$sort: {time: 1}}, + { + $group: { + _id: { + symbol: "$meta.symbol", + minute: { + $subtract: [ + {$dateTrunc: {date: "$time", unit: "minute"}}, + {$dateTrunc: {date: new Date(startTime), unit: "minute"}}, + ] + } + }, + "average_price": {$avg: {$multiply: ["$price", "$amount"]}}, + "average_amount": {$avg: "$amount"} + } + }, + {$addFields: {"average_price": {$divide: ["$average_price", "$average_amount"]}}}, + {$sort: {_id: 1}} + ], + ["minute"]); + +runTest( + [ + {$sort: {time: 1}}, + { + $group: { + _id: {$dateTrunc: {date: "$time", unit: "minute"}}, + "average_price": {$avg: {$multiply: ["$price", "$amount"]}}, + "average_amount": {$avg: "$amount"} + } + }, + {$addFields: {"average_price": {$divide: ["$average_price", "$average_amount"]}}}, + {$sort: {_id: 1}} + ], + ["_id"]); + +runTest( + [ + {$sort: {time: 1}}, + { + $group: { + _id: "$time", + "average_price": {$avg: {$multiply: ["$price", "$amount"]}}, + "average_amount": {$avg: "$amount"} + } + }, + {$addFields: {"average_price": {$divide: ["$average_price", "$average_amount"]}}}, + {$sort: {_id: 1}} + ], + ["_id"]); +})(); |