diff options
Diffstat (limited to 'jstests/core/timeseries')
40 files changed, 324 insertions, 3882 deletions
diff --git a/jstests/core/timeseries/bucket_unpacking_with_sort.js b/jstests/core/timeseries/bucket_unpacking_with_sort.js index 695e94daec9..029697be74a 100644 --- a/jstests/core/timeseries/bucket_unpacking_with_sort.js +++ b/jstests/core/timeseries/bucket_unpacking_with_sort.js @@ -17,9 +17,6 @@ * requires_timeseries, * # Explain of a resolved view must be executed by mongos. * directly_against_shardsvrs_incompatible, - * # Time-series hint by index key did not exist in 5.0. - * # Also, the 5.0 version of this feature does not support reverse collscan. - * requires_fcv_60, * ] */ (function() { diff --git a/jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js b/jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js deleted file mode 100644 index e74d1c3e914..00000000000 --- a/jstests/core/timeseries/bucket_unpacking_with_sort_extended_range.js +++ /dev/null @@ -1,269 +0,0 @@ -/** - * Test that sort queries work properly on dates ouside the 32 bit epoch range, - * [1970-01-01 00:00:00 UTC - 2038-01-29 03:13:07 UTC], when a collection scan is used. - * - * @tags: [ - * # Explain of a resolved view must be executed by mongos. - * directly_against_shardsvrs_incompatible, - * # This complicates aggregation extraction. - * do_not_wrap_aggregations_in_facets, - * # Refusing to run a test that issues an aggregation command with explain because it may - * # return incomplete results if interrupted by a stepdown. - * does_not_support_stepdowns, - * # We need a timeseries collection. - * requires_timeseries, - * # Cannot insert into a time-series collection in a multi-document transaction. - * does_not_support_transactions, - * # Time-series hint by index key did not exist in 5.0 - * requires_fcv_60, - * ] - */ - -(function() { -"use strict"; - -load("jstests/aggregation/extras/utils.js"); // For getExplainedPipelineFromAggregation. -load("jstests/core/timeseries/libs/timeseries.js"); -load('jstests/libs/analyze_plan.js'); - -if (!TimeseriesTest.bucketUnpackWithSortEnabled(db.getMongo())) { - jsTestLog("Skipping test because 'BucketUnpackWithSort' is disabled."); - return; -} - -const timeFieldName = "t"; - -// Create unindexed collection -const coll = db.timeseries_internal_bounded_sort_extended_range; -const buckets = db['system.buckets.' + coll.getName()]; -coll.drop(); -assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField: 't'}})); - -// Create collection indexed on time -const collIndexed = db.timeseries_internal_bounded_sort_extended_range_with_index; -const bucketsIndexed = db['system.buckets.' + collIndexed.getName()]; -collIndexed.drop(); -assert.commandWorked(db.createCollection(collIndexed.getName(), {timeseries: {timeField: 't'}})); -assert.commandWorked(collIndexed.createIndex({'t': 1})); - -jsTestLog(collIndexed.getIndexes()); -jsTestLog(bucketsIndexed.getIndexes()); - -function numShards() { - return db.getSiblingDB('config').shards.count(); -} -for (const collection of [buckets, bucketsIndexed]) { - if (FixtureHelpers.isSharded(collection) && numShards() >= 2) { - // Split and move data to create an interesting scenario: we have some data on each shard, - // but all the extended-range data is on a non-primary shard. This means view resolution is - // unaware of the extended-range data, because that happens on the primary shard. - - const shards = db.getSiblingDB('config').shards.find().toArray(); - const [shardName0, shardName1] = shards.map(doc => doc._id); - - assert.commandWorked(db.adminCommand({movePrimary: db.getName(), to: shardName0})); - const collName = collection.getFullName(); - // Our example data has documents between 2000-2003, and these dates are non-wrapping. - // So this goes on the primary shard, and everything else goes on the non-primary. - assert.commandWorked(sh.splitAt(collName, {'control.min.t': ISODate('2000-01-01')})); - assert.commandWorked(sh.splitAt(collName, {'control.min.t': ISODate('2003-01-01')})); - assert.commandWorked( - sh.moveChunk(collName, {'control.min.t': ISODate('1969-01-01')}, shardName1)); - assert.commandWorked( - sh.moveChunk(collName, {'control.min.t': ISODate('2000-01-01')}, shardName0)); - assert.commandWorked( - sh.moveChunk(collName, {'control.min.t': ISODate('2003-01-01')}, shardName1)); - } -} - -const intervalMillis = 60000; -function insertBucket(start) { - jsTestLog("Inserting bucket starting with " + Date(start).toString()); - - const batchSize = 1000; - - const batch = - Array.from({length: batchSize}, (_, j) => ({t: new Date(start + j * intervalMillis)})); - - assert.commandWorked(coll.insert(batch)); - assert.commandWorked(collIndexed.insert(batch)); -} - -// Insert some data. We'll insert 5 buckets in each range, with values < 0, -// values between 0 and FFFFFFFF(unsigned), and values > FFFFFFFF. It turns out, however, -// that Javascript's Date doesn't handle dates beyond 2039 either, so we rely on lower dates -// to test for unexpected behavior. -function insertDocuments() { - // We want to choose the underflow and overflow lower bits in such a way that we - // encourage wrong results when the upper bytes are removed. - const underflowMin = new Date("1969-01-01").getTime(); // Year before the 32 bit epoch - const normalMin = new Date("2002-01-01").getTime(); // Middle of the 32 bit epoch - - insertBucket(underflowMin); - - var numBatches = 5; - - const batchOffset = Math.floor(intervalMillis / (numBatches + 1)); - for (let i = 0; i < numBatches; ++i) { - const start = normalMin + i * batchOffset; - insertBucket(start); - } - assert.gt(buckets.aggregate([{$count: 'n'}]).next().n, 1, 'Expected more than one bucket'); -} - -insertDocuments(); - -const unpackStage = getAggPlanStages(coll.explain().aggregate(), '$_internalUnpackBucket')[0]; - -function assertSorted(result, ascending) { - let prev = ascending ? {t: -Infinity} : {t: Infinity}; - for (const doc of result) { - if (ascending) { - assert.lt(+prev.t, - +doc.t, - 'Found two docs not in ascending time order: ' + tojson({prev, doc})); - } else { - assert.gt(+prev.t, - +doc.t, - 'Found two docs not in descending time order: ' + tojson({prev, doc})); - } - - prev = doc; - } -} - -function checkAgainstReferenceBoundedSortUnexpected( - collection, reference, pipeline, hint, sortOrder) { - const options = hint ? {hint: hint} : {}; - - const bucket = db['system.buckets.' + coll.getName()]; - - const plan = collection.explain().aggregate(pipeline, options); - if (FixtureHelpers.isSharded(buckets) && numShards() >= 2) { - // With a sharded collection, some shards might not have any extended-range data, - // so they might still use $_internalBoundedSort. But we know at least one - // shard has extended-range data, so we know at least one shard has to - // use a blocking sort. - const bounded = getAggPlanStages(plan, "$_internalBoundedSort"); - const blocking = getAggPlanStages(plan, "$sort"); - assert.gt(blocking.length, 0, {bounded, blocking, plan}); - assert.lt(bounded.length, - FixtureHelpers.numberOfShardsForCollection(buckets), - {bounded, blocking, plan}); - } else { - const stages = getAggPlanStages(plan, "$_internalBoundedSort"); - assert.eq([], stages, plan); - } - - const opt = collection.aggregate(pipeline, options).toArray(); - assertSorted(opt, sortOrder); - - assert.eq(reference.length, opt.length); - for (var i = 0; i < opt.length; ++i) { - assert.docEq(reference[i], opt[i]); - } -} - -function checkAgainstReferenceBoundedSortExpected( - collection, reference, pipeline, hint, sortOrder) { - const options = hint ? {hint: hint} : {}; - - const plan = collection.explain().aggregate(pipeline, options); - const stages = getAggPlanStages(plan, "$_internalBoundedSort"); - assert.neq([], stages, plan); - - const opt = collection.aggregate(pipeline, options).toArray(); - assertSorted(opt, sortOrder); - - assert.eq(reference.length, opt.length); - for (var i = 0; i < opt.length; ++i) { - assert.docEq(reference[i], opt[i]); - } -} - -function runTest(ascending) { - const reference = buckets - .aggregate([ - unpackStage, - {$_internalInhibitOptimization: {}}, - {$sort: {t: ascending ? 1 : -1}}, - ]) - .toArray(); - assertSorted(reference, ascending); - - // Check plan using collection scan - checkAgainstReferenceBoundedSortUnexpected(coll, - reference, - [ - {$sort: {t: ascending ? 1 : -1}}, - ], - {}, - ascending); - - // Check plan using hinted collection scan - checkAgainstReferenceBoundedSortUnexpected(coll, - reference, - [ - {$sort: {t: ascending ? 1 : -1}}, - ], - {"$natural": ascending ? 1 : -1}, - ascending); - - const referenceIndexed = bucketsIndexed - .aggregate([ - unpackStage, - {$_internalInhibitOptimization: {}}, - {$sort: {t: ascending ? 1 : -1}}, - ]) - .toArray(); - assertSorted(referenceIndexed, ascending); - - // Check plan using index scan. If we've inserted a date before 1-1-1970, we round the min - // up towards 1970, rather then down, which has the effect of increasing the control.min.t. - // This means the minimum time in the bucket is likely to be lower than indicated and thus, - // actual dates may be out of order relative to what's indicated by the bucket bounds. - checkAgainstReferenceBoundedSortUnexpected(collIndexed, - referenceIndexed, - [ - {$sort: {t: ascending ? 1 : -1}}, - ], - {}, - ascending); - - // Check plan using hinted index scan - checkAgainstReferenceBoundedSortUnexpected(collIndexed, - referenceIndexed, - [ - {$sort: {t: ascending ? 1 : -1}}, - ], - {"t": 1}, - ascending); - - // Check plan using hinted collection scan - checkAgainstReferenceBoundedSortUnexpected(collIndexed, - referenceIndexed, - [ - {$sort: {t: ascending ? 1 : -1}}, - ], - {"$natural": ascending ? 1 : -1}, - ascending); - - // The workaround in all cases is to create a reverse index on the time field, though - // it's necessary to force use of the reversed index. - const reverseIdxName = "reverseIdx"; - collIndexed.createIndex({t: -1}, {name: reverseIdxName}); - - checkAgainstReferenceBoundedSortExpected(collIndexed, - referenceIndexed, - [ - {$sort: {t: ascending ? 1 : -1}}, - ], - {"t": -1}, - ascending); - - collIndexed.dropIndex(reverseIdxName); -} - -runTest(false); // descending -runTest(true); // ascending -})(); diff --git a/jstests/core/timeseries/bucket_unpacking_with_sort_plan_cache.js b/jstests/core/timeseries/bucket_unpacking_with_sort_plan_cache.js index e69a8d7a860..25e1c99312e 100644 --- a/jstests/core/timeseries/bucket_unpacking_with_sort_plan_cache.js +++ b/jstests/core/timeseries/bucket_unpacking_with_sort_plan_cache.js @@ -21,8 +21,6 @@ * directly_against_shardsvrs_incompatible, * # We use the profiler to get info in order to force replanning. * requires_profiling, - * # The 5.0 version of this feature does not support reverse collscan. - * requires_fcv_60, * ] */ (function() { diff --git a/jstests/core/timeseries/libs/timeseries.js b/jstests/core/timeseries/libs/timeseries.js index 540e0bcea55..ea8d71c87a1 100644 --- a/jstests/core/timeseries/libs/timeseries.js +++ b/jstests/core/timeseries/libs/timeseries.js @@ -11,20 +11,6 @@ var TimeseriesTest = class { return FeatureFlagUtil.isEnabled(conn, "TimeseriesBucketCompression"); } - static bucketsMayHaveMixedSchemaData(coll) { - const catalog = coll.aggregate([{$listCatalog: {}}]).toArray()[0]; - const tsMixedSchemaOptionNewFormat = catalog.md.options.storageEngine && - catalog.md.options.storageEngine.wiredTiger && - catalog.md.options.storageEngine.wiredTiger.configString; - // TODO SERVER-92533 Simplify once SERVER-91195 is backported to all supported branches - if (tsMixedSchemaOptionNewFormat !== undefined) { - return tsMixedSchemaOptionNewFormat == - "app_metadata=(timeseriesBucketsMayHaveMixedSchemaData=true)"; - } else { - return catalog.md.timeseriesBucketsMayHaveMixedSchemaData; - } - } - /** * Returns whether time-series updates and deletes are supported. */ @@ -66,16 +52,9 @@ var TimeseriesTest = class { } static bucketUnpackWithSortEnabled(conn) { - const resp50 = conn.adminCommand({getParameter: 1, featureFlagBucketUnpackWithSort50: 1}); - const resp = conn.adminCommand({getParameter: 1, featureFlagBucketUnpackWithSort: 1}); - assert(Boolean(resp50.ok) ^ Boolean(resp.ok), - `Expected exactly one of these feature flags to exist: ${tojson(resp50)} vs ${ - tojson(resp)}`); - if (resp50.ok) { - return assert.commandWorked(resp50).featureFlagBucketUnpackWithSort50.value; - } else { - return assert.commandWorked(resp).featureFlagBucketUnpackWithSort.value; - } + return assert + .commandWorked(conn.adminCommand({getParameter: 1, featureFlagBucketUnpackWithSort: 1})) + .featureFlagBucketUnpackWithSort.value; } /** @@ -220,7 +199,7 @@ var TimeseriesTest = class { static ensureDataIsDistributedIfSharded(coll, splitPointDate) { const db = coll.getDB(); - const buckets = db[this.getBucketsCollName(coll.getName())]; + const buckets = db["system.buckets." + coll.getName()]; if (FixtureHelpers.isSharded(buckets)) { const timeFieldName = db.getCollectionInfos({name: coll.getName()})[0].options.timeseries.timeField; @@ -269,8 +248,4 @@ var TimeseriesTest = class { } } } - - static getBucketsCollName(collName) { - return `system.buckets.${collName}`; - } }; diff --git a/jstests/core/timeseries/nondefault_collation.js b/jstests/core/timeseries/nondefault_collation.js index 01f9b8c635c..3ff1c63d330 100644 --- a/jstests/core/timeseries/nondefault_collation.js +++ b/jstests/core/timeseries/nondefault_collation.js @@ -1,13 +1,8 @@ /** - * Correctness tests for TS collections with collation that might not match the explicit collation, - * specified in the query. - * - * Queries on timeseries attempt various optimizations to avoid unpacking of buckets. These rely on - * the meta field and the control data (currently, min and max), computed for each bucket. - * Collection's collation might affect the computed control values. + * Test ensures that users can specify non-default collation when querying on time-series + * collections. * * @tags: [ - * assumes_unsharded_collection, * requires_non_retryable_writes, * requires_pipeline_optimization, * requires_getmore, @@ -26,221 +21,123 @@ load("jstests/libs/analyze_plan.js"); const coll = db.timeseries_nondefault_collation; const bucketsColl = db.getCollection('system.buckets.' + coll.getName()); +coll.drop(); // implicitly drops bucketsColl. + +const timeFieldName = 'time'; +const metaFieldName = 'meta'; + const numericOrdering = { - locale: "en_US", - numericOrdering: true, - strength: 1 // case and diacritics ignored + collation: {locale: "en_US", numericOrdering: true} }; + const caseSensitive = { - locale: "en_US", - strength: 1, - caseLevel: true + collation: {locale: "en_US", strength: 1, caseLevel: true, numericOrdering: true} }; + const diacriticSensitive = { - locale: "en_US", - strength: 2, - caseLevel: false + collation: {locale: "en_US", strength: 2} }; -const insensitive = { - locale: "en_US", + +const englishCollation = { + locale: 'en', strength: 1 }; -// Find on meta field isn't different from a find on any other view, but let's check it anyway. -(function testFind_MetaField() { - coll.drop(); - - assert.commandWorked(db.createCollection( - coll.getName(), - {timeseries: {timeField: 'time', metaField: 'meta'}, collation: numericOrdering})); - assert.contains(bucketsColl.getName(), db.getCollectionNames()); - - assert.commandWorked(coll.insert({time: ISODate(), meta: "1", value: 42})); - assert.commandWorked(coll.insert({time: ISODate(), meta: "10", value: 42})); - assert.commandWorked(coll.insert({time: ISODate(), meta: "5", value: 42})); - - // Use the collection's collation with numeric ordering. - let res1 = coll.find({meta: {$gt: "4"}}); - assert.eq(2, res1.itcount(), res1.toArray()); // should match "5" and "10" - - // Use explicit collation with lexicographic ordering. - let res2 = coll.find({meta: {$gt: "4"}}).collation(insensitive); - assert.eq(1, res2.itcount(), res2.toArray()); // should match only "5" -}()); - -// For the measurement fields each bucket computes additional "control values", such as min/max and -// might use them to avoid unpacking. -(function testFind_MeasurementField() { - coll.drop(); - - assert.commandWorked(db.createCollection( - coll.getName(), - {timeseries: {timeField: 'time', metaField: 'meta'}, collation: numericOrdering})); - assert.contains(bucketsColl.getName(), db.getCollectionNames()); - - // The 'numericOrdering' on the collection means that the max of the bucket with the three docs - // below is "10" (while the lexicographic max is "5"). - assert.commandWorked(coll.insert({time: ISODate(), meta: 42, value: "1"})); - assert.commandWorked(coll.insert({time: ISODate(), meta: 42, value: "10"})); - assert.commandWorked(coll.insert({time: ISODate(), meta: 42, value: "5"})); - - // A query with default collation would use the bucket's min/max and find the matches. We are - // not checking the unpacking optimizations here as it's not a concern of collation per se. - let res1 = coll.find({value: {$gt: "4"}}); - assert.eq(2, res1.itcount(), res1.toArray()); // should match "5" and "10" - - // If a query with 'insensitive' collation, which doesn't do numeric ordering, used the bucket's - // min/max it would miss the bucket. Check, that it doesn't. - let res2 = coll.find({value: {$gt: "4"}}).collation(insensitive); - assert.eq(1, res2.itcount(), res2.toArray()); // should match only "5" -}()); - -(function testFind_OnlyQueryHasCollation() { - coll.drop(); - - assert.commandWorked( - db.createCollection(coll.getName(), {timeseries: {timeField: 'time', metaField: 'meta'}})); - - // This should generate a bucket with control.min.value = 'C' and control.max.value = 'c'. - assert.commandWorked(coll.insert({time: ISODate(), meta: 42, value: "C"})); - assert.commandWorked(coll.insert({time: ISODate(), meta: 42, value: "b"})); - assert.commandWorked(coll.insert({time: ISODate(), meta: 42, value: "c"})); - - // A query with default collation would use the bucket's min/max and find the two matches. - const resWithNoCollation = coll.find({value: {$lt: "c"}}); - assert.eq(2, - resWithNoCollation.itcount(), - resWithNoCollation.toArray()); // should match "C" and "b". - - // If a query with 'insensitive' collation used the bucket's min/max it would miss the bucket. - // Check, that it doesn't. - const resWithCollation_find = coll.find({value: {$lt: "c"}}).collation(insensitive); - assert.eq(1, - resWithCollation_find.itcount(), - resWithCollation_find.toArray()); // should match only "b". - - // Run the same test with aggregate command. - const resWithCollation_agg = - coll.aggregate([{$match: {value: {$lt: "c"}}}], {collation: insensitive}).toArray(); - assert.eq(1, resWithCollation_agg.length, resWithCollation_agg); -}()); - -(function testAgg_GroupByMetaField() { - coll.drop(); - - assert.commandWorked(db.createCollection( - coll.getName(), - {timeseries: {timeField: 'time', metaField: 'meta'}, collation: numericOrdering})); - assert.contains(bucketsColl.getName(), db.getCollectionNames()); - - assert.commandWorked(coll.insert({time: ISODate(), meta: "1", val: 1})); - assert.commandWorked(coll.insert({time: ISODate(), meta: "5", val: 1})); - - // Using collection's collation with numeric ordering. - let res1 = - coll.aggregate([{$bucket: {groupBy: "$meta", boundaries: ["1", "10", "50"]}}]).toArray(); - assert.eq(1, res1.length); - assert.eq({_id: "1", count: 2}, res1[0]); - - // Using explicit collation with lexicographic ordering. - let res2 = coll.aggregate([{$bucket: {groupBy: "$meta", boundaries: ["1", "10", "50"]}}], - {collation: insensitive}) - .toArray(); - assert.eq(2, res2.length); - assert.eq({_id: "1", count: 1}, res2[0]); // "1" goes here - assert.eq({_id: "10", count: 1}, res2[1]); // "5" goes here -}()); - -(function testAgg_GroupByMeasurementField() { - coll.drop(); - - assert.commandWorked(db.createCollection( - coll.getName(), - {timeseries: {timeField: 'time', metaField: 'meta'}, collation: insensitive})); - assert.contains(bucketsColl.getName(), db.getCollectionNames()); - - // Cause two different buckets with various case/diacritics in each for the measurement 'name'. - assert.commandWorked(coll.insert({time: ISODate(), meta: "a", name: 'A'})); - assert.commandWorked(coll.insert({time: ISODate(), meta: "a", name: 'a'})); - assert.commandWorked(coll.insert({time: ISODate(), meta: "a", name: 'á'})); - assert.commandWorked(coll.insert({time: ISODate(), meta: "b", name: 'A'})); - assert.commandWorked(coll.insert({time: ISODate(), meta: "b", name: 'a'})); - assert.commandWorked(coll.insert({time: ISODate(), meta: "b", name: 'ä'})); - - // Test with the collection's collation, which is case and diacritic insensitive. - assert.eq(1, coll.aggregate([{$sortByCount: "$name"}]).itcount()); - - // Test with explicit collation that is different from the collection's. - assert.eq(2, coll.aggregate([{$sortByCount: "$name"}], {collation: caseSensitive}).itcount()); - assert.eq(3, - coll.aggregate([{$sortByCount: "$name"}], {collation: diacriticSensitive}).itcount()); -}()); - -// For $group queries that would put whole buckets into the same group, it might be possible to -// avoid unpacking if the information the group is computing is exposed in the control data of each -// bucket. Currently, we only do this optimization for min/max with the meta as the group key. -(function testAgg_MinMaxOptimization() { - coll.drop(); - - assert.commandWorked(db.createCollection( - coll.getName(), - {timeseries: {timeField: 'time', metaField: 'meta'}, collation: numericOrdering})); - assert.contains(bucketsColl.getName(), db.getCollectionNames()); - - // These two docs will be placed in the same bucket, and the max for the bucket will be computed - // using collection's collation, that is, it should be "10". - assert.commandWorked(coll.insert({time: ISODate(), meta: 42, val: "10"})); - assert.commandWorked(coll.insert({time: ISODate(), meta: 42, val: "5"})); - - // Let's check our understanding of what happens with the bucketing as otherwise the tests below - // won't be testing what we think they are. - let buckets = bucketsColl.find().toArray(); - assert.eq(1, buckets.length, "All docs should be placed into the same bucket"); - assert.eq("10", buckets[0].control.max.val, "Computed max control for 'val' measurement"); - - // Use the collection's collation with numeric ordering. - let res1 = coll.aggregate([{$group: {_id: "$meta", v: {$max: "$val"}}}]).toArray(); - assert.eq("10", res1[0].v, "max val in numeric ordering per the collection's collation"); - - // Use the collection's collation with lexicographic ordering. - let res2 = - coll.aggregate([{$group: {_id: "$meta", v: {$max: "$val"}}}], {collation: insensitive}) - .toArray(); - assert.eq("5", res2[0].v, "max val in lexicographic ordering per the query collation"); -}()); - -(function testFind_IndexWithDifferentCollation() { - coll.drop(); - - assert.commandWorked(db.createCollection( - coll.getName(), - {timeseries: {timeField: 'time', metaField: 'meta'}, collation: diacriticSensitive})); - assert.contains(bucketsColl.getName(), db.getCollectionNames()); - - // Create index with a different collation. - assert.commandWorked(coll.createIndex({meta: 1}, {collation: insensitive})); - - // We only check that the correct plan is chosen so the contents of the collection don't matter - // as long as it's not empty. - assert.commandWorked(coll.insert({time: ISODate(), meta: 42})); - assert.commandWorked(coll.insert({time: ISODate(), meta: "the answer"})); - - // Queries that don't specify explicit collation should use the collection's default collation - // which isn't compatible with the index, so the index should NOT be used. - let query = coll.find({meta: "str"}).explain(); - assert(!aggPlanHasStage(query, "IXSCAN"), query); - - // Queries with an explicit collation which isn't compatible with the index, should NOT do - // index scan. - query = coll.find({meta: "str"}).collation(caseSensitive).explain(); - assert(!aggPlanHasStage(query, "IXSCAN"), query); - - // Queries with the same collation as in the index, should do index scan. - query = coll.find({meta: "str"}).collation(insensitive).explain(); - assert(aggPlanHasStage(query, "IXSCAN"), query); +const simpleCollation = { + collation: {locale: "simple"} +}; - // Numeric queries that don't rely on collation should do index scan. - query = coll.find({meta: 1}).explain(); - assert(aggPlanHasStage(query, "IXSCAN"), query); -}()); +assert.commandWorked(db.createCollection(coll.getName(), { + timeseries: {timeField: timeFieldName, metaField: metaFieldName}, + collation: englishCollation +})); +assert.contains(bucketsColl.getName(), db.getCollectionNames()); + +assert.commandWorked( + coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "1", name: 'A', name2: "á"})); +assert.commandWorked( + coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "2", name: 'a', name2: "á"})); +assert.commandWorked( + coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "5", name: 'A', name2: "á"})); +assert.commandWorked( + coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "10", name: 'a', name2: "á"})); +assert.commandWorked( + coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "20", name: 'A', name2: "a"})); +assert.commandWorked( + coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "50", name: 'B', name2: "a"})); +assert.commandWorked( + coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "100", name: 'b', name2: "a"})); +assert.commandWorked( + coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "200", name: 'B', name2: "a"})); +assert.commandWorked( + coll.insert({[timeFieldName]: ISODate(), [metaFieldName]: "500", name: 'b', name2: "a"})); + +// Default collation is case and diacretic insensitive. +assert.eq(2, coll.aggregate([{$sortByCount: "$name"}]).itcount()); +assert.eq(1, coll.aggregate([{$sortByCount: "$name2"}]).itcount()); + +// Test that a explicit collation different from collection's default passes for a timeseries +// collection. +let results = + coll.aggregate([{$bucket: {groupBy: "$meta", boundaries: ["1", "10", "100", "1000"]}}], + numericOrdering) + .toArray(); +assert.eq(3, results.length); +assert.eq({_id: "1", count: 3}, results[0]); +assert.eq({_id: "10", count: 3}, results[1]); +assert.eq({_id: "100", count: 3}, results[2]); + +assert.eq(4, coll.aggregate([{$sortByCount: "$name"}], caseSensitive).itcount()); +assert.eq(2, coll.aggregate([{$sortByCount: "$name2"}], diacriticSensitive).itcount()); + +coll.drop(); +const defaultCollation = { + locale: "en", + numericOrdering: true, + caseLevel: true, + strength: 2 +}; +assert.commandWorked(db.createCollection(coll.getName(), { + timeseries: {timeField: timeFieldName, metaField: metaFieldName}, + collation: defaultCollation +})); +assert.contains(bucketsColl.getName(), db.getCollectionNames()); +assert.commandWorked(coll.createIndex({[metaFieldName]: 1}, {collation: {locale: "simple"}})); + +assert.commandWorked(coll.insert( + {[timeFieldName]: ISODate(), [metaFieldName]: 1, name: 'A', name2: "á", value: "1"})); +assert.commandWorked(coll.insert( + {[timeFieldName]: ISODate(), [metaFieldName]: 2, name: 'a', name2: "á", value: "11"})); +assert.commandWorked(coll.insert( + {[timeFieldName]: ISODate(), [metaFieldName]: 1, name: 'A', name2: "á", value: "50"})); +assert.commandWorked(coll.insert( + {[timeFieldName]: ISODate(), [metaFieldName]: 1, name: 'a', name2: "á", value: "100"})); +assert.commandWorked(coll.insert( + {[timeFieldName]: ISODate(), [metaFieldName]: "2", name: 'A', name2: "a", value: "3"})); +assert.commandWorked(coll.insert( + {[timeFieldName]: ISODate(), [metaFieldName]: "5", name: 'B', name2: "a", value: "-100"})); +assert.commandWorked(coll.insert( + {[timeFieldName]: ISODate(), [metaFieldName]: "1", name: 'b', name2: "a", value: "-200"})); +assert.commandWorked(coll.insert( + {[timeFieldName]: ISODate(), [metaFieldName]: "2", name: 'B', name2: "a", value: "1000"})); +assert.commandWorked(coll.insert( + {[timeFieldName]: ISODate(), [metaFieldName]: "5", name: 'b', name2: "a", value: "4"})); + +// This collection has been created using non simple collation. The collection was then indexed on +// its metadata using simple collation. These tests confirm that queries on the indexed field using +// nondefault (simple) collation use the index. They also confirm that queries that don't involve +// strings but do use default collation, on indexed fields, also use the index. +const nonDefaultCollationQuery = coll.find({meta: 2}, {collation: englishCollation}).explain(); +assert(aggPlanHasStage(nonDefaultCollationQuery, "IXSCAN"), nonDefaultCollationQuery); + +const simpleNonDefaultCollationQuery = coll.find({meta: 2}, simpleCollation).explain(); +assert(aggPlanHasStage(simpleNonDefaultCollationQuery, "IXSCAN"), simpleNonDefaultCollationQuery); + +const defaultCollationQuery = coll.find({meta: 1}, {collation: defaultCollation}).explain(); +assert(aggPlanHasStage(defaultCollationQuery, "IXSCAN"), defaultCollationQuery); + +// This test guarantees that the bucket's min/max matches the query's min/max regardless of +// collation. +results = coll.find({value: {$gt: "4"}}, simpleCollation); +assert.eq(4, results.itcount()); }()); diff --git a/jstests/core/timeseries/partialFilterExpression_with_internalBucketGeoWithin.js b/jstests/core/timeseries/partialFilterExpression_with_internalBucketGeoWithin.js index 61521e7f86e..5be75d79e28 100644 --- a/jstests/core/timeseries/partialFilterExpression_with_internalBucketGeoWithin.js +++ b/jstests/core/timeseries/partialFilterExpression_with_internalBucketGeoWithin.js @@ -10,11 +10,12 @@ */ load("jstests/libs/analyze_plan.js"); -load("jstests/libs/feature_flag_util.js"); load('jstests/noPassthrough/libs/index_build.js'); - (function() { -if (FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { +const isFeatureEnabled = db.adminCommand({getParameter: 1, featureFlagTimeseriesMetricIndexes: 1}) + .featureFlagTimeseriesMetricIndexes.value; + +if (isFeatureEnabled) { const timeFieldName = "timestamp"; const coll = db.partialFilterExpression_with_internalBucketGeoWithin; diff --git a/jstests/core/timeseries/timeseries_bucket_rename.js b/jstests/core/timeseries/timeseries_bucket_rename.js new file mode 100644 index 00000000000..98a0b73b810 --- /dev/null +++ b/jstests/core/timeseries/timeseries_bucket_rename.js @@ -0,0 +1,28 @@ +/** + * Tests that a system.buckets collection cannot be renamed. + * + * @tags: [ + * does_not_support_stepdowns, + * does_not_support_transactions, + * requires_getmore, + * ] + */ +(function() { +'use strict'; + +const coll = db.timeseries_bucket_rename; +const bucketsColl = db.getCollection('system.buckets.' + coll.getName()); + +const timeFieldName = 'time'; + +coll.drop(); +assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField: timeFieldName}})); +assert.contains(bucketsColl.getName(), db.getCollectionNames()); + +assert.commandFailedWithCode(db.adminCommand({ + renameCollection: bucketsColl.getFullName(), + to: db.getName() + ".otherColl", + dropTarget: false +}), + ErrorCodes.IllegalOperation); +})(); diff --git a/jstests/core/timeseries/timeseries_collection_uuid.js b/jstests/core/timeseries/timeseries_collection_uuid.js deleted file mode 100644 index 2ba80cc8f16..00000000000 --- a/jstests/core/timeseries/timeseries_collection_uuid.js +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Tests using the collectionUUID parameter when operating on a time-series collection. - * - * @tags: [ - * does_not_support_transactions, - * requires_fcv_60, - * requires_timeseries, - * ] - */ - -(function() { -"use strict"; - -const dbName = jsTestName(); -const collName = "coll"; - -const testDB = db.getSiblingDB(dbName); -testDB.dropDatabase(); - -assert.commandWorked( - testDB.createCollection(collName, {timeseries: {timeField: "t", metaField: "m"}})); -const coll = testDB[collName]; -const bucketsColl = testDB["system.buckets." + collName]; - -const nonexistentUUID = UUID(); -const bucketsCollUUID = testDB.getCollectionInfos({name: bucketsColl.getName()})[0].info.uuid; - -const testInsert = function(uuid, ordered) { - assert.commandFailedWithCode(testDB.runCommand({ - insert: collName, - documents: [{t: ISODate()}], - collectionUUID: uuid, - ordered: ordered, - }), - ErrorCodes.CollectionUUIDMismatch); -}; - -testInsert(nonexistentUUID, true); -testInsert(nonexistentUUID, false); -testInsert(bucketsCollUUID, true); -testInsert(bucketsCollUUID, false); -}()); diff --git a/jstests/core/timeseries/timeseries_collmod.js b/jstests/core/timeseries/timeseries_collmod.js index 07c9166b632..69fd7dbeb82 100644 --- a/jstests/core/timeseries/timeseries_collmod.js +++ b/jstests/core/timeseries/timeseries_collmod.js @@ -19,17 +19,6 @@ assert.commandWorked( db.createCollection(collName, {timeseries: {timeField: "time", granularity: 'seconds'}})); assert.commandWorked(coll.createIndex({"time": 1})); -// Setting prepareUnique should return an error on a time-series collection. -assert.commandFailedWithCode( - db.runCommand( - {"collMod": collName, "index": {"keyPattern": {"time": 1}, "prepareUnique": true}}), - ErrorCodes.InvalidOptions); - -assert.commandFailedWithCode( - db.runCommand( - {"collMod": collName, "index": {"keyPattern": {"time": 1}, "prepareUnique": false}}), - ErrorCodes.InvalidOptions); - // Tries to convert a time-series secondary index to TTL index. assert.commandFailedWithCode( db.runCommand( @@ -67,4 +56,4 @@ assert.commandWorked(db.runCommand({"collMod": collName, "expireAfterSeconds": 6 // Successfully sets the granularity for a time-series collection. assert.commandWorked( db.runCommand({"collMod": collName, "timeseries": {"granularity": "minutes"}})); -})(); +})();
\ No newline at end of file diff --git a/jstests/core/timeseries/timeseries_computed_field.js b/jstests/core/timeseries/timeseries_computed_field.js deleted file mode 100644 index fab03d1a3b0..00000000000 --- a/jstests/core/timeseries/timeseries_computed_field.js +++ /dev/null @@ -1,856 +0,0 @@ -/** - * Test use of computed fields in aggregations on time series collections. - * @tags: [ - * requires_timeseries, - * does_not_support_stepdowns, - * directly_against_shardsvrs_incompatible, - * # "Explain of a resolved view must be executed by mongos" - * directly_against_shardsvrs_incompatible, - * # Some suites use mixed-binary cluster setup where some nodes might have the flag enabled while - * # others -- not. For this test we need control over whether the flag is set on the node that - * # ends up executing the query. - * assumes_standalone_mongod - * ] - */ -load("jstests/core/timeseries/libs/timeseries.js"); - -TimeseriesTest.run((insert) => { - const datePrefix = 1680912440; - - let coll = db.timeseries_computed_field; - const bucketsColl = db.getCollection('system.buckets.' + coll.getName()); - - const timeFieldName = 'time'; - const metaFieldName = 'measurement'; - - coll.drop(); - assert.commandWorked(db.createCollection(coll.getName(), { - timeseries: {timeField: timeFieldName, metaField: metaFieldName}, - })); - assert.contains(bucketsColl.getName(), db.getCollectionNames()); - - insert(coll, { - _id: 0, - [timeFieldName]: new Date(datePrefix + 100), // ISODate("1970-01-20T10:55:12.540Z") - [metaFieldName]: "cpu", - topLevelScalar: 123, - topLevelScalarDouble: 123.8778645, - topLevelLargeNumber: 12345678912345, - topLevelArray: [1, 2, 3, 4], - arrOfObj: [{x: 1}, {x: 2}, {x: 3}, {x: 4}], - obj: {a: 123}, - length: 0, - }); - insert(coll, { - _id: 1, - [timeFieldName]: new Date(datePrefix + 200), // ISODate("1970-01-20T10:55:12.640Z") - [metaFieldName]: "cpu", - topLevelScalar: 456, - topLevelScalarDouble: 546.76858699, - topLevelArray: [101, 102, 103, 104], - arrOfObj: [{x: 101}, {x: 102}, {x: 103}, {x: 104}], - obj: {a: 456}, - length: 23, - }); - // Insert a document that will be placed in a different bucket. - insert(coll, { - _id: 2, - [timeFieldName]: new Date(datePrefix + 300), - [metaFieldName]: "gpu", - length: -2, - }); - - // Computing a field on a dotted path which is an array, then grouping on it. Note that the - // semantics for setting a computed field on a dotted array path are particularly strange, but - // should be preserved for backwards compatibility. - { - const res = coll.aggregate([ - {$addFields: {"arrOfObj.x": {$trim: {input: "test string"}}}}, - {$match: {topLevelScalar: {$gte: 0}}}, - {$group: {_id: null, max: {$max: "$arrOfObj.x"}}} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].max, ["test string", "test string", "test string", "test string"], res); - } - - { - const res = coll.aggregate([ - {$addFields: {"arrOfObj.x": {$add: ["$topLevelScalar", 1]}}}, - {$match: {topLevelScalar: {$gte: 0}}}, - {$group: {_id: null, max: {$max: "$arrOfObj.x"}}} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].max, [457, 457, 457, 457], res); - } - - // Computing a field and then filtering by it. - { - const res = coll.aggregate([ - {$addFields: {"arrOfObj.x": {$trim: {input: "test string"}}}}, - {$match: {"arrOfObj.x": "test string"}}, - {$group: {_id: null, max: {$max: "$arrOfObj.x"}}} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].max, ["test string", "test string", "test string", "test string"], res); - } - - { - // Computing a field based on a dotted path which does not traverse arrays. - const res = coll.aggregate([ - {$addFields: {"computedA": {$add: ["$obj.a", 1]}}}, - // Only one document should have a value where obj.a was 457 - // (456 + 1). - {$match: {"computedA": 457}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - - { - // Include an unnecessary computed field before counting the number of documents. - const res = coll.aggregate([{$addFields: {"a": 1}}, {$count: "count"}]).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 3, res); - } - - // mathematical expressions - { - let pipeline = [ - {$addFields: {"computedA": {$add: ["$topLevelScalar", 1]}}}, - {$match: {"computedA": 457}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$subtract: ["$topLevelScalar", 1]}}}, - {$match: {"computedA": 455}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$multiply: ["$topLevelScalar", 10]}}}, - {$match: {"computedA": 4560}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$divide: ["$topLevelScalar", 2]}}}, - {$match: {"computedA": 228}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$add: [1, "$topLevelScalar"]}}}, - {$match: {"computedA": 457}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$subtract: [200, "$topLevelScalar"]}}}, - {$match: {"computedA": 77}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$multiply: [10, "$topLevelScalar"]}}}, - {$match: {"computedA": 4560}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$divide: [4560, "$topLevelScalar"]}}}, - {$match: {"computedA": 10}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - - { - let pipeline = [ - {$addFields: {"computedA": {$add: ["$topLevelScalar", "$topLevelScalar"]}}}, - {$match: {"computedA": 912}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$subtract: ["$topLevelScalar", "$topLevelScalar"]}}}, - {$match: {"computedA": 0}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$multiply: ["$topLevelScalar", "$topLevelScalar"]}}}, - {$match: {"computedA": 15129}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$divide: ["$topLevelScalar", "$topLevelScalar"]}}}, - {$match: {"computedA": 1}}, - {$count: "count"} - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - let pipeline = [ - { - $addFields: { - "hourDiff": { - $dateDiff: { - "startDate": "$time", - "endDate": new Date("1970-01-21"), - "unit": "hour" - } - } - } - }, - {$match: {"hourDiff": {$gte: 12}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "open": {"$first": "$topLevelScalar"}, - } - } - ]; - - assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "open": 123}], - coll.aggregate(pipeline).toArray()); - } - - { - let pipeline = [ - {$addFields: {"mt": {$multiply: ["$topLevelScalar", 10]}}}, - {$match: {"mt": {$gte: 2000}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "open": {"$first": "$topLevelScalar"}, - } - } - ]; - - assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "open": 456}], - coll.aggregate(pipeline).toArray()); - } - - { - let pipeline = [ - { - $addFields: { - "hourDiff": { - $dateDiff: { - "startDate": "$time", - "endDate": new Date("1970-01-21"), - "unit": "hour" - } - } - } - }, - {$match: {$or: [{topLevelScalar: {$gt: 200, $lt: 800}}, {hourDiff: {$gte: 12}}]}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "open": {"$first": "$topLevelScalar"}, - } - } - ]; - - assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "open": 123}], - coll.aggregate(pipeline).toArray()); - } - - { - // Try a project stage which adds and remove subfields. - const res = coll.aggregate([ - {$project: {"obj.newField": "$topLevelScalar"}}, - {$project: {"_id": 0, "obj.a": 0}} - ]) - .toArray(); - assert.eq(res.length, 3, res); - assert.eq(res[0], {"obj": {"newField": 123}}, res); - assert.eq(res[1], {"obj": {"newField": 456}}, res); - assert.eq(res[2], {"obj": {}}, res); - } - - { - // Try a replaceRoot stage which remove all fields. - const res = coll.aggregate([ - {$match: {"time": {$gte: new Date(datePrefix + 200)}}}, - {$addFields: {}}, - {$project: {"measurement0": {$floor: "$topLevelScalar"}}}, - {$replaceRoot: {newRoot: {}}} - ]) - .toArray(); - assert.eq(res.length, 2, res); - assert.eq(res[0], {}, res); - assert.eq(res[1], {}, res); - } - - { - // Try a $match that works on fields that are not projected. - const res = - coll.aggregate([ - {$project: {"obj.a": 1}}, - {$match: {$or: [{"topLevelScalar": {$gt: 10}}, {$expr: {$literal: true}}]}}, - {$sort: {_id: 1}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 3, res); - } - - { - const res = coll.aggregate([{ - "$project": { - "t": { - "$dateDiff": { - "startDate": "$time", - "endDate": new Date(datePrefix + 150), - "unit": "millisecond" - } - } - } - }]) - .toArray(); - assert.eq(res.length, 3, res); - assert.docEq(res[0], {_id: 0, "t": NumberLong(50)}, res); - assert.docEq(res[1], {_id: 1, "t": NumberLong(-50)}, res); - assert.docEq(res[2], {_id: 2, "t": NumberLong(-150)}, res); - } - - { - const res = coll.aggregate([{ - "$project": { - "t": { - "$dateDiff": { - "startDate": new Date(datePrefix - 10), - "endDate": "$time", - "unit": "millisecond" - } - } - } - }]) - .toArray(); - assert.eq(res.length, 3, res); - assert.docEq(res[0], {_id: 0, "t": NumberLong(110)}, res); - assert.docEq(res[1], {_id: 1, "t": NumberLong(210)}, res); - assert.docEq(res[2], {_id: 2, "t": NumberLong(310)}, res); - } - - { - const res = coll.aggregate([ - { - $addFields: { - "computedField": { - $dateTrunc: { - date: "$time", - unit: "second", - } - } - } - }, - {$match: {computedField: new Date(datePrefix - 440)}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 3, res); - } - - { - const res = coll.aggregate([{$match: {topLevelScalar: {$exists: true}}}, {$count: "count"}]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = - coll.aggregate([{$match: {topLevelScalar: {$lt: 456}}}, {$count: "count"}]).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - - { - const res = - coll.aggregate([{$match: {topLevelScalar: {$lte: 456}}}, {$count: "count"}]).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = - coll.aggregate([{$match: {topLevelScalar: {$gt: 123}}}, {$count: "count"}]).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - - { - const res = - coll.aggregate([{$match: {topLevelScalar: {$gte: 123}}}, {$count: "count"}]).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = - coll.aggregate([{$match: {topLevelScalar: {$eq: 123}}}, {$count: "count"}]).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - - { - const res = - coll.aggregate([{$match: {topLevelScalar: {$ne: 123}}}, {$count: "count"}]).toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = coll.aggregate([ - { - $match: { - $or: [ - {"topLevelScalar": 123}, - {"topLevelScalar": 456}, - ] - } - }, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = coll.aggregate([ - {$addFields: {"computedField": {$and: ["$topLevelScalar", true]}}}, - {$match: {computedField: true}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = coll.aggregate([ - {$addFields: {"computedField": {$and: ["$topLevelScalar", "$obj.a"]}}}, - {$match: {computedField: true}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = coll.aggregate([ - {$addFields: {"computedField": {$or: ["$topLevelScalar", false]}}}, - {$match: {computedField: true}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = coll.aggregate([ - {$addFields: {"computedField": {$or: ["$topLevelScalar", "$obj.a"]}}}, - {$match: {computedField: true}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = coll.aggregate([ - {$addFields: {"computedField": {$not: ["$topLevelScalar"]}}}, - {$match: {computedField: true}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - - { - const res = - coll.aggregate([ - {$addFields: {"computedField": {$anyElementTrue: [["$topLevelScalar"]]}}}, - {$match: {computedField: true}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 2, res); - } - - { - const res = - coll.aggregate([ - {$match: {length: {$ne: 0}}}, - {$set: {"computedA": {$multiply: ["$topLevelScalar", "$topLevelScalar"]}}}, - {$addFields: {"ratio": {$divide: ["$computedA", "$length"]}}}, - {$match: {ratio: {$gt: 0}}}, - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0]._id, 1, res); - } - - { - let pipeline = [ - {$addFields: {"computedA": {$round: ["$topLevelScalarDouble", 2]}}}, - {$match: {"computedA": {$gte: 100}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "value": {$sum: "$computedA"}, - } - } - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "value": 670.65}], res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$round: ["$topLevelScalarDouble"]}}}, - {$match: {"computedA": {$gte: 100}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "value": {$sum: "$computedA"}, - } - } - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "value": 671}], res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$round: [2, "$topLevelScalarDouble"]}}}, - {$match: {"computedA": {$gte: 100}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "value": {$sum: "$computedA"}, - } - } - ]; - - assert.throws(() => coll.aggregate(pipeline)); - } - { - let pipeline = [ - {$addFields: {"computedA": {$round: ["$topLevelScalarDouble", "$topLevelScalar"]}}}, - {$match: {"computedA": {$gte: 100}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "value": {$sum: "$computedA"}, - } - } - ]; - - assert.throws(() => coll.aggregate(pipeline)); - } - - { - let pipeline = [ - {$addFields: {"computedA": {$trunc: ["$topLevelScalarDouble", 2]}}}, - {$match: {"computedA": {$gte: 100}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "value": {$sum: "$computedA"}, - } - } - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "value": 670.63}], res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$trunc: ["$topLevelScalarDouble"]}}}, - {$match: {"computedA": {$gte: 100}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "value": {$sum: "$computedA"}, - } - } - ]; - - const res = coll.aggregate(pipeline).toArray(); - assert.docEq([{"_id": {"time": ISODate("1970-01-20T10:55:00Z")}, "value": 669}], res); - } - { - let pipeline = [ - {$addFields: {"computedA": {$trunc: [2, "$topLevelScalarDouble"]}}}, - {$match: {"computedA": {$gte: 100}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "value": {$sum: "$computedA"}, - } - } - ]; - - assert.throws(() => coll.aggregate(pipeline)); - } - { - let pipeline = [ - {$addFields: {"computedA": {$trunc: ["$topLevelScalarDouble", "$topLevelScalar"]}}}, - {$match: {"computedA": {$gte: 100}}}, - { - $group: { - "_id": {"time": {"$dateTrunc": {"date": "$time", "unit": "minute"}}}, - "value": {$sum: "$computedA"}, - } - } - ]; - - assert.throws(() => coll.aggregate(pipeline)); - } - - { - let pipeline = [ - {$addFields: {md: {$mod: ["$topLevelScalar", 5]}}}, - { - $group: { - "_id": {$dateTrunc: {"date": "$time", "unit": "minute"}}, - "total1": {$sum: "$md"}, - "total2": {$sum: "$topLevelScalar"} - } - } - ]; - - assert.docEq([{"_id": ISODate("1970-01-20T10:55:00Z"), "total1": 4, "total2": 579}], - coll.aggregate(pipeline).toArray()); - } - - { - let pipeline = [ - {$addFields: {md: {$mod: [500, "$topLevelScalar"]}}}, - { - $group: { - "_id": {$dateTrunc: {"date": "$time", "unit": "minute"}}, - "total1": {$sum: "$md"}, - "total2": {$sum: "$topLevelScalar"} - } - } - ]; - - assert.docEq([{"_id": ISODate("1970-01-20T10:55:00Z"), "total1": 52, "total2": 579}], - coll.aggregate(pipeline).toArray()); - } - - { - let pipeline = [ - {$addFields: {md: {$mod: ["$topLevelScalar", "$topLevelScalar"]}}}, - { - $group: { - "_id": {$dateTrunc: {"date": "$time", "unit": "minute"}}, - "total1": {$sum: "$md"}, - "total2": {$sum: "$topLevelScalar"} - } - } - ]; - - assert.docEq([{"_id": ISODate("1970-01-20T10:55:00Z"), "total1": 0, "total2": 579}], - coll.aggregate(pipeline).toArray()); - } - - { - let pipeline = [ - {$addFields: {md: {$mod: ["$topLevelScalar", 5]}}}, - {$match: {md: {$mod: [4, 1]}}}, - { - $group: { - "_id": {$dateTrunc: {"date": "$time", "unit": "minute"}}, - "total1": {$sum: "$md"}, - "total2": {$sum: "$topLevelScalar"} - } - } - ]; - - assert.docEq([{"_id": ISODate("1970-01-20T10:55:00Z"), "total1": 1, "total2": 456}], - coll.aggregate(pipeline).toArray()); - } - { - let pipeline = [{$match: {topLevelLargeNumber: {$mod: [1, 0]}}}, {$count: "count"}]; - - const res = coll.aggregate(pipeline).toArray(); - - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - { - const res = - coll.aggregate([ - { - $addFields: { - "computedField": - {$dateAdd: {startDate: "$time", unit: "millisecond", amount: 100}} - } - }, - {$match: {computedField: new Date(datePrefix + 400)}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - - { - const res = coll.aggregate([ - { - $addFields: { - "computedField": { - $dateSubtract: - {startDate: "$time", unit: "millisecond", amount: 100} - } - } - }, - {$match: {computedField: new Date(datePrefix)}}, - {$count: "count"} - ]) - .toArray(); - assert.eq(res.length, 1, res); - assert.eq(res[0].count, 1, res); - } - - { - // Test the case where a computed meta field is computed to missing. - const res = - coll.aggregate([{"$project": {"_id": 0, [metaFieldName]: "$$REMOVE"}}]).toArray(); - assert.eq(res.length, coll.count(), res); - for (let doc of res) { - assert.eq(doc, {}, res); - } - } - - { - // Test the case where a computed meta field is computed to missing because of a project - // out. - const res = coll.aggregate([ - {"$project": {[metaFieldName]: 0}}, - {"$project": {[metaFieldName]: "$" + metaFieldName}}, - {$match: {a: 1}} - ]) - .toArray(); - assert.eq(res.length, 0, res); - } - - { - // Test the case where a field is projected out and then projected back in. - const res = coll.aggregate([ - {"$project": {"_id": 0, [metaFieldName]: 1}}, - {"$project": {"_id": 0, [timeFieldName]: "$" + timeFieldName}}, - {"$project": {"_id": 1, [timeFieldName]: 1}} - ]) - .toArray(); - assert.eq(res.length, coll.count(), res); - for (let doc of res) { - assert.eq(doc, {}, res); - } - } - - { - // Test the case where a field is projected out and then projected back in. - const res = coll.aggregate([ - {"$project": {"_id": 0, [metaFieldName]: 1}}, - {"$project": {"_id": 0, [timeFieldName]: 1}} - ]) - .toArray(); - assert.eq(res.length, coll.count(), res); - for (let doc of res) { - assert.eq(doc, {}, res); - } - } - - { - // Test the case where a field is projected out and then projected back in. - const res = coll.aggregate([ - {"$project": {"_id": 0, [metaFieldName]: 0}}, - {"$project": {"_id": 0, [metaFieldName]: 1}}, - {"$project": {"_id": 0, [timeFieldName]: 1}} - ]) - .toArray(); - assert.eq(res.length, coll.count(), res); - for (let doc of res) { - assert.eq(doc, {}, res); - } - } -}); diff --git a/jstests/core/timeseries/timeseries_create_collection.js b/jstests/core/timeseries/timeseries_create_collection.js index 99616605163..feb2f443d26 100644 --- a/jstests/core/timeseries/timeseries_create_collection.js +++ b/jstests/core/timeseries/timeseries_create_collection.js @@ -17,15 +17,6 @@ assert.commandWorked(testDB.dropDatabase()); const timeFieldName = 'time'; const coll = testDB.t; -// Fails to create a time-series collection with null-embedded timeField or metaField. -assert.commandFailedWithCode( - testDB.createCollection(coll.getName(), {timeseries: {timeField: '\0time'}}), - ErrorCodes.BadValue); -assert.commandFailedWithCode( - testDB.createCollection(coll.getName(), - {timeseries: {timeField: timeFieldName, metaField: 't\0ag'}}), - ErrorCodes.BadValue); - // Create a timeseries collection, listCollection should show view and bucket collection assert.commandWorked( testDB.createCollection(coll.getName(), {timeseries: {timeField: timeFieldName}})); diff --git a/jstests/core/timeseries/timeseries_delete_hint.js b/jstests/core/timeseries/timeseries_delete_hint.js index 69fb2bc5a91..0facca816e1 100644 --- a/jstests/core/timeseries/timeseries_delete_hint.js +++ b/jstests/core/timeseries/timeseries_delete_hint.js @@ -18,11 +18,11 @@ (function() { "use strict"; +load("jstests/core/timeseries/libs/timeseries.js"); load("jstests/libs/curop_helpers.js"); -load("jstests/libs/feature_flag_util.js"); load('jstests/libs/parallel_shell_helpers.js'); -if (!FeatureFlagUtil.isEnabled(db, "TimeseriesUpdatesAndDeletes")) { +if (!TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(db.getMongo())) { jsTestLog("Skipping test because the time-series updates and deletes feature flag is disabled"); return; } @@ -71,7 +71,7 @@ const validateDeleteIndex = (docsToInsert, : assert.commandWorked( testDB.runCommand({delete: coll.getName(), deletes: deleteQuery})); assert.eq(res["n"], expectedNRemoved); - assert.sameMembers(coll.find({}, {_id: 0}).toArray(), expectedRemainingDocs); + assert.docEq(coll.find({}, {_id: 0}).toArray(), expectedRemainingDocs); assert(coll.drop()); }, docsToInsert, diff --git a/jstests/core/timeseries/timeseries_field_parsed_as_bson.js b/jstests/core/timeseries/timeseries_field_parsed_as_bson.js deleted file mode 100644 index 70210a90072..00000000000 --- a/jstests/core/timeseries/timeseries_field_parsed_as_bson.js +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Tests that timeseries timeField is parsed as bson. - * - * @tags: [ - * # We need a timeseries collection. - * requires_timeseries, - * # Cannot insert into a time-series collection in a multi-document transaction - * does_not_support_transactions, - * multiversion_incompatible - * ] - */ - -(function() { -'use strict'; - -const collName = "timeseries_field_parsed_as_bson"; -const coll = db.getCollection(collName); - -coll.drop(); -const timeField = "badInput']}}}}}}"; -assert.commandWorked(db.createCollection(collName, {timeseries: {timeField: timeField}})); - -const timeseriesCollInfo = db.getCollectionInfos({name: "system.buckets." + collName})[0]; -jsTestLog("Timeseries system collection info: " + tojson(timeseriesCollInfo)); -const properties = {}; -properties[timeField] = { - "bsonType": "date" -}; -const expectedValidator = { - "$jsonSchema": { - "bsonType": "object", - "required": ["_id", "control", "data"], - "properties": { - "_id": {"bsonType": "objectId"}, - "control": { - "bsonType": "object", - "required": ["version", "min", "max"], - "properties": { - "version": {"bsonType": "number"}, - "min": - {"bsonType": "object", "required": [timeField], "properties": properties}, - "max": - {"bsonType": "object", "required": [timeField], "properties": properties}, - "closed": {"bsonType": "bool"}, - "count": {"bsonType": "number", "minimum": 1} - }, - "additionalProperties": false - }, - "data": {"bsonType": "object"}, - "meta": {} - }, - "additionalProperties": false - } -}; - -assert(timeseriesCollInfo.options); -assert.eq(timeseriesCollInfo.options.validator, expectedValidator); - -const doc = { - a: 1, - [timeField]: new Date("2021-01-01") -}; -assert.commandWorked(coll.insert(doc)); -assert.docEq([doc], coll.aggregate([{$match: {}}, {$project: {_id: 0}}]).toArray()); - -coll.drop(); -assert.commandWorked(db.createCollection(collName, {timeseries: {timeField: "\\"}})); -coll.drop(); -assert.commandWorked(db.createCollection(collName, {timeseries: {timeField: "\\\\"}})); -})(); diff --git a/jstests/core/timeseries/timeseries_filter_extended_range.js b/jstests/core/timeseries/timeseries_filter_extended_range.js deleted file mode 100644 index 2b86e29c943..00000000000 --- a/jstests/core/timeseries/timeseries_filter_extended_range.js +++ /dev/null @@ -1,185 +0,0 @@ -/** - * Test that find/match type queries work properly on dates ouside the 32 bit epoch range, - * [1970-01-01 00:00:00 UTC - 2038-01-29 03:13:07 UTC]. - * - * @tags: [ - * # Refusing to run a test that issues an aggregation command with explain because it may - * # return incomplete results if interrupted by a stepdown. - * does_not_support_stepdowns, - * # We need a timeseries collection. - * requires_timeseries, - * # Cannot insert into a time-series collection in a multi-document transaction - * does_not_support_transactions, - * # Explain of a resolved view must be executed by mongos. - * directly_against_shardsvrs_incompatible, - * ] - */ - -(function() { -"use strict"; -const timeFieldName = "time"; - -/* - * Creates a collection, populates it, runs the `query` and ensures that the result set - * is equal to `results`. - * - * If overflow is set we create a document with dates above the 32 bit range (year 2040) - * If underflow is set, we create a document with dates below the 32 bit range (year 1965) - */ -function runTest(underflow, overflow, query, results) { - // Setup our DB & our collections. - const tsColl = db.getCollection(jsTestName()); - tsColl.drop(); - - assert.commandWorked( - db.createCollection(tsColl.getName(), {timeseries: {timeField: timeFieldName}})); - - const dates = [ - // If underflow, we want to insert a date that would fall below the epoch - // i.e. 1970-01-01 00:00:00 UTC. Otherwise we use a date within the epoch. - {[timeFieldName]: underflow ? new Date("1965-01-01") : new Date("1971-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - // If overflow, we want to insert a date that would use more than 32 bit milliseconds after - // the epoch. This overflow will occur 2038-01-29 03:13:07 UTC. Otherwise we go slightly - // before the end of the 32 bit epoch. - {[timeFieldName]: overflow ? new Date("2040-01-01") : new Date("2030-01-01")} - ]; - assert.commandWorked(tsColl.insert(dates)); - - // Make sure the expected results are in the correct order for comparison below. - function cmpTimeFields(a, b) { - return (b[timeFieldName].getTime() - a[timeFieldName].getTime()); - } - results.sort(cmpTimeFields); - - const pipeline = [{$match: query}, {$project: {_id: 0, [timeFieldName]: 1}}]; - - const plan = tsColl.explain().aggregate(pipeline); - - // Verify agg pipeline. We don't want to go through a plan that encourages a sort order to - // avoid BUS and index selection, so we sort after gathering the results. - const aggActuals = tsColl.aggregate(pipeline).toArray(); - aggActuals.sort(cmpTimeFields); - assert.docEq(results, aggActuals, JSON.stringify(plan, null, 4)); - - // Verify the equivalent find command. We again don't want to go through a plan that - // encourages a sort order to avoid BUS and index selection, so we sort after gathering the - // results. - let findActuals = tsColl.find(query, {_id: 0, [timeFieldName]: 1}).toArray(); - findActuals.sort(cmpTimeFields); - assert.docEq(findActuals, results); -} - -runTest(false, - false, - {[timeFieldName]: {$eq: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1980-01-01")}]); -runTest(false, - true, - {[timeFieldName]: {$eq: new Date("2040-01-01")}}, - [{[timeFieldName]: new Date("2040-01-01")}]); -runTest(true, - false, - {[timeFieldName]: {$eq: new Date("1965-01-01")}}, - [{[timeFieldName]: new Date("1965-01-01")}]); - -runTest(false, - false, - {[timeFieldName]: {$lt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1971-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); -runTest(false, - true, - {[timeFieldName]: {$lt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1971-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); -runTest(true, - false, - {[timeFieldName]: {$lt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1965-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); -runTest(true, - true, - {[timeFieldName]: {$lt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1965-01-01")}, {[timeFieldName]: new Date("1975-01-01")}]); - -runTest(false, - false, - {[timeFieldName]: {$gt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2030-01-01")}]); -runTest(false, - true, - {[timeFieldName]: {$gt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2040-01-01")}]); -runTest(true, - false, - {[timeFieldName]: {$gt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2030-01-01")}]); -runTest(true, - true, - {[timeFieldName]: {$gt: new Date("1980-01-01")}}, - [{[timeFieldName]: new Date("1995-01-01")}, {[timeFieldName]: new Date("2040-01-01")}]); - -runTest(false, false, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1971-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")} -]); -runTest(false, true, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1971-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")} -]); -runTest(true, false, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1965-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")} -]); -runTest(true, true, {[timeFieldName]: {$lte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1965-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - {[timeFieldName]: new Date("1980-01-01")} -]); - -runTest(false, false, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2030-01-01")} -]); -runTest(false, true, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2040-01-01")} -]); -runTest(true, false, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2030-01-01")} -]); -runTest(true, true, {[timeFieldName]: {$gte: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1980-01-01")}, - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2040-01-01")} -]); - -// Verify ranges that straddle the lower and upper epoch boundaries work properly. -runTest( - true, false, {[timeFieldName]: {$gt: new Date("1920-01-01"), $lt: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1965-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - ]); -runTest( - false, true, {[timeFieldName]: {$gt: new Date("1980-01-01"), $lt: new Date("2050-01-01")}}, [ - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2040-01-01")}, - ]); -runTest( - false, false, {[timeFieldName]: {$gt: new Date("1920-01-01"), $lt: new Date("1980-01-01")}}, [ - {[timeFieldName]: new Date("1971-01-01")}, - {[timeFieldName]: new Date("1975-01-01")}, - ]); -runTest( - false, false, {[timeFieldName]: {$gt: new Date("1980-01-01"), $lt: new Date("2050-01-01")}}, [ - {[timeFieldName]: new Date("1995-01-01")}, - {[timeFieldName]: new Date("2030-01-01")}, - ]); -})(); diff --git a/jstests/core/timeseries/timeseries_geonear_lookup.js b/jstests/core/timeseries/timeseries_geonear_lookup.js deleted file mode 100644 index 78f68eb014e..00000000000 --- a/jstests/core/timeseries/timeseries_geonear_lookup.js +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Test that time-series works as expected with $geoNear within a $lookup. - * - * @tags: [ - * does_not_support_transactions, - * requires_pipeline_optimization, - * # We need a timeseries collection. - * requires_timeseries, - * references_foreign_collection, - * ] - */ - -// create a timeseries collection -const timeFieldName = "time"; -const metaFieldName = "tags"; -const testDB = db; - -const tsColl = db.getCollection("ts_coll"); - -const kMaxDistance = Math.PI * 2.0; - -tsColl.drop(); -assert.commandWorked(testDB.createCollection( - tsColl.getName(), {timeseries: {timeField: timeFieldName, metaField: metaFieldName}})); - -assert.commandWorked(tsColl.createIndex({'tags.loc': '2dsphere'})); - -tsColl.insert({time: ISODate(), tags: {loc: [40, 40], descr: 0}, value: 0}); - -const coll2 = db.getCollection("store_min_max_values"); -coll2.drop(); -assert.commandWorked(coll2.insert({_id: 0, minimumDist: 0.0, maximumDist: kMaxDistance})); - -coll2.aggregate([{$lookup: {from: tsColl.getName(), -let: {minVal: "$minimumDist",maxVal:"$maximumDist"}, -pipeline: [ - {$geoNear: {near: {type: "Point", coordinates: [0, 0]}, - key: 'tags.loc', - distanceField: "tags.distance"}}], - as: 'output'}}]); diff --git a/jstests/core/timeseries/timeseries_geonear_measurements.js b/jstests/core/timeseries/timeseries_geonear_measurements.js index abbf7db9596..5c5427a02d1 100644 --- a/jstests/core/timeseries/timeseries_geonear_measurements.js +++ b/jstests/core/timeseries/timeseries_geonear_measurements.js @@ -23,10 +23,10 @@ (function() { "use strict"; +load("jstests/core/timeseries/libs/timeseries.js"); load("jstests/libs/analyze_plan.js"); -load("jstests/libs/feature_flag_util.js"); -if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { +if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; diff --git a/jstests/core/timeseries/timeseries_graph_lookup.js b/jstests/core/timeseries/timeseries_graph_lookup.js index 25f47aa2cef..7b152cc6b1f 100644 --- a/jstests/core/timeseries/timeseries_graph_lookup.js +++ b/jstests/core/timeseries/timeseries_graph_lookup.js @@ -5,7 +5,6 @@ * does_not_support_transactions, * requires_timeseries, * requires_fcv_51, - * references_foreign_collection, * ] */ (function() { diff --git a/jstests/core/timeseries/timeseries_groupby_reorder.js b/jstests/core/timeseries/timeseries_groupby_reorder.js deleted file mode 100644 index 774d867d3f6..00000000000 --- a/jstests/core/timeseries/timeseries_groupby_reorder.js +++ /dev/null @@ -1,233 +0,0 @@ -/** - * Test the behavior of $group on time-series collections. Specifically, we are targeting rewrites - * that replace bucket unpacking with $group over the buckets collection. Currently, only $min/$max - * are supported for the rewrites. - * - * @tags: [ - * directly_against_shardsvrs_incompatible, - * does_not_support_stepdowns, - * does_not_support_transactions, - * requires_fcv_60, - * ] - */ -(function() { -"use strict"; - -load("jstests/libs/fixture_helpers.js"); -load("jstests/core/timeseries/libs/timeseries.js"); - -const coll = db.timeseries_groupby_reorder; -coll.drop(); - -// We will only check correctness of the results here as checking the plan in JSTests is brittle and -// is better done in document_source_internal_unpack_bucket_test/group_reorder_test.cpp. For the -// cases when the re-write isn't applicable, the used datasets should yield wrong result if the -// re-write is applied. -function runGroupRewriteTest(docs, pipeline, expectedResults) { - coll.drop(); - db.createCollection(coll.getName(), {timeseries: {metaField: "meta", timeField: "time"}}); - coll.insertMany(docs); - assert.docEq(expectedResults, coll.aggregate(pipeline).toArray(), () => { - return `Pipeline: ${tojson(pipeline)}. Explain: ${ - tojson(coll.explain().aggregate(pipeline))}`; - }); -} - -// Test with measurement group key -- a rewrite in this situation would be wrong. -(function testNonMetaGroupKey() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, key: 2, val: 1}, // global min - {time: t, meta: 1, key: 1, val: 3}, // min for key = 1 - {time: t, meta: 1, key: 1, val: 5}, // max for key = 1 - {time: t, meta: 1, key: 2, val: 7}, // global max - ]; - runGroupRewriteTest(docs, - [{$group: {_id: "$key", min: {$min: "$val"}}}, {$match: {_id: 1}}], - [{"_id": 1, "min": 3}]); - runGroupRewriteTest(docs, - [{$group: {_id: "$key", max: {$max: "$val"}}}, {$match: {_id: 1}}], - [{"_id": 1, "max": 5}]); -})(); - -// While a group with const group key can be re-written in terms of a group on the buckets, we don't -// currently do it. However, when/if we start doing it, it should work. -(function testConstGroupKey_NoFilter() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, val: 0}, - {time: t, meta: 1, val: 1}, - {time: t, meta: 1, val: 5}, - ]; - runGroupRewriteTest( - docs, [{$group: {_id: null, min: {$min: "$val"}}}], [{"_id": null, "min": 0}]); - runGroupRewriteTest( - docs, [{$group: {_id: null, max: {$max: "$val"}}}], [{"_id": null, "max": 5}]); -})(); - -// With a filter on meta the group re-write would still apply if the group key is const. -(function testConstGroupKey_WithFilterOnMeta() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, val: 0}, - {time: t, meta: 2, val: 1}, - {time: t, meta: 2, val: 3}, - {time: t, meta: 1, val: 5}, - ]; - runGroupRewriteTest(docs, - [{$match: {meta: 2}}, {$group: {_id: null, min: {$min: "$val"}}}], - [{"_id": null, "min": 1}]); - runGroupRewriteTest(docs, - [{$match: {meta: 2}}, {$group: {_id: null, max: {$max: "$val"}}}], - [{"_id": null, "max": 3}]); -})(); - -// In presense of a non-meta filter the group re-write doesn't apply even if the group key is const. -(function testConstGroupKey_WithFilterOnMeasurement() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, val: 0, include: false}, - {time: t, meta: 1, val: 1, include: true}, - {time: t, meta: 1, val: 5, include: false}, - ]; - runGroupRewriteTest(docs, - [{$match: {include: true}}, {$group: {_id: null, min: {$min: "$val"}}}], - [{"_id": null, "min": 1}]); - runGroupRewriteTest(docs, - [{$match: {include: true}}, {$group: {_id: null, max: {$max: "$val"}}}], - [{"_id": null, "max": 1}]); -})(); - -// Test with meta group key. The group re-write applies. -(function testMetaGroupKey_NoFilter() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, val: 5}, - {time: t, meta: 2, val: 4}, - {time: t, meta: 2, val: 3}, - {time: t, meta: 1, val: 1}, - ]; - runGroupRewriteTest(docs, - [{$group: {_id: "$meta", min: {$min: "$val"}}}, {$match: {_id: 2}}], - [{"_id": 2, "min": 3}]); - runGroupRewriteTest(docs, - [{$group: {_id: "$meta", max: {$max: "$val"}}}, {$match: {_id: 2}}], - [{"_id": 2, "max": 4}]); -})(); - -// Test with meta group key preceeded by a filter on the meta key. The re-write still applies. -(function testMetaGroupKey_WithFilterOnMeta() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, val: 5}, - {time: t, meta: 2, val: 4}, - {time: t, meta: 2, val: 3}, - {time: t, meta: 1, val: 1}, - ]; - runGroupRewriteTest(docs, - [{$match: {meta: 2}}, {$group: {_id: "$meta", min: {$min: "$val"}}}], - [{"_id": 2, "min": 3}]); - runGroupRewriteTest(docs, - [{$match: {meta: 2}}, {$group: {_id: "$meta", max: {$max: "$val"}}}], - [{"_id": 2, "max": 4}]); -})(); - -// Test with meta group key preceeded by a filter on a measurement key. The re-write doesn't apply. -(function testMetaGroupKey_WithFilterOnMeasurement() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, val: 3, include: false}, - {time: t, meta: 1, val: 4, include: true}, - {time: t, meta: 1, val: 5, include: false}, - ]; - runGroupRewriteTest(docs, - [{$match: {include: true}}, {$group: {_id: "$meta", min: {$min: "$val"}}}], - [{"_id": 1, "min": 4}]); - runGroupRewriteTest(docs, - [{$match: {include: true}}, {$group: {_id: "$meta", max: {$max: "$val"}}}], - [{"_id": 1, "max": 4}]); -})(); - -// Test SERVER-73822 fix: complex $min and $max (i.e. not just straight field refs) work correctly. -(function testMetaGroupKey_WithNonPathExpressionUnderMinMax() { - const t = new Date(); - // min(a+b) != min(a) + min(b) and max(a+b) != max(a) + max(b) - const docs = [ - {time: t, meta: 1, a: 1, b: 20}, // max(a + b) - {time: t, meta: 1, a: 2, b: 10}, - {time: t, meta: 1, a: 3, b: 1}, // min(a + b) - ]; - runGroupRewriteTest(docs, - [{$group: {_id: "$meta", min: {$min: {$add: ["$a", "$b"]}}}}], - [{"_id": 1, "min": 4}]); - runGroupRewriteTest(docs, - [{$group: {_id: "$meta", max: {$max: {$add: ["$a", "$b"]}}}}], - [{"_id": 1, "max": 21}]); -})(); - -// Test with meta group key and a non-min/max accumulator that doesn't use any fields. The buckets -// still have to be unpacked because we don't know the number of events in uncompressed buckets. -(function testMetaGroupKey_WithAccumulatorNotUsingAnyFields() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, val: 3}, - {time: t, meta: 3, val: 4}, - {time: t, meta: 1, val: 5}, - ]; - runGroupRewriteTest( - docs, [{$group: {_id: "$meta", x: {$sum: 1}}}, {$match: {_id: 1}}], [{"_id": 1, "x": 2}]); -})(); - -// Test with meta group key and a non-min/max accumulator that uses only the meta field. This query -// is _not_ eligible for the re-write w/o bucket unpacking because while it doesn't depend on any -// fields of the individual events it still depends on the number of events in a bucket. -(function testMetaGroupKey_WithNonMinMaxAccumulatorOnMeta() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, val: 3}, - {time: t, meta: 3, val: 4}, - {time: t, meta: 1, val: 5}, - ]; - runGroupRewriteTest(docs, - [{$group: {_id: "$meta", x: {$sum: "$meta"}}}, {$match: {_id: 1}}], - [{"_id": 1, "x": 2}]); -})(); - -// In presence of a filter $min and $max on the meta field cannot be re-written because a filter -// might end up selecting nothing in buckets with a particular meta. -(function testMetaGroupKey_WithAccumulatorOnMeta_WithFilterOnMeasurement() { - const t = new Date(); - const docs = [ - {time: t, meta: 1, include: false}, - ]; - runGroupRewriteTest( - docs, [{$match: {include: true}}, {$group: {_id: "$meta", x: {$min: "$meta"}}}], []); -})(); - -// Test min/max on the time field (cannot rewrite $min because the control.time.min is rounded -// down). -(function testMetaGroupKey_WithMinMaxOnTime() { - const docs = [ - {time: ISODate("2023-07-20T23:16:47.683Z"), meta: 1}, - ]; - runGroupRewriteTest(docs, - [{$group: {_id: "$meta", min: {$min: "$time"}}}], - [{_id: 1, min: ISODate("2023-07-20T23:16:47.683Z")}]); -})(); - -// Test a group key which is an object with single field path. Ensure that the output perseveres the -// user requested object structure for _id field. -(function testMetaGroupKey_ObjectWithSingleFieldPathElement() { - const t = new Date(); - const docs = [ - {time: t, meta: {a: 1}, val: 5}, - {time: t, meta: {a: 2}, val: 4}, - {time: t, meta: {a: 2}, val: 3}, - {time: t, meta: {a: 1}, val: 1}, - ]; - runGroupRewriteTest( - docs, - [{$group: {_id: {d: "$meta.a"}, min: {$min: "$val"}}}, {$match: {_id: {d: 2}}}], - [{"_id": {d: 2}, "min": 3}]); -})(); -})(); diff --git a/jstests/core/timeseries/timeseries_index.js b/jstests/core/timeseries/timeseries_index.js index 9547362d105..bef86453999 100644 --- a/jstests/core/timeseries/timeseries_index.js +++ b/jstests/core/timeseries/timeseries_index.js @@ -11,9 +11,8 @@ (function() { "use strict"; -load("jstests/core/timeseries/libs/timeseries.js"); -load("jstests/libs/feature_flag_util.js"); load("jstests/libs/fixture_helpers.js"); +load("jstests/core/timeseries/libs/timeseries.js"); TimeseriesTest.run((insert) => { const collNamePrefix = 'timeseries_index_'; @@ -226,7 +225,7 @@ TimeseriesTest.run((insert) => { runTest({[metaFieldName + '.location']: "2d", [metaFieldName + '.tag1']: -1}, {'meta.location': "2d", 'meta.tag1': -1}); - if (FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { + if (TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { // Measurement 2dsphere index runTest({'loc': '2dsphere'}, {'data.loc': '2dsphere_bucket'}); } @@ -243,7 +242,7 @@ TimeseriesTest.run((insert) => { coll.getName(), {timeseries: {timeField: timeFieldName, metaField: metaFieldName}})); assert.commandWorked(insert(coll, doc), 'failed to insert doc: ' + tojson(doc)); - if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { + if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { // Reject index keys that do not include the metadata field. assert.commandFailedWithCode(coll.createIndex({not_metadata: 1}), ErrorCodes.CannotCreateIndex); @@ -266,7 +265,7 @@ TimeseriesTest.run((insert) => { [ErrorCodes.CannotCreateIndex, ErrorCodes.InvalidOptions]); }; - if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { + if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { // Partial indexes are not supported on time-series collections if the time-series metric // feature flag is disabled. testCreateIndexFailed({[metaFieldName]: 1}, {partialFilterExpression: {meta: {$gt: 5}}}); diff --git a/jstests/core/timeseries/timeseries_index_partial.js b/jstests/core/timeseries/timeseries_index_partial.js index 014158c3542..eb2512b1bd7 100644 --- a/jstests/core/timeseries/timeseries_index_partial.js +++ b/jstests/core/timeseries/timeseries_index_partial.js @@ -2,8 +2,6 @@ * Test creating and using partial indexes, on a time-series collection. * * @tags: [ - * # TODO (SERVER-73316): remove - * assumes_against_mongod_not_mongos, * does_not_support_stepdowns, * does_not_support_transactions, * requires_fcv_52, @@ -14,10 +12,10 @@ (function() { "use strict"; +load("jstests/core/timeseries/libs/timeseries.js"); load("jstests/libs/analyze_plan.js"); -load("jstests/libs/feature_flag_util.js"); -if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { +if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; @@ -80,24 +78,10 @@ assert.commandFailedWithCode(coll.createIndex({a: 1}, {partialFilterExpression: // Test creating and using a partial index. { - let ixscanInWinningPlan = 0; - - // Make sure the {a: 1} index was considered for this query. + // Make sure the query uses the {a: 1} index. function checkPlan(predicate) { const explain = coll.find(predicate).explain(); - let scan = getAggPlanStage(explain, 'IXSCAN'); - // If scan is not present, check rejected plans - if (scan === null) { - const rejectedPlans = getRejectedPlans(getAggPlanStage(explain, "$cursor")["$cursor"]); - if (rejectedPlans.length === 1) { - const scans = getPlanStages(getRejectedPlan(rejectedPlans[0]), "IXSCAN"); - if (scans.length === 1) { - scan = scans[0]; - } - } - } else { - ixscanInWinningPlan++; - } + const scan = getAggPlanStage(explain, 'IXSCAN'); const indexes = buckets.getIndexes(); assert(scan, "Expected an index scan for predicate: " + tojson(predicate) + @@ -106,10 +90,10 @@ assert.commandFailedWithCode(coll.createIndex({a: 1}, {partialFilterExpression: } // Make sure the query results match a collection-scan plan. function checkResults(predicate) { - const result = coll.aggregate([{$match: predicate}], {hint: {a: 1}}).toArray(); + const result = coll.aggregate({$match: predicate}).toArray(); const unindexed = coll.aggregate([{$_internalInhibitOptimization: {}}, {$match: predicate}]).toArray(); - assert.sameMembers(result, unindexed); + assert.docEq(result, unindexed); } function checkPlanAndResults(predicate) { checkPlan(predicate); @@ -149,6 +133,10 @@ assert.commandFailedWithCode(coll.createIndex({a: 1}, {partialFilterExpression: const t1 = ISODate('2000-01-01T00:00:01Z'); const t2 = ISODate('2000-01-01T00:00:02Z'); + // When the collection is sharded, there is an index on time that can win, instead of the + // partial index. So only check the results in that case, not the plan. + const check = FixtureHelpers.isSharded(buckets) ? checkResults : checkPlanAndResults; + assert.commandWorked(coll.dropIndex({a: 1})); assert.commandWorked( coll.createIndex({a: 1}, {partialFilterExpression: {[timeField]: {$lt: t1}}})); @@ -177,7 +165,6 @@ assert.commandFailedWithCode(coll.createIndex({a: 1}, {partialFilterExpression: assert.commandWorked(coll.dropIndex({a: 1})); assert.sameMembers(coll.getIndexes(), extraIndexes); assert.sameMembers(buckets.getIndexes(), extraBucketIndexes); - assert.gt(ixscanInWinningPlan, 0); } // Check that partialFilterExpression can use a mixture of metadata, time, and measurement fields, diff --git a/jstests/core/timeseries/timeseries_index_spec.js b/jstests/core/timeseries/timeseries_index_spec.js index d3822dc642b..f13c64dd80b 100644 --- a/jstests/core/timeseries/timeseries_index_spec.js +++ b/jstests/core/timeseries/timeseries_index_spec.js @@ -16,7 +16,6 @@ "use strict"; load("jstests/core/timeseries/libs/timeseries.js"); -load("jstests/libs/feature_flag_util.js"); TimeseriesTest.run(() => { const collName = "timeseries_index_spec"; @@ -86,7 +85,7 @@ TimeseriesTest.run(() => { {name: "time_meta_field_downgradable"})); verifyAndDropIndex(/*isDowngradeCompatible=*/true, "time_meta_field_downgradable"); - if (FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { + if (TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { assert.commandWorked(coll.createIndex({x: 1}, {name: "x_1"})); verifyAndDropIndex(/*isDowngradeCompatible=*/false, "x_1"); @@ -111,7 +110,7 @@ TimeseriesTest.run(() => { // Creating an index directly on the buckets collection is permitted. However, these types of // index creations will not have an "originalSpec" field and rely on the reverse mapping // mechanism. - if (FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { + if (TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { assert.commandWorked( bucketsColl.createIndex({"control.min.y": 1, "control.max.y": 1}, {name: "y"})); diff --git a/jstests/core/timeseries/timeseries_index_use.js b/jstests/core/timeseries/timeseries_index_use.js index 6ce128ecbbd..8770047f4d2 100644 --- a/jstests/core/timeseries/timeseries_index_use.js +++ b/jstests/core/timeseries/timeseries_index_use.js @@ -50,9 +50,8 @@ const generateTest = (useHint) => { /** * Creates the index specified by the spec and options, then explains the query to ensure - * that the created index is used or was considered by multi-planner. - * Runs the query and verifies that the expected number of documents are matched. - * Finally, deletes the created index. + * that the created index is used. Runs the query and verifies that the expected number of + * documents are matched. Finally, deletes the created index. */ const testQueryUsesIndex = function( filter, numMatches, indexSpec, indexOpts = {}, queryOpts = {}) { @@ -68,23 +67,9 @@ const generateTest = (useHint) => { assert.eq(numMatches, query.itcount()); const explain = query.explain(); - if (useHint) { - const ixscan = getAggPlanStage(explain, "IXSCAN"); - assert.neq(null, ixscan, tojson(explain)); - assert.eq("testIndexName", ixscan.indexName, tojson(ixscan)); - } else { - let ixscan = getAggPlanStage(explain, "IXSCAN"); - // If ixscan is not present, check rejected plans - if (ixscan === null) { - const rejectedPlans = - getRejectedPlans(getAggPlanStage(explain, "$cursor")["$cursor"]); - assert.eq(1, rejectedPlans.length); - const ixscans = getPlanStages(getRejectedPlan(rejectedPlans[0]), "IXSCAN"); - assert.eq(1, ixscans.length); - ixscan = ixscans[0]; - } - assert.eq("testIndexName", ixscan.indexName, tojson(ixscan)); - } + const ixscan = getAggPlanStage(explain, "IXSCAN"); + assert.neq(null, ixscan, tojson(explain)); + assert.eq("testIndexName", ixscan.indexName, tojson(ixscan)); assert.commandWorked(coll.dropIndex("testIndexName")); }; diff --git a/jstests/core/timeseries/timeseries_insert_mixed_schema_bucket.js b/jstests/core/timeseries/timeseries_insert_mixed_schema_bucket.js deleted file mode 100644 index cbce31e9795..00000000000 --- a/jstests/core/timeseries/timeseries_insert_mixed_schema_bucket.js +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Tests directly inserting a time-series bucket with mixed schema. - * - * @tags: [ - * # $listCatalog does not include the tenant prefix in its results. - * command_not_supported_in_serverless, - * requires_timeseries, - * # $listCatalog not supported inside of a multi-document transaction - * does_not_support_transactions, - * # $listCatalog only exists since v6 - * multiversion_incompatible, - * ] - */ -(function() { -"use strict"; - -load("jstests/core/timeseries/libs/timeseries.js"); // For 'TimeseriesTest'. - -TestData.skipEnforceTimeseriesBucketsAreAlwaysCompressedOnValidate = true; - -const testDB = db.getSiblingDB(jsTestName()); -const collName = "ts"; - -assert.commandWorkedOrFailedWithCode(testDB.runCommand({drop: collName}), - ErrorCodes.NamespaceNotFound); -assert.commandWorked( - testDB.createCollection(collName, {timeseries: {timeField: "t", metaField: "m"}})); -const coll = testDB[collName]; -const bucketsColl = testDB["system.buckets." + collName]; - -const bucket = { - _id: ObjectId("65a6eb806ffc9fa4280ecac4"), - control: { - version: NumberInt(1), - min: { - _id: ObjectId("65a6eba7e6d2e848e08c3750"), - t: ISODate("2024-01-16T20:48:00Z"), - a: 1, - }, - max: { - _id: ObjectId("65a6eba7e6d2e848e08c3751"), - t: ISODate("2024-01-16T20:48:39.448Z"), - a: "a", - }, - }, - meta: 0, - data: { - _id: { - 0: ObjectId("65a6eba7e6d2e848e08c3750"), - 1: ObjectId("65a6eba7e6d2e848e08c3751"), - }, - t: { - 0: ISODate("2024-01-16T20:48:39.448Z"), - 1: ISODate("2024-01-16T20:48:39.448Z"), - }, - a: { - 0: "a", - 1: 1, - }, - } -}; - -assert.commandFailedWithCode(bucketsColl.insert(bucket), - ErrorCodes.CannotInsertTimeseriesBucketsWithMixedSchema); -assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), false); -assert.commandWorked( - testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: true})); -assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), true); -assert.commandWorked(bucketsColl.insert(bucket)); -assert.commandWorked(bucketsColl.deleteOne({_id: bucket._id})); -assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), true); -assert.commandWorked( - testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: false})); -assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), false); -})(); diff --git a/jstests/core/timeseries/timeseries_internal_bucket_geo_within.js b/jstests/core/timeseries/timeseries_internal_bucket_geo_within.js index 2e350ebdbb0..9c237d8dd77 100644 --- a/jstests/core/timeseries/timeseries_internal_bucket_geo_within.js +++ b/jstests/core/timeseries/timeseries_internal_bucket_geo_within.js @@ -4,7 +4,7 @@ * collection. * * @tags: [ - * requires_fcv_60, + * requires_fcv_51, * requires_pipeline_optimization, * requires_timeseries, * does_not_support_stepdowns, @@ -47,6 +47,10 @@ for (let collScanStage of collScanStages) { "field": "loc" } }; + // TODO SERVER-60373 Fix duplicate predicates for sharded time-series collection + if (FixtureHelpers.isSharded(bucketsColl)) { + expectedPredicate = {$and: [expectedPredicate, expectedPredicate]}; + } assert.docEq(expectedPredicate, collScanStage.filter, collScanStages); } diff --git a/jstests/core/timeseries/timeseries_lastpoint_top.js b/jstests/core/timeseries/timeseries_lastpoint_top.js index c85f48fb076..2efd352b574 100644 --- a/jstests/core/timeseries/timeseries_lastpoint_top.js +++ b/jstests/core/timeseries/timeseries_lastpoint_top.js @@ -27,7 +27,6 @@ assert.commandWorked(testDB.dropDatabase()); // Do not run the rest of the tests if the lastpoint optimization is disabled. if (!FeatureFlagUtil.isEnabled(db, "LastPointQuery")) { - jsTestLog("Skipping the test."); return; } diff --git a/jstests/core/timeseries/timeseries_lookup.js b/jstests/core/timeseries/timeseries_lookup.js index 6d559e9356d..c173764a591 100644 --- a/jstests/core/timeseries/timeseries_lookup.js +++ b/jstests/core/timeseries/timeseries_lookup.js @@ -5,13 +5,11 @@ * does_not_support_transactions, * requires_timeseries, * requires_fcv_51, - * references_foreign_collection, * ] */ (function() { "use strict"; -load("jstests/aggregation/extras/utils.js"); load("jstests/core/timeseries/libs/timeseries.js"); TimeseriesTest.run((insert) => { @@ -26,44 +24,43 @@ TimeseriesTest.run((insert) => { Random.setRandomSeed(); const hosts = TimeseriesTest.generateHosts(numHosts); - let - testFunc = function(collAOption, collBOption) { - // Prepare two time-series collections. - const collA = testDB.getCollection("a"); - const collB = testDB.getCollection("b"); - collA.drop(); - collB.drop(); - assert.commandWorked(testDB.createCollection(collA.getName(), collAOption)); - assert.commandWorked(testDB.createCollection(collB.getName(), collBOption)); - - let entryCountPerHost = new Array(numHosts).fill(0); - - // Insert into collA, one entry per host. - for (let i = 0; i < numHosts; i++) { - let host = hosts[i]; - assert.commandWorked(insert(collA, { - measurement: "cpu", - time: ISODate(), - tags: host.tags, - })); - } - - // Insert some random documents to collB. - for (let i = 0; i < numDocs; i++) { - let host = TimeseriesTest.getRandomElem(hosts); - assert.commandWorked(insert(collB, { - measurement: "cpu", - time: ISODate(), - tags: host.tags, - })); - // Here we extract the hostId from "host.tags.hostname". It is expected that the - // "host.tags.hostname" is in the form of 'host_<hostNum>'. - entryCountPerHost[parseInt( - host.tags.hostname.substring(5, host.tags.hostname.length))]++; - } - - // Equality Match - let results = collA.aggregate([ + let testFunc = function(collAOption, collBOption) { + // Prepare two time-series collections. + const collA = testDB.getCollection("a"); + const collB = testDB.getCollection("b"); + collA.drop(); + collB.drop(); + assert.commandWorked(testDB.createCollection(collA.getName(), collAOption)); + assert.commandWorked(testDB.createCollection(collB.getName(), collBOption)); + + let entryCountPerHost = new Array(numHosts).fill(0); + + // Insert into collA, one entry per host. + for (let i = 0; i < numHosts; i++) { + let host = hosts[i]; + assert.commandWorked(insert(collA, { + measurement: "cpu", + time: ISODate(), + tags: host.tags, + })); + } + + // Insert some random documents to collB. + for (let i = 0; i < numDocs; i++) { + let host = TimeseriesTest.getRandomElem(hosts); + assert.commandWorked(insert(collB, { + measurement: "cpu", + time: ISODate(), + tags: host.tags, + })); + // Here we extract the hostId from "host.tags.hostname". It is expected that the + // "host.tags.hostname" is in the form of 'host_<hostNum>'. + entryCountPerHost[parseInt( + host.tags.hostname.substring(5, host.tags.hostname.length))]++; + } + + // Equality Match + let results = collA.aggregate([ { $lookup: { from: collB.getName(), @@ -82,106 +79,13 @@ TimeseriesTest.run((insert) => { }, {$sort: {host: 1}} ]).toArray(); - assert.eq(numHosts, results.length, results); - for (let i = 0; i < numHosts; i++) { - assert.eq({host: "host_" + i, matchedB: entryCountPerHost[i]}, results[i], results); - } + assert.eq(numHosts, results.length, results); + for (let i = 0; i < numHosts; i++) { + assert.eq({host: "host_" + i, matchedB: entryCountPerHost[i]}, results[i], results); + } - // Equality Match With Let (uncorrelated) - // Make sure injected $sequentialDocumentCache (right after unpack bucket stage) - // in the inner pipeline is removed. - results = collA.aggregate([ - { - $lookup: { - from: collB.getName(), - let: {"outer_hostname": "$tags.hostname"}, - pipeline: [ - // $match will be pushed before unpack bucket stage - {$match: {$expr: {$eq: ["$$outer_hostname", hosts[0].tags.hostname]}}}, - ], - as: "matchedB" - } - }, { - $project: { - _id: 0, - host: "$tags.hostname", - matchedB: { - $size: "$matchedB" - } - } - }, - {$sort: {host: 1}} - ]).toArray(); - assert.eq(numHosts, results.length, results); - for (let i = 0; i < numHosts; i++) { - const matched = i === 0 ? numDocs : 0; - assert.eq({host: "host_" + i, matchedB: matched}, results[i], results); - } - - // Equality Match With Let (uncorrelated) - // Make sure injected $sequentialDocumentCache in the inner pipeline is removed. - // $sequentialDocumentCache is not located right after unpack bucket stage. - results = collA.aggregate([ - { - $lookup: { - from: collB.getName(), - let: {"outer_hostname": "$tags.hostname"}, - pipeline: [ - {$match: {$expr: {$eq: ["$$outer_hostname", hosts[0].tags.hostname]}}}, - {$set: {foo: {$const: 123}}}, // uncorrelated - ], - as: "matchedB" - } - }, { - $project: { - _id: 0, - host: "$tags.hostname", - matchedB: { - $size: "$matchedB" - } - } - }, - {$sort: {host: 1}} - ]).toArray(); - assert.eq(numHosts, results.length, results); - for (let i = 0; i < numHosts; i++) { - const matched = i === 0 ? numDocs : 0; - assert.eq({host: "host_" + i, matchedB: matched}, results[i], results); - } - - // Equality Match With Let (correlated, no $match re-order) - // Make sure injected $sequentialDocumentCache in the inner pipeline is removed. - // $sequentialDocumentCache is located at the very end of pipeline. - results = collA.aggregate([ - { - $lookup: { - from: collB.getName(), - let: {"outer_hostname": "$tags.hostname"}, - pipeline: [ - {$match: {$expr: {$eq: ["$$outer_hostname", hosts[0].tags.hostname]}}}, - {$set: {foo: "$$outer_hostname"}}, // correlated - ], - as: "matchedB" - } - }, { - $project: { - _id: 0, - host: "$tags.hostname", - matchedB: { - $size: "$matchedB" - } - } - }, - {$sort: {host: 1}} - ]).toArray(); - assert.eq(numHosts, results.length, results); - for (let i = 0; i < numHosts; i++) { - const matched = i === 0 ? numDocs : 0; - assert.eq({host: "host_" + i, matchedB: matched}, results[i], results); - } - - // Unequal joins - results = collA.aggregate([ + // Unequal joins + results = collA.aggregate([ { $lookup: { from: collB.getName(), @@ -210,28 +114,13 @@ TimeseriesTest.run((insert) => { }, {$sort: {host: 1}} ]).toArray(); - assert.eq(numHosts, results.length, results); - let expectedCount = 0; - for (let i = 0; i < numHosts; i++) { - expectedCount += entryCountPerHost[i]; - assert.eq( - {host: "host_" + i, matchedB: expectedCount}, results[i], entryCountPerHost); - } - - // $sequenceDocumentsCache might optimize out $internalUnpackBucket and cause a - // crash on such query. - results = collA.aggregate([ - {$lookup: { - from: collB.getName(), - as: 'docs', - let: {}, - pipeline: [ - {$sort: {_id: 1}} - ] - }} - ]).toArray(); - assert.eq(numHosts, results.length, results); - }; + assert.eq(numHosts, results.length, results); + let expectedCount = 0; + for (let i = 0; i < numHosts; i++) { + expectedCount += entryCountPerHost[i]; + assert.eq({host: "host_" + i, matchedB: expectedCount}, results[i], entryCountPerHost); + } + }; // Exhaust the combinations of non-time-series and time-series collections for inner and outer // $lookup collections. @@ -247,91 +136,3 @@ TimeseriesTest.run((insert) => { }); }); })(); - -{ - // Test that we get the right results for $lookup on a timeseries collection with an internal - // pipeline containing both correlated and uncorrelated $match stages. Ensures we do not cache - // the results of this internal pipeline incorrectly - // (src/mongo/db/pipeline/document_source_sequential_document_cache.h) regardless of the order - // of the $match stages. - const testDB = db.getSiblingDB(jsTestName()); - testDB.local.insertMany([{_id: 0, key: 1}, {_id: 1, key: 2}]); - testDB.createCollection("foreign", {timeseries: {timeField: "time", metaField: "meta"}}); - testDB.foreign.insertMany([ - {time: new Date(), _id: 0, meta: 1, val1: 42, val2: 100}, - {time: new Date(), _id: 2, meta: 2, val1: 17, val2: 100} - ]); - - // The $sequentialCache (document_source_sequential_document_cache.h) decides whether or not it - // can cache the results of the internal lookup pipeline based on if the pipeline is correlated - // or uncorrelated. With timeseries, many rewrites occur which can merge $match stages together, - // have them pushed into the $_internalUnpackBucket stage as an eventFilter, or push them in - // front of the - // $_internalUnpackBucket stage. We need to make sure that the ability of the cache to recognize - // when there is a correlated $match in the pipeline to remain regardless of these rewrites. - (function testUncorrelatedFollowedByCorrelatedMatch() { - const lookupStage = {$lookup: { - from: "foreign", - let: {lkey: "$key"}, - pipeline: [ - {$match: {$expr: {$lt: ["$val1","$val2"]}}}, - {$match: {$expr: {$eq: ["$meta","$$lkey"]}}}, - {$project: {lkey: "$$lkey",fkey: "$meta",val: "$val1",_id: 0,}}, - ], - as: "joined" - }}; - const result = testDB.local.aggregate(lookupStage); - assertArrayEq({ - actual: result.toArray(), - expected: [ - {"_id": 1, "key": 2, "joined": [{"lkey": 2, "fkey": 2, "val": 17}]}, - {"_id": 0, "key": 1, "joined": [{"lkey": 1, "fkey": 1, "val": 42}]} - ], - extraErrorMsg: `Unexpected result of running pipeline ${tojson(lookupStage)}` - }); - })(); - - (function testCorrelatedFollowedByUncorrelatedMatch() { - const lookupStage = {$lookup: { - from: "foreign", - let: {lkey: "$key"}, - pipeline: [ - {$match: {$expr: {$eq: ["$meta","$$lkey"]}}}, - {$match: {$expr: {$lt: ["$val1","$val2"]}}}, - {$project: {lkey: "$$lkey",fkey: "$meta",val: "$val1",_id: 0,}}, - ], - as: "joined" - }}; - const result = testDB.local.aggregate(lookupStage); - assertArrayEq({ - actual: result.toArray(), - expected: [ - {"_id": 1, "key": 2, "joined": [{"lkey": 2, "fkey": 2, "val": 17}]}, - {"_id": 0, "key": 1, "joined": [{"lkey": 1, "fkey": 1, "val": 42}]} - ], - extraErrorMsg: `Unexpected result of running pipeline ${tojson(lookupStage)}` - }); - })(); - - (function testUncorrelatedFollowedByUncorrelatedMatch() { - const lookupStage = {$lookup: { - from: "foreign", - let: {lkey: "$key"}, - pipeline: [ - {$match: {$expr: {$lt: ["$meta","$val1"]}}}, - {$match: {$expr: {$lt: ["$val1","$val2"]}}}, - {$project: {meta: "$meta",val: "$val1",_id: 0,}}, - ], - as: "joined" - }}; - const result = testDB.local.aggregate(lookupStage); - assertArrayEq({ - actual: result.toArray(), - expected: [ - {"_id": 0, "key": 1, "joined": [{"meta": 1, "val": 42}, {"meta": 2, "val": 17}]}, - {"_id": 1, "key": 2, "joined": [{"meta": 1, "val": 42}, {"meta": 2, "val": 17}]} - ], - extraErrorMsg: `Unexpected result of running pipeline ${tojson(lookupStage)}` - }); - })(); -} diff --git a/jstests/core/timeseries/timeseries_match_pushdown.js b/jstests/core/timeseries/timeseries_match_pushdown.js deleted file mode 100644 index 93c841da9ec..00000000000 --- a/jstests/core/timeseries/timeseries_match_pushdown.js +++ /dev/null @@ -1,634 +0,0 @@ -/** - * Tests that the $match stage followed by unpacking stage has been pushed down with correct - * predicates. - * - * @tags: [ - * requires_timeseries, - * requires_fcv_60, - * does_not_support_stepdowns, - * does_not_support_transactions, - * directly_against_shardsvrs_incompatible, - * ] - */ -(function() { -"use strict"; - -load("jstests/libs/analyze_plan.js"); // For getAggPlanStages - -const coll = db.timeseries_match_pushdown; -const timeField = 'time'; -const metaField = 'meta'; -const measureField = 'a'; - -// The docs and queries are designed so that some buckets match the query entirely, some buckets -// match the query partially, and some with no matches. -const defaultDocs = [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: null, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: [1, 2, 3], [metaField]: 2} -]; - -/** - * Setup the collection and run a $match query with the specified 'eventFilter' or a 'pipeline'. - * Assert the 'wholeBucketFilter' is attached correctly to the unpacking stage, and has the expected - * result 'expectedDocs'. - */ -const runTest = function({docsToInsert, pipeline, eventFilter, wholeBucketFilter, expectedDocs}) { - // Set up the collection. Each test will have it's own collection setup. - coll.drop(); - assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField, metaField}})); - // Insert documents into the collection. - if (!docsToInsert) { - docsToInsert = defaultDocs; - } - assert.commandWorked(coll.insert(docsToInsert)); - - if (!pipeline) { - pipeline = [{$match: eventFilter}]; - } - const explain = assert.commandWorked(coll.explain().aggregate(pipeline)); - const unpackStages = getAggPlanStages(explain, '$_internalUnpackBucket'); - assert.eq(1, - unpackStages.length, - "Should only have a single $_internalUnpackBucket stage: " + tojson(explain)); - const unpackStage = unpackStages[0].$_internalUnpackBucket; - assert.docEq(unpackStage.eventFilter, eventFilter, "Incorrect eventFilter: " + tojson(explain)); - if (wholeBucketFilter) { - assert.docEq(unpackStage.wholeBucketFilter, - wholeBucketFilter, - "Incorrect wholeBucketFilter: " + tojson(explain)); - } else { - assert(!unpackStage.wholeBucketFilter, "Incorrect wholeBucketFilter: " + tojson(explain)); - } - - const docs = coll.aggregate([...pipeline, {$sort: {time: 1}}]).toArray(); - assert.eq(docs.length, expectedDocs.length, "Incorrect docs: " + tojson(docs)); - docs.forEach((doc, i) => { - // Do not need to check document _id, since checking time is already unique. - delete doc._id; - assert.docEq(doc, expectedDocs[i], "Incorrect docs: " + tojson(docs)); - }); -}; - -const aTime = ISODate('2022-01-01T00:00:03'); -const bTime = ISODate('2022-01-01T00:00:07'); -const bMeta = 3; -const aMeasure = 3; -const minTimeField = `control.min.${timeField}`; -const maxTimeField = `control.max.${timeField}`; - -// $gt on time -runTest({ - eventFilter: {[timeField]: {$gt: aTime}}, - wholeBucketFilter: {[minTimeField]: {$gt: aTime}}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: null, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: [1, 2, 3], [metaField]: 2}, - ], -}); - -// $gt on measurement -runTest({ - eventFilter: {[measureField]: {$gt: aMeasure}}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - ], -}); - -// $gt in $expr on time -runTest({ - pipeline: [{$match: {$expr: {$gt: [`$${timeField}`, {$const: aTime}]}}}], - eventFilter: { - $and: [ - {[timeField]: {$_internalExprGt: aTime}}, - {$expr: {$gt: [`$${timeField}`, {$const: aTime}]}}, - ] - }, - wholeBucketFilter: { - $and: [ - {[minTimeField]: {$_internalExprGt: aTime}}, - {[minTimeField]: {$_internalExprGt: aTime}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: null, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: [1, 2, 3], [metaField]: 2}, - ], -}); - -// $gte on time -runTest({ - eventFilter: {[timeField]: {$gte: aTime}}, - wholeBucketFilter: {[minTimeField]: {$gte: aTime}}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: null, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: [1, 2, 3], [metaField]: 2} - ], -}); - -// $gte on measurement -runTest({ - eventFilter: {[measureField]: {$gte: aMeasure}}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: [1, 2, 3], [metaField]: 2} - ], -}); - -// $gte in $expr on time -runTest({ - pipeline: [{$match: {$expr: {$gte: [`$${timeField}`, {$const: aTime}]}}}], - eventFilter: { - $and: [ - {[timeField]: {$_internalExprGte: aTime}}, - {$expr: {$gte: [`$${timeField}`, {$const: aTime}]}}, - ] - }, - wholeBucketFilter: { - $and: [ - {[minTimeField]: {$_internalExprGte: aTime}}, - {[minTimeField]: {$_internalExprGte: aTime}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: null, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: [1, 2, 3], [metaField]: 2} - ], -}); - -// $lt on time -runTest({ - eventFilter: {[timeField]: {$lt: aTime}}, - wholeBucketFilter: {[maxTimeField]: {$lt: aTime}}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - ], -}); - -// $lt on measurement -runTest({ - eventFilter: {[measureField]: {$lt: aMeasure}}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: [1, 2, 3], [metaField]: 2} - ], -}); - -// $lt in $expr on time -runTest({ - pipeline: [{$match: {$expr: {$lt: [`$${timeField}`, {$const: aTime}]}}}], - eventFilter: { - $and: [ - {[timeField]: {$_internalExprLt: aTime}}, - {$expr: {$lt: [`$${timeField}`, {$const: aTime}]}}, - ] - }, - wholeBucketFilter: { - $and: [ - {[maxTimeField]: {$_internalExprLt: aTime}}, - {[maxTimeField]: {$_internalExprLt: aTime}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - ], -}); - -// $lte on time -runTest({ - eventFilter: {[timeField]: {$lte: aTime}}, - wholeBucketFilter: {[maxTimeField]: {$lte: aTime}}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - ], -}); - -// $lte in $expr on time -runTest({ - pipeline: [{$match: {$expr: {$lte: [`$${timeField}`, {$const: aTime}]}}}], - eventFilter: { - $and: [ - {[timeField]: {$_internalExprLte: aTime}}, - {$expr: {$lte: [`$${timeField}`, {$const: aTime}]}}, - ] - }, - wholeBucketFilter: { - $and: [ - {[maxTimeField]: {$_internalExprLte: aTime}}, - {[maxTimeField]: {$_internalExprLte: aTime}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - ], -}); - -// $lte on measurement -runTest({ - eventFilter: {[measureField]: {$lte: aMeasure}}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: [1, 2, 3], [metaField]: 2} - ], -}); - -// $eq on time -runTest({ - eventFilter: {[timeField]: {$eq: aTime}}, - wholeBucketFilter: {$and: [{[minTimeField]: {$eq: aTime}}, {[maxTimeField]: {$eq: aTime}}]}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - ], -}); - -// $eq in $expr on time -runTest({ - pipeline: [{$match: {$expr: {$eq: [`$${timeField}`, {$const: aTime}]}}}], - eventFilter: { - $and: [ - {[timeField]: {$_internalExprEq: aTime}}, - {$expr: {$eq: [`$${timeField}`, {$const: aTime}]}}, - ] - }, - wholeBucketFilter: { - $and: [ - {[minTimeField]: {$_internalExprEq: aTime}}, - {[maxTimeField]: {$_internalExprEq: aTime}}, - {[minTimeField]: {$_internalExprEq: aTime}}, - {[maxTimeField]: {$_internalExprEq: aTime}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - ], -}); - -// $eq on measurement -runTest({ - docsToInsert: [ - ...defaultDocs, - {[timeField]: ISODate('2022-01-01T00:00:06'), [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: null, [metaField]: 1} - ], - eventFilter: {[measureField]: {$eq: aMeasure}}, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: [1, 2, 3], [metaField]: 2} - ], -}); - -// $and on time -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, - ], - eventFilter: {$and: [{[timeField]: {$gt: aTime}}, {[timeField]: {$lt: bTime}}]}, - wholeBucketFilter: { - $and: [ - {[minTimeField]: {$gt: aTime}}, - {[maxTimeField]: {$lt: bTime}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, - ], -}); - -//$or on time -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, - ], - eventFilter: {$or: [{[timeField]: {$lte: aTime}}, {[timeField]: {$gte: bTime}}]}, - wholeBucketFilter: { - $or: [ - {[maxTimeField]: {$lte: aTime}}, - {[minTimeField]: {$gte: bTime}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, - ], -}); - -// $match on time and meta -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, - ], - pipeline: [{$match: {$and: [{[timeField]: {$gt: aTime}}, {[metaField]: {$lte: bMeta}}]}}], - eventFilter: {[timeField]: {$gt: aTime}}, - wholeBucketFilter: { - [minTimeField]: {$gt: aTime}, - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, - {[timeField]: ISODate('2022-01-01T00:00:07'), [measureField]: 7, [metaField]: 3}, - {[timeField]: ISODate('2022-01-01T00:00:08'), [measureField]: 8, [metaField]: 3}, - ], -}); - -// $match on time and meta inside $expr. There should not be a wholeBucketFilter, since the entire -// $and expression cannot be rewritten as a MatchExpression, and for $expr predicates we only -// generate a wholeBucketFilter for single predicates on the timeField. -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:03'), [metaField]: 1, [measureField]: 3}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [metaField]: 1, [measureField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [metaField]: 2, [measureField]: 3}, - ], - pipeline: [{ - $match: { - $expr: { - $and: - [{$eq: [`$${metaField}`, `$${measureField}`]}, {$gt: [`$${timeField}`, aTime]}] - } - } - }], - eventFilter: { - $and: [ - {[timeField]: {$_internalExprGt: aTime}}, - { - $expr: { - $and: [ - {$eq: [`$${metaField}`, `$${measureField}`]}, - {$gt: [`$${timeField}`, {$const: aTime}]} - ] - } - }, - ] - }, - expectedDocs: [{[timeField]: ISODate('2022-01-01T00:00:04'), [metaField]: 1, [measureField]: 1}] -}); - -// $match on time and meta inside $expr. The entire $and expression can be rewritten into a -// MatchExpression. However, for $expr predicates we only generate a wholeBucketFilter for single -// predicates on the timeField. -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:03'), [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [metaField]: 2}, - ], - pipeline: - [{$match: {$expr: {$and: [{$eq: [`$${metaField}`, 1]}, {$gt: [`$${timeField}`, aTime]}]}}}], - eventFilter: { - $and: [ - {[timeField]: {$_internalExprGt: aTime}}, - { - $expr: { - $and: [ - {$eq: [`$${metaField}`, {$const: 1}]}, - {$gt: [`$${timeField}`, {$const: aTime}]} - ] - } - }, - {[metaField]: {$_internalExprEq: 1}}, - {[timeField]: {$_internalExprGt: aTime}}, - ] - }, - expectedDocs: [{[timeField]: ISODate('2022-01-01T00:00:04'), [metaField]: 1}] -}); - -// $match on time or meta -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 3}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 4}, - ], - eventFilter: {$or: [{[timeField]: {$lte: aTime}}, {[metaField]: {$gt: bMeta}}]}, - wholeBucketFilter: { - $or: [ - {[maxTimeField]: {$lte: aTime}}, - {[metaField]: {$gt: bMeta}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 4}, - ], -}); - -// double $match -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, - ], - pipeline: [{$match: {[timeField]: {$gt: aTime}}}, {$match: {[timeField]: {$lt: bTime}}}], - eventFilter: {$and: [{[timeField]: {$gt: aTime}}, {[timeField]: {$lt: bTime}}]}, - wholeBucketFilter: { - $and: [ - {[minTimeField]: {$gt: aTime}}, - {[maxTimeField]: {$lt: bTime}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, - ], -}); - -// triple $match -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 0}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [measureField]: 4, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, - ], - pipeline: [ - {$match: {[timeField]: {$gt: aTime}}}, - {$match: {[timeField]: {$lt: bTime}}}, - {$match: {[timeField]: {$lt: aTime}}}, - ], - eventFilter: { - $and: - [{[timeField]: {$gt: aTime}}, {[timeField]: {$lt: bTime}}, {[timeField]: {$lt: aTime}}] - }, - wholeBucketFilter: { - $and: [ - {[minTimeField]: {$gt: aTime}}, - {[maxTimeField]: {$lt: bTime}}, - {[maxTimeField]: {$lt: aTime}}, - ] - }, - expectedDocs: [], -}); - -// $and inside $expr with comparison on meta and measurement. There should not be a -// wholeBucketFilter, since the entire $and expression cannot be rewritten as a MatchExpression, and -// for $expr predicates we only generate a wholeBucketFilter for single predicates on the timeField. -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:00'), [measureField]: 0, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 3}, - ], - pipeline: [{ - $match: { - $expr: - {$and: [{$lt: [`$${measureField}`, `$${metaField}`]}, {$gt: [`$${metaField}`, 1]}]} - } - }], - eventFilter: { - $and: [ - {"meta": {$_internalExprGt: 1}}, - { - $expr: { - $and: [ - {$lt: [`$${measureField}`, `$${metaField}`]}, - {$gt: [`$${metaField}`, {$const: 1}]} - ] - } - } - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [metaField]: 2}, - ] -}); - -// Same test as above, but the entire $and expression can be rewritten as a MatchExpression. -// However, for $expr predicates we only generate a wholeBucketFilter for single predicates on the -// timeField. -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:00'), [measureField]: 0, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: null, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:06'), [measureField]: 6, [metaField]: 3}, - ], - pipeline: - [{$match: {$expr: {$and: [{$gte: [`$${measureField}`, 2]}, {$lt: [`$${metaField}`, 3]}]}}}], - eventFilter: { - $and: [ - {[measureField]: {$_internalExprGte: 2}}, - { - $expr: { - $and: [ - {$gte: [`$${measureField}`, {$const: 2}]}, - {$lt: [`$${metaField}`, {$const: 3}]} - ] - } - }, - {[measureField]: {$_internalExprGte: 2}}, - {[metaField]: {$_internalExprLt: 3}}, - ] - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 2}, - ] -}); - -// Same test as above but with $or and different comparison operators. -runTest({ - docsToInsert: [ - {[timeField]: ISODate('2022-01-01T00:00:00'), [measureField]: 0, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:05'), [measureField]: 5, [metaField]: 3}, - ], - pipeline: [{ - $match: { - $expr: - {$or: [{$lt: [`$${measureField}`, `$${metaField}`]}, {$lte: [`$${metaField}`, 2]}]} - } - }], - eventFilter: { - $expr: { - $or: [ - {$lt: [`$${measureField}`, `$${metaField}`]}, - {$lte: [`$${metaField}`, {$const: 2}]} - ] - } - }, - expectedDocs: [ - {[timeField]: ISODate('2022-01-01T00:00:00'), [measureField]: 0, [metaField]: 1}, - {[timeField]: ISODate('2022-01-01T00:00:01'), [measureField]: 1, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:02'), [measureField]: 2, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:03'), [measureField]: 3, [metaField]: 2}, - {[timeField]: ISODate('2022-01-01T00:00:04'), [metaField]: 2}, - ] -}); -})(); diff --git a/jstests/core/timeseries/timeseries_match_pushdown_with_project.js b/jstests/core/timeseries/timeseries_match_pushdown_with_project.js deleted file mode 100644 index 2d8e872c33a..00000000000 --- a/jstests/core/timeseries/timeseries_match_pushdown_with_project.js +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Tests that the unpacking stage has correct unpacking behaviour when $match is pushed into it. - * - * @tags: [ - * requires_timeseries, - * requires_fcv_60, - * does_not_support_stepdowns, - * does_not_support_transactions, - * directly_against_shardsvrs_incompatible, - * ] - */ -(function() { -"use strict"; - -load("jstests/libs/analyze_plan.js"); // For getAggPlanStages - -const coll = db.timeseries_match_pushdown_with_project; -coll.drop(); - -const timeField = 'time'; -const metaField = 'meta'; -assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField, metaField}})); - -const aTime = ISODate('2022-01-01T00:00:00'); -assert.commandWorked(coll.insert([ - {[timeField]: aTime, a: 1, b: 1, _id: 1}, - {[timeField]: aTime, a: 2, b: 2, _id: 2}, - {[timeField]: aTime, a: 3, b: 3, _id: 3}, - {[timeField]: aTime, a: 4, b: 4, _id: 4}, - {[timeField]: aTime, a: 5, b: 5, _id: 5}, - {[timeField]: aTime, a: 6, b: 6, _id: 6}, - {[timeField]: aTime, a: 7, b: 7, _id: 7}, - {[timeField]: aTime, a: 8, b: 8, _id: 8}, - {[timeField]: aTime, a: 9, b: 9, _id: 9}, -])); - -/** - * Runs a 'pipeline', asserts the bucket unpacking 'behaviour' (either include or exclude) is - * expected. - */ -const runTest = function({pipeline, behaviour, expectedDocs}) { - const explain = assert.commandWorked(coll.explain().aggregate(pipeline)); - const unpackStages = getAggPlanStages(explain, '$_internalUnpackBucket'); - assert.eq(1, - unpackStages.length, - "Should only have a single $_internalUnpackBucket stage: " + tojson(explain)); - const unpackStage = unpackStages[0].$_internalUnpackBucket; - if (behaviour.include) { - assert(unpackStage.include, - "Unpacking stage must have 'include' behaviour: " + tojson(explain)); - assert.sameMembers(behaviour.include, unpackStage.include); - } - if (behaviour.exclude) { - assert(unpackStage.exclude, - "Unpacking stage must have 'exclude' behaviour: " + tojson(explain)); - assert.sameMembers(behaviour.exclude, unpackStage.exclude); - } - - const docs = coll.aggregate([...pipeline, {$sort: {a: 1, b: 1, _id: 1}}]).toArray(); - assert.eq(docs.length, expectedDocs.length, "Incorrect docs: " + tojson(docs)); - docs.forEach((doc, i) => { - assert.docEq(doc, expectedDocs[i], "Incorrect docs: " + tojson(docs)); - }); -}; - -runTest({ - pipeline: [{$match: {a: {$gt: 5}}}, {$project: {b: 1}}], - behaviour: {include: ['_id', 'a', 'b']}, - expectedDocs: [ - {b: 6, _id: 6}, - {b: 7, _id: 7}, - {b: 8, _id: 8}, - {b: 9, _id: 9}, - ], -}); - -runTest({ - pipeline: [{$match: {a: {$gt: 5}}}, {$project: {_id: 0, b: 1}}], - behaviour: {include: ['a', 'b']}, - expectedDocs: [ - {b: 6}, - {b: 7}, - {b: 8}, - {b: 9}, - ], -}); - -runTest({ - pipeline: [{$match: {a: {$gt: 5}}}, {$project: {a: 1}}], - behaviour: {include: ['_id', 'a']}, - expectedDocs: [ - {a: 6, _id: 6}, - {a: 7, _id: 7}, - {a: 8, _id: 8}, - {a: 9, _id: 9}, - ], -}); - -runTest({ - pipeline: [{$match: {a: {$gt: 5}}}, {$project: {_id: 0, a: 1}}], - behaviour: {include: ['a']}, - expectedDocs: [ - {a: 6}, - {a: 7}, - {a: 8}, - {a: 9}, - ], -}); - -runTest({ - pipeline: [{$match: {a: {$gt: 5}}}, {$project: {a: 0}}], - behaviour: {exclude: []}, - expectedDocs: [ - {[timeField]: aTime, b: 6, _id: 6}, - {[timeField]: aTime, b: 7, _id: 7}, - {[timeField]: aTime, b: 8, _id: 8}, - {[timeField]: aTime, b: 9, _id: 9}, - ], -}); - -runTest({ - pipeline: [{$match: {a: {$gt: 5}}}, {$project: {b: 0}}], - behaviour: {exclude: []}, - expectedDocs: [ - {[timeField]: aTime, a: 6, _id: 6}, - {[timeField]: aTime, a: 7, _id: 7}, - {[timeField]: aTime, a: 8, _id: 8}, - {[timeField]: aTime, a: 9, _id: 9}, - ], -}); -})(); diff --git a/jstests/core/timeseries/timeseries_merge.js b/jstests/core/timeseries/timeseries_merge.js index fb6a23df803..59d4dbee889 100644 --- a/jstests/core/timeseries/timeseries_merge.js +++ b/jstests/core/timeseries/timeseries_merge.js @@ -6,7 +6,6 @@ * does_not_support_stepdowns, * does_not_support_transactions, * requires_timeseries, - * references_foreign_collection, * ] */ (function() { diff --git a/jstests/core/timeseries/timeseries_metric_index_2dsphere.js b/jstests/core/timeseries/timeseries_metric_index_2dsphere.js index 7096ace9513..e47119b5bbd 100644 --- a/jstests/core/timeseries/timeseries_metric_index_2dsphere.js +++ b/jstests/core/timeseries/timeseries_metric_index_2dsphere.js @@ -18,9 +18,8 @@ load("jstests/core/timeseries/libs/timeseries.js"); load("jstests/libs/analyze_plan.js"); -load("jstests/libs/feature_flag_util.js"); -if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { +if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { return; } diff --git a/jstests/core/timeseries/timeseries_metric_index_ascending_descending.js b/jstests/core/timeseries/timeseries_metric_index_ascending_descending.js index 8a827b9d07b..bf1dc10e625 100644 --- a/jstests/core/timeseries/timeseries_metric_index_ascending_descending.js +++ b/jstests/core/timeseries/timeseries_metric_index_ascending_descending.js @@ -14,10 +14,9 @@ "use strict"; load("jstests/core/timeseries/libs/timeseries.js"); -load("jstests/libs/feature_flag_util.js"); load("jstests/libs/fixture_helpers.js"); -if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { +if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; diff --git a/jstests/core/timeseries/timeseries_metric_index_compound.js b/jstests/core/timeseries/timeseries_metric_index_compound.js index c81f9a4c53c..2ecc4732b70 100644 --- a/jstests/core/timeseries/timeseries_metric_index_compound.js +++ b/jstests/core/timeseries/timeseries_metric_index_compound.js @@ -13,9 +13,8 @@ "use strict"; load("jstests/core/timeseries/libs/timeseries.js"); -load("jstests/libs/feature_flag_util.js"); -if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { +if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; diff --git a/jstests/core/timeseries/timeseries_predicates.js b/jstests/core/timeseries/timeseries_predicates.js index ca78464cb31..a273cc8b128 100644 --- a/jstests/core/timeseries/timeseries_predicates.js +++ b/jstests/core/timeseries/timeseries_predicates.js @@ -17,7 +17,7 @@ const tsColl = db.timeseries_predicates_timeseries; coll.drop(); tsColl.drop(); assert.commandWorked( - db.createCollection(tsColl.getName(), {timeseries: {timeField: 'time', metaField: 'mt'}})); + db.createCollection(tsColl.getName(), {timeseries: {timeField: 'time', metaField: 'meta'}})); const bucketsColl = db.getCollection('system.buckets.' + tsColl.getName()); // Test that 'predicate' behaves correctly on the example documents, @@ -86,18 +86,18 @@ checkAllBucketings({x: {$exists: true}}, [ // Test $or... { - // ... on metric + mt. + // ... on metric + meta. checkAllBucketings({ $or: [ {x: {$lt: 0}}, - {'mt.y': {$gt: 0}}, + {'meta.y': {$gt: 0}}, ] }, [ - {x: +1, mt: {y: -1}}, - {x: +1, mt: {y: +1}}, - {x: -1, mt: {y: -1}}, - {x: -1, mt: {y: +1}}, + {x: +1, meta: {y: -1}}, + {x: +1, meta: {y: +1}}, + {x: -1, meta: {y: -1}}, + {x: -1, meta: {y: +1}}, ]); // ... when one argument can't be pushed down. @@ -131,18 +131,18 @@ checkAllBucketings({x: {$exists: true}}, [ // Test $and... { - // ... on metric + mt. + // ... on metric + meta. checkAllBucketings({ $and: [ {x: {$lt: 0}}, - {'mt.y': {$gt: 0}}, + {'meta.y': {$gt: 0}}, ] }, [ - {x: +1, mt: {y: -1}}, - {x: +1, mt: {y: +1}}, - {x: -1, mt: {y: -1}}, - {x: -1, mt: {y: +1}}, + {x: +1, meta: {y: -1}}, + {x: +1, meta: {y: +1}}, + {x: -1, meta: {y: -1}}, + {x: -1, meta: {y: +1}}, ]); // ... when one argument can't be pushed down. @@ -180,28 +180,28 @@ checkAllBucketings({ $or: [ { $and: [ - {'mt.a': {$gt: 0}}, + {'meta.a': {$gt: 0}}, {'x': {$lt: 0}}, ] }, { $and: [ - {'mt.b': {$gte: 0}}, + {'meta.b': {$gte: 0}}, {time: {$gt: ISODate('2020-01-01')}}, ] }, ] }, [ - {mt: {a: -1, b: -1}, x: -1, time: ISODate('2020-02-01')}, - {mt: {a: -1, b: -1}, x: -1, time: ISODate('2019-12-31')}, - {mt: {a: -1, b: -1}, x: +1, time: ISODate('2020-02-01')}, - {mt: {a: -1, b: -1}, x: +1, time: ISODate('2019-12-31')}, + {meta: {a: -1, b: -1}, x: -1, time: ISODate('2020-02-01')}, + {meta: {a: -1, b: -1}, x: -1, time: ISODate('2019-12-31')}, + {meta: {a: -1, b: -1}, x: +1, time: ISODate('2020-02-01')}, + {meta: {a: -1, b: -1}, x: +1, time: ISODate('2019-12-31')}, - {mt: {a: +1, b: -1}, x: -1, time: ISODate('2020-02-01')}, - {mt: {a: +1, b: -1}, x: -1, time: ISODate('2019-12-31')}, - {mt: {a: +1, b: -1}, x: +1, time: ISODate('2020-02-01')}, - {mt: {a: +1, b: -1}, x: +1, time: ISODate('2019-12-31')}, + {meta: {a: +1, b: -1}, x: -1, time: ISODate('2020-02-01')}, + {meta: {a: +1, b: -1}, x: -1, time: ISODate('2019-12-31')}, + {meta: {a: +1, b: -1}, x: +1, time: ISODate('2020-02-01')}, + {meta: {a: +1, b: -1}, x: +1, time: ISODate('2019-12-31')}, ]); // Test nested $and / $or where some leaf predicates cannot be pushed down. @@ -209,72 +209,72 @@ checkAllBucketings({ $or: [ { $and: [ - {'mt.a': {$gt: 0}}, + {'meta.a': {$gt: 0}}, {'x': {$exists: false}}, ] }, { $and: [ - {'mt.b': {$gte: 0}}, + {'meta.b': {$gte: 0}}, {time: {$gt: ISODate('2020-01-01')}}, ] }, ] }, [ - {mt: {a: -1, b: -1}, time: ISODate('2020-02-01')}, - {mt: {a: -1, b: -1}, time: ISODate('2019-12-31')}, - {mt: {a: -1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')}, - {mt: {a: -1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')}, + {meta: {a: -1, b: -1}, time: ISODate('2020-02-01')}, + {meta: {a: -1, b: -1}, time: ISODate('2019-12-31')}, + {meta: {a: -1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')}, + {meta: {a: -1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')}, - {mt: {a: +1, b: -1}, time: ISODate('2020-02-01')}, - {mt: {a: +1, b: -1}, time: ISODate('2019-12-31')}, - {mt: {a: +1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')}, - {mt: {a: +1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')}, + {meta: {a: +1, b: -1}, time: ISODate('2020-02-01')}, + {meta: {a: +1, b: -1}, time: ISODate('2019-12-31')}, + {meta: {a: +1, b: -1}, x: 'asdf', time: ISODate('2020-02-01')}, + {meta: {a: +1, b: -1}, x: 'asdf', time: ISODate('2019-12-31')}, ]); -// Test $exists on mt, inside $or. +// Test $exists on meta, inside $or. checkAllBucketings({ $or: [ - {"mt.a": {$exists: true}}, + {"meta.a": {$exists: true}}, {"x": {$gt: 2}}, ] }, [ - {mt: {a: 1}, x: 1}, - {mt: {a: 2}, x: 2}, - {mt: {a: 3}, x: 3}, - {mt: {a: 4}, x: 4}, - {mt: {}, x: 1}, - {mt: {}, x: 2}, - {mt: {}, x: 3}, - {mt: {}, x: 4}, + {meta: {a: 1}, x: 1}, + {meta: {a: 2}, x: 2}, + {meta: {a: 3}, x: 3}, + {meta: {a: 4}, x: 4}, + {meta: {}, x: 1}, + {meta: {}, x: 2}, + {meta: {}, x: 3}, + {meta: {}, x: 4}, ]); -// Test $in on mt, inside $or. +// Test $in on meta, inside $or. checkAllBucketings({ $or: [ - {"mt.a": {$in: [1, 3]}}, + {"meta.a": {$in: [1, 3]}}, {"x": {$gt: 2}}, ] }, [ - {mt: {a: 1}, x: 1}, - {mt: {a: 2}, x: 2}, - {mt: {a: 3}, x: 3}, - {mt: {a: 4}, x: 4}, - {mt: {}, x: 1}, - {mt: {}, x: 2}, - {mt: {}, x: 3}, - {mt: {}, x: 4}, + {meta: {a: 1}, x: 1}, + {meta: {a: 2}, x: 2}, + {meta: {a: 3}, x: 3}, + {meta: {a: 4}, x: 4}, + {meta: {}, x: 1}, + {meta: {}, x: 2}, + {meta: {}, x: 3}, + {meta: {}, x: 4}, ]); -// Test geo predicates on mt, inside $or. +// Test geo predicates on meta, inside $or. for (const pred of ['$geoWithin', '$geoIntersects']) { checkAllBucketings({ $or: [ { - "mt.location": { + "meta.location": { [pred]: { $geometry: { type: "Polygon", @@ -293,90 +293,66 @@ for (const pred of ['$geoWithin', '$geoIntersects']) { ] }, [ - {mt: {location: [1, 1]}, x: 1}, - {mt: {location: [1, 1]}, x: 2}, - {mt: {location: [1, 1]}, x: 3}, - {mt: {location: [1, 1]}, x: 4}, - {mt: {location: [5, 5]}, x: 1}, - {mt: {location: [5, 5]}, x: 2}, - {mt: {location: [5, 5]}, x: 3}, - {mt: {location: [5, 5]}, x: 4}, + {meta: {location: [1, 1]}, x: 1}, + {meta: {location: [1, 1]}, x: 2}, + {meta: {location: [1, 1]}, x: 3}, + {meta: {location: [1, 1]}, x: 4}, + {meta: {location: [5, 5]}, x: 1}, + {meta: {location: [5, 5]}, x: 2}, + {meta: {location: [5, 5]}, x: 3}, + {meta: {location: [5, 5]}, x: 4}, ]); } -// Test $mod on mt, inside $or. +// Test $mod on meta, inside $or. // $mod is an example of a predicate that we don't handle specially in time-series optimizations: // it can be pushed down if and only if it's on a metadata field. checkAllBucketings({ $or: [ - {"mt.a": {$mod: [2, 0]}}, + {"meta.a": {$mod: [2, 0]}}, {"x": {$gt: 4}}, ] }, [ - {mt: {a: 1}, x: 1}, - {mt: {a: 2}, x: 2}, - {mt: {a: 3}, x: 3}, - {mt: {a: 4}, x: 4}, - {mt: {a: 5}, x: 5}, - {mt: {a: 6}, x: 6}, - {mt: {a: 7}, x: 7}, - {mt: {a: 8}, x: 8}, + {meta: {a: 1}, x: 1}, + {meta: {a: 2}, x: 2}, + {meta: {a: 3}, x: 3}, + {meta: {a: 4}, x: 4}, + {meta: {a: 5}, x: 5}, + {meta: {a: 6}, x: 6}, + {meta: {a: 7}, x: 7}, + {meta: {a: 8}, x: 8}, ]); -// Test $elemMatch on mt, inside $or. +// Test $elemMatch on meta, inside $or. checkAllBucketings({ $or: [ - {"mt.a": {$elemMatch: {b: 3}}}, + {"meta.a": {$elemMatch: {b: 3}}}, {"x": {$gt: 4}}, ] }, [ - {x: 1, mt: {a: []}}, - {x: 2, mt: {a: [{b: 2}]}}, - {x: 3, mt: {a: [{b: 3}]}}, - {x: 4, mt: {a: [{b: 2}, {b: 3}]}}, - {x: 5, mt: {a: []}}, - {x: 6, mt: {a: [{b: 2}]}}, - {x: 7, mt: {a: [{b: 3}]}}, - {x: 8, mt: {a: [{b: 2}, {b: 3}]}}, + {x: 1, meta: {a: []}}, + {x: 2, meta: {a: [{b: 2}]}}, + {x: 3, meta: {a: [{b: 3}]}}, + {x: 4, meta: {a: [{b: 2}, {b: 3}]}}, + {x: 5, meta: {a: []}}, + {x: 6, meta: {a: [{b: 2}]}}, + {x: 7, meta: {a: [{b: 3}]}}, + {x: 8, meta: {a: [{b: 2}, {b: 3}]}}, ]); checkAllBucketings({ $or: [ - {"mt.a": {$elemMatch: {b: 2, c: 3}}}, + {"meta.a": {$elemMatch: {b: 2, c: 3}}}, {"x": {$gt: 3}}, ] }, [ - {x: 1, mt: {a: []}}, - {x: 2, mt: {a: [{b: 2, c: 3}]}}, - {x: 3, mt: {a: [{b: 2}, {c: 3}]}}, - {x: 4, mt: {a: []}}, - {x: 5, mt: {a: [{b: 2, c: 3}]}}, - {x: 6, mt: {a: [{b: 2}, {c: 3}]}}, + {x: 1, meta: {a: []}}, + {x: 2, meta: {a: [{b: 2, c: 3}]}}, + {x: 3, meta: {a: [{b: 2}, {c: 3}]}}, + {x: 4, meta: {a: []}}, + {x: 5, meta: {a: [{b: 2, c: 3}]}}, + {x: 6, meta: {a: [{b: 2}, {c: 3}]}}, ]); - -// Test a standalone $elemMatch on mt. -checkAllBucketings({"mt.a": {$elemMatch: {b: 3}}}, [ - {mt: {a: []}}, - {mt: {a: [{b: 2}]}}, - {mt: {a: [{b: 3}]}}, - {mt: {a: [{b: 2}, {b: 3}]}}, - {mt: {a: []}}, - {mt: {a: [{b: 2}]}}, - {mt: {a: [{b: 3}]}}, - {mt: {a: [{b: 2}, {b: 3}]}}, -]); - -// Test a standalone $size on mt. -checkAllBucketings({"mt.a": {$size: 1}}, [ - {mt: {a: []}}, - {mt: {a: [{b: 2}]}}, - {mt: {a: [{b: 3}]}}, - {mt: {a: [{b: 2}, {b: 3}]}}, - {mt: {a: []}}, - {mt: {a: [{b: 2}]}}, - {mt: {a: [{b: 3}]}}, - {mt: {a: [{b: 2}, {b: 3}]}}, -]); })(); diff --git a/jstests/core/timeseries/timeseries_project.js b/jstests/core/timeseries/timeseries_project.js index b735a8a3b66..9375f06240d 100644 --- a/jstests/core/timeseries/timeseries_project.js +++ b/jstests/core/timeseries/timeseries_project.js @@ -4,7 +4,7 @@ * @tags: [ * does_not_support_stepdowns, * does_not_support_transactions, - * requires_fcv_60, + * requires_fcv_53, * ] */ (function() { @@ -97,7 +97,6 @@ const doc = { time: new Date("2019-10-11T14:39:18.670Z"), x: 5, a: 3, - obj: {a: 3}, }; assert.commandWorked(tsColl.insert(doc)); assert.commandWorked(regColl.insert(doc)); @@ -108,192 +107,10 @@ let tsDoc = tsColl.aggregate(pipeline).toArray(); let regDoc = regColl.aggregate(pipeline).toArray(); assert.docEq(tsDoc, regDoc); -pipeline = [{$project: {_id: 0, obj: "$x", b: {$add: ["$obj.a", 1]}}}]; -tsDoc = tsColl.aggregate(pipeline).toArray(); -regDoc = regColl.aggregate(pipeline).toArray(); -assert.docEq(tsDoc, regDoc); - // Test $addFields. pipeline = [{$addFields: {a: "$x", b: "$a"}}, {$project: {_id: 0}}]; tsDoc = tsColl.aggregate(pipeline).toArray(); regDoc = regColl.aggregate(pipeline).toArray(); assert.docEq(tsDoc, regDoc); - -pipeline = [{$addFields: {obj: "$x", b: {$add: ["$obj.a", 1]}}}, {$project: {_id: 0}}]; -tsDoc = tsColl.aggregate(pipeline).toArray(); -regDoc = regColl.aggregate(pipeline).toArray(); -assert.docEq(tsDoc, regDoc); - -pipeline = [{$project: {a: 1, _id: 0}}, {$project: {newMeta: "$x"}}]; -tsDoc = tsColl.aggregate(pipeline).toArray(); -regDoc = regColl.aggregate(pipeline).toArray(); -assert.docEq(tsDoc, regDoc); -})(); - -// Test that an includes projection that doesn't include previously included computed meta fields -// outputs correctly. -(function testIncludeComputedMetaFieldThenProjectWithoutComputedMetaField() { - coll.drop(); - assert.commandWorked( - db.createCollection(coll.getName(), {timeseries: {timeField: 'time', metaField: 'tag'}})); - - const doc = {_id: 0, time: ISODate("2024-01-01T00:00:00.000Z"), tag: {field: "A"}}; - assert.commandWorked(coll.insert(doc)); - let result = {}; - - // Added field which uses meta should not be in the result. - result = - coll.aggregate( - [{$addFields: {time: {$dateFromParts: {year: "$tag.none"}}}}, {$project: {tag: 1}}]) - .toArray(); - assert.docEq([{"tag": {"field": "A"}, "_id": 0}], result); - - // Test with non-null value. - result = coll.aggregate([{$addFields: {time: {$toLower: "$tag.field"}}}, {$project: {tag: 1}}]) - .toArray(); - assert.docEq([{"tag": {"field": "A"}, "_id": 0}], result); - - // Testing with $match between $addFields and $project. - result = coll.aggregate([ - {$addFields: {time: {$dateFromParts: {year: "$tag.none"}}}}, - {$match: {tag: {field: "A"}}}, - {$project: {tag: 1}} - ]) - .toArray(); - assert.docEq([{"tag": {field: "A"}, "_id": 0}], result); - - // Test with non-null value. - result = - coll.aggregate( - [{$addFields: {time: "$tag.field"}}, {$match: {time: "A"}}, {$project: {tag: 1}}]) - .toArray(); - assert.docEq([{"tag": {field: "A"}, "_id": 0}], result); - - // Testing with $project after $project. - result = coll.aggregate([{$project: {tag: 1}}, {$project: {time: 1}}]).toArray(); - assert.docEq([{"_id": 0}], result); - - // Testing with $set. - result = coll.aggregate([{$set: {tag: "$tag.field"}}, {$project: {time: 1}}]).toArray(); - assert.docEq([{"_id": 0, "time": ISODate("2024-01-01T00:00:00Z")}], result); - - // Test that computedMetaFields that are included by the $project are still included. - result = coll.aggregate([ - {$addFields: {hi: {$dateFromParts: {year: "$tag.none"}}}}, - {$addFields: {bye: {$dateFromParts: {year: "$tag.none"}}}}, - {$project: {hi: 1}} - ]) - .toArray(); - assert.docEq([{"_id": 0, "hi": null}], result); - - // Test with non-null value. - result = coll.aggregate([ - {$addFields: {hi: "$tag.field"}}, - {$addFields: {bye: "$tag.field"}}, - {$project: {hi: 1}} - ]) - .toArray(); - assert.docEq([{"_id": 0, "hi": "A"}], result); - - // Test that $project that replaces field that is used works. - result = coll.aggregate([ - {$addFields: {hello: "$tag.field"}}, - {$project: {newTime: "$time", time: "$tag.field"}} - ]) - .toArray(); - - assert.docEq([{"_id": 0, newTime: ISODate("2024-01-01T00:00:00.000Z"), time: "A"}], result); - - // Test adding fields and including one of them. - result = coll.aggregate([ - {$addFields: {a: {$toLower: "$tag.field"}}}, - {$addFields: {b: {$toUpper: "$tag.field"}}}, - {$project: {b: 1}} - ]) - .toArray(); - assert.docEq([{b: "A", "_id": 0}], result); -})(); - -// Test that an excludes projection that excludes previously included computed meta fields -// outputs correctly. -(function testIncludeComputedMetaFieldThenExcludeComputedMetaField() { - coll.drop(); - assert.commandWorked( - db.createCollection(coll.getName(), {timeseries: {timeField: 'time', metaField: 'tag'}})); - - const doc = {_id: 0, time: ISODate("2024-01-01T00:00:00.000Z"), tag: {field: "A"}}; - assert.commandWorked(coll.insert(doc)); - let result = {}; - - // Excluded '_computedMetaProjField' should not be included. - result = coll.aggregate([ - {$addFields: {hello: {$dateFromParts: {year: "$tag.none"}}}}, - {$project: {hello: 0}} - ]) - .toArray(); - assert.docEq([{"time": ISODate("2024-01-01T00:00:00Z"), "tag": {"field": "A"}, "_id": 0}], - result); - - // Test with non-null value. - result = coll.aggregate([{$addFields: {time: {$toLower: "$tag.field"}}}, {$project: {time: 0}}]) - .toArray(); - assert.docEq([{"tag": {"field": "A"}, "_id": 0}], result); - - // Testing with $match between $addFields and $project. - result = coll.aggregate([ - {$addFields: {time: {$dateFromParts: {year: "$tag.none"}}}}, - {$match: {tag: {field: "A"}}}, - {$project: {time: 0}} - ]) - .toArray(); - assert.docEq([{"tag": {field: "A"}, "_id": 0}], result); - - // Test with non-null value. - result = - coll.aggregate( - [{$addFields: {time: "$tag.field"}}, {$match: {time: "A"}}, {$project: {time: 0}}]) - .toArray(); - assert.docEq([{"tag": {field: "A"}, "_id": 0}], result); - - // Testing with exclude $project after include $project. - result = coll.aggregate([{$project: {tag: 1}}, {$project: {tag: 0}}]).toArray(); - assert.docEq([{"_id": 0}], result); - - // Testing with $set. - result = coll.aggregate([{$set: {tag: "$tag.field"}}, {$project: {tag: 0}}]).toArray(); - assert.docEq([{"time": ISODate("2024-01-01T00:00:00Z"), "_id": 0}], result); - - // Exclude one added field. - result = coll.aggregate([ - {$addFields: {hi: {$dateFromParts: {year: "$tag.none"}}}}, - {$addFields: {bye: {$dateFromParts: {year: "$tag.none"}}}}, - {$project: {hi: 0}} - ]) - .toArray(); - assert.docEq( - [{"time": ISODate("2024-01-01T00:00:00Z"), "tag": {"field": "A"}, "_id": 0, "bye": null}], - result); - - // Test with non-null value. - result = coll.aggregate([ - {$addFields: {hi: "$tag.field"}}, - {$addFields: {bye: "$tag.field"}}, - {$project: {hi: 0}} - ]) - .toArray(); - assert.docEq( - [{"time": ISODate("2024-01-01T00:00:00Z"), "tag": {"field": "A"}, "_id": 0, "bye": "A"}], - result); - - // Test adding fields and excluding one of them. - result = coll.aggregate([ - {$addFields: {a: {$toLower: "$tag.field"}}}, - {$addFields: {b: {$toUpper: "$tag.field"}}}, - {$project: {b: 0}} - ]) - .toArray(); - assert.docEq( - [{"time": ISODate("2024-01-01T00:00:00Z"), "tag": {"field": "A"}, "_id": 0, "a": "a"}], - result); })(); })(); diff --git a/jstests/core/timeseries/timeseries_project_pushdown.js b/jstests/core/timeseries/timeseries_project_pushdown.js deleted file mode 100644 index 91b009aa00d..00000000000 --- a/jstests/core/timeseries/timeseries_project_pushdown.js +++ /dev/null @@ -1,196 +0,0 @@ -/** - * Test the behavior of $project with pipelines that require the whole document. Specifically, we - * are targeting that rewrites for $project with $getField, or '$$ROOT'/'$$CURRENT' should only - * occur in certain situations. - * - * @tags: [ - * requires_timeseries, - * does_not_support_stepdowns, - * does_not_support_transactions, - * ] - */ -(function() { -"use strict"; - -const timeField = "t"; -const metaField = "meta1"; -const coll = db.timeseries_project_pushdown; - -function runTest({docs, pipeline, expectedResults}) { - coll.drop(); - assert.commandWorked(db.createCollection( - coll.getName(), {timeseries: {timeField: timeField, metaField: metaField}})); - assert.commandWorked(coll.insertMany(docs)); - const results = coll.aggregate(pipeline).toArray(); - assert.sameMembers(expectedResults, results, () => { - return `Pipeline: ${tojson(pipeline)}. Explain: ${ - tojson(coll.explain().aggregate(pipeline))}`; - }); -} - -// -// The following tests confirm the behavior of queries in the form: {$getField: "fieldName"}. -// -(function testMeta_OnlyStringField() { - runTest({ - docs: [{_id: 1, [timeField]: new Date(), [metaField]: 2}], - pipeline: [{$project: {new: {$getField: metaField}, _id: 0}}], - expectedResults: [{new: 2}] - }); -})(); - -// $getField does not traverse objects, and should not be rewritten when it relies on a mix of -// metaField and measurement fields. Let's validate that behavior. -(function testDottedPath_OnlyStringField() { - runTest({ - docs: [ - {_id: 1, [timeField]: new Date(), [metaField]: 2, "a.b": 3, a: {b: 2}}, - {_id: 2, [timeField]: new Date(), [metaField]: 2, a: {b: 2}} // missing field. - ], - pipeline: [{$project: {new: {$add: [`$${metaField}`, {$getField: "a.b"}]}}}], - expectedResults: [{_id: 1, new: 5}, {_id: 2, new: null}] - }); -})(); - -// -// The following tests confirm the behavior of queries in the form: {$getField: {$literal: -// "fieldName"}}. -// -(function testDottedPath_LiteralExpr() { - runTest({ - docs: [ - {_id: 1, [timeField]: new Date(), [metaField]: 2, "a.$b": 3, a: {b: 4}}, - {_id: 2, [timeField]: new Date(), [metaField]: 2, a: {b: 2}} // missing field. - ], - pipeline: [{$project: {new: {$add: [`$${metaField}`, {$getField: {$literal: "a.$b"}}]}}}], - expectedResults: [{_id: 1, new: 5}, {_id: 2, new: null}] - }); -}()); - -// There is a difference between the metaField "meta1", and "$meta1". Field paths are allowed to -// have a '$' and are accessed by $getField. We need to validate we return the correct results when -// we have both the metaField and "$meta1". -(function testDollarMeta_LiteralExpr() { - runTest({ - docs: [ - {_id: 1, [timeField]: new Date(), [metaField]: 2}, - {_id: 2, [timeField]: new Date(), [metaField]: 2, "$meta1": 3} // missing field. - ], - pipeline: [{$project: {new: {$add: [`$${metaField}`, {$getField: {$literal: "$meta1"}}]}}}], - expectedResults: [{_id: 1, new: null}, {_id: 2, new: 5}] - }); -})(); - -// The following tests confirm the behavior of queries in the form: {$getField: {input: "string", -// field: "string"}}. -// -(function testMetaField_FieldAndInputString() { - runTest({ - docs: [ - {_id: 1, [timeField]: new Date(), [metaField]: {b: 4}}, - {_id: 2, [timeField]: new Date(), [metaField]: {c: 5}} // missing subfield. - ], - pipeline: [{$project: {new: {$getField: {input: `$${metaField}`, field: "b"}}}}], - expectedResults: [{_id: 1, new: 4}, {_id: 2}] - }); -})(); - -// Validate the correct results are returned when there is a field with '$' inside the metaField. -// The rewrite should be valid and return the correct field. -(function testMetaFieldObj_FieldAndInputString() { - runTest({ - docs: [ - {_id: 1, [timeField]: new Date(), [metaField]: {"a.$b": 4}}, - {_id: 2, [timeField]: new Date(), [metaField]: {c: 5}} // missing subfield. - ], - pipeline: [{$project: {new: {$getField: {input: `$${metaField}`, field: "a.$b"}}}}], - expectedResults: [{_id: 1, new: 4}, {_id: 2}] - }); -})(); - -// When we rely on both the metaField and a measurementField we should not perform the rewrite and -// return the correct result. -(function testDollarMeta_FieldAndInputString() { - runTest({ - docs: [ - {_id: 1, [timeField]: new Date(), [metaField]: 2, a: {"$meta1": 4}}, - {_id: 2, [timeField]: new Date(), [metaField]: 2, a: {c: 5}} // missing subfield. - ], - - pipeline: [{ - $project: { - new: { - $add: [`$${metaField}`, {$getField: {input: "$a", field: {$literal: "$meta1"}}}] - } - } - }], - expectedResults: [{_id: 1, new: 6}, {_id: 2, new: null}] - }); -})(); - -// same test as above but with $addFields and not $project. -(function testDollarMeta_FieldAndInputString_AddFields() { - const time = new Date(); - runTest({ - docs: [ - {_id: 1, [timeField]: time, [metaField]: 2, a: {"$meta1": 4}}, - {_id: 2, [timeField]: time, [metaField]: 2, a: {c: 5}} // missing subfield. - ], - pipeline: [{ - $addFields: { - new: { - $add: [`$${metaField}`, {$getField: {input: "$a", field: {$literal: "$meta1"}}}] - } - } - }], - expectedResults: [ - {[timeField]: time, [metaField]: 2, a: {"$meta1": 4}, _id: 1, new: 6}, - {[timeField]: time, [metaField]: 2, a: {c: 5}, _id: 2, new: null} - ] - }); -})(); - -// This test validates that $project with '$$ROOT' which requires the whole document returns the -// correct results. -(function testProject_WithROOT() { - const time = new Date(); - runTest({ - docs: [ - {_id: 1, [timeField]: time, [metaField]: 2, a: 2}, - {_id: 2, [timeField]: time, [metaField]: 2, b: 3} - ], - pipeline: [{$project: {new: "$$ROOT", _id: 1}}], - expectedResults: [ - {_id: 1, new: {_id: 1, [timeField]: time, [metaField]: 2, a: 2}}, - {_id: 2, new: {_id: 2, [timeField]: time, [metaField]: 2, b: 3}} - ] - }); -})(); - -(function testAddFields_WithROOT() { - const time = new Date(); - runTest({ - docs: [ - {_id: 1, [timeField]: time, [metaField]: 2, a: 2}, - {_id: 2, [timeField]: time, [metaField]: 2, b: 3} - ], - pipeline: [{$addFields: {new: "$$ROOT"}}], - expectedResults: [ - { - _id: 1, - [timeField]: time, - [metaField]: 2, - a: 2, - new: {_id: 1, [timeField]: time, [metaField]: 2, a: 2} - }, - { - _id: 2, - [timeField]: time, - [metaField]: 2, - b: 3, - new: {_id: 2, [timeField]: time, [metaField]: 2, b: 3} - } - ] - }); -})(); -})(); diff --git a/jstests/core/timeseries/timeseries_resume_after.js b/jstests/core/timeseries/timeseries_resume_after.js index 66d020d43b4..a2b31972c85 100644 --- a/jstests/core/timeseries/timeseries_resume_after.js +++ b/jstests/core/timeseries/timeseries_resume_after.js @@ -8,7 +8,6 @@ * does_not_support_stepdowns, * does_not_support_transactions, * requires_getmore, - * requires_fcv_60 * ] */ (function() { @@ -92,25 +91,5 @@ TimeseriesTest.run((insert) => { resumeToken = res.cursor.postBatchResumeToken; jsTestLog("Got resume token " + tojson(resumeToken)); - - // Test that '$_resumeAfter' fails if the recordId is Long. - assert.commandFailedWithCode(db.runCommand({ - find: bucketsColl.getName(), - filter: {}, - $_requestResumeToken: true, - $_resumeAfter: {'$recordId': NumberLong(10)}, - hint: {$natural: 1} - }), - 7738600); - - // Test that '$_resumeAfter' fails if querying the time-series view. - assert.commandFailedWithCode(db.runCommand({ - find: coll.getName(), - filter: {}, - $_requestResumeToken: true, - $_resumeAfter: {'$recordId': BinData(5, '1234')}, - hint: {$natural: 1} - }), - ErrorCodes.InvalidPipelineOperator); }); })(); diff --git a/jstests/core/timeseries/timeseries_sparse_index.js b/jstests/core/timeseries/timeseries_sparse_index.js index 7c9f7556e38..9ffd008ff62 100644 --- a/jstests/core/timeseries/timeseries_sparse_index.js +++ b/jstests/core/timeseries/timeseries_sparse_index.js @@ -13,9 +13,8 @@ "use strict"; load("jstests/core/timeseries/libs/timeseries.js"); -load("jstests/libs/feature_flag_util.js"); -if (!FeatureFlagUtil.isEnabled(db, "TimeseriesMetricIndexes")) { +if (!TimeseriesTest.timeseriesMetricIndexesEnabled(db.getMongo())) { jsTestLog( "Skipped test as the featureFlagTimeseriesMetricIndexes feature flag is not enabled."); return; diff --git a/jstests/core/timeseries/timeseries_streaming_group.js b/jstests/core/timeseries/timeseries_streaming_group.js deleted file mode 100644 index 6d97eb51621..00000000000 --- a/jstests/core/timeseries/timeseries_streaming_group.js +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Tests that the $group stage will be replaced with $_internalStreamingGroup when group id is - * monotonic on time and documents are sorted on time. - * - * @tags: [ - * # Explain of a resolved view must be executed by mongos. - * directly_against_shardsvrs_incompatible, - * does_not_support_stepdowns, - * does_not_support_transactions, - * requires_fcv_60, - * requires_timeseries, - * ] - */ -(function() { -"use strict"; - -load("jstests/libs/analyze_plan.js"); // For getAggPlanStages -load("jstests/libs/fail_point_util.js"); // For configureFailPoint - -const ts = db.timeseries_streaming_group; -ts.drop(); - -const coll = db.timeseires_streaming_group_regular_collection; -coll.drop(); - -assert.commandWorked( - db.createCollection(ts.getName(), {timeseries: {timeField: "time", metaField: "meta"}})); - -const numTimes = 100; -const numSymbols = 10; -const minPrice = 100; -const maxPrice = 200; -const minAmount = 1; -const maxAmount = 20; - -Random.setRandomSeed(1); - -const randRange = function(min, max) { - return min + Random.randInt(max - min); -}; - -const symbols = []; -for (let i = 0; i < numSymbols; i++) { - let randomName = ""; - const randomStrLen = 5; - for (let j = 0; j < randomStrLen; j++) { - randomName += String.fromCharCode("A".charCodeAt(0) + Random.randInt(26)); - } - symbols.push(randomName); -} - -const documents = []; -const startTime = 1641027600000; -for (let i = 0; i < numTimes; i++) { - for (const symbol of symbols) { - documents.push({ - time: new Date(startTime + i * 1000), - price: randRange(minPrice, maxPrice), - amount: randRange(minAmount, maxAmount), - meta: {"symbol": symbol} - }); - } -} - -assert.commandWorked(ts.insert(documents)); -assert.commandWorked(coll.insert(documents)); - -// Incorrect use of $_internalStreamingGroup should return error -assert.commandFailedWithCode(db.runCommand({ - aggregate: "timeseires_streaming_group_regular_collection", - pipeline: [{ - $_internalStreamingGroup: { - _id: {symbol: "$symbol", time: "$time"}, - count: {$sum: 1}, - $monotonicIdFields: ["price"] - } - }], - cursor: {}, -}), - 7026705); -assert.commandFailedWithCode(db.runCommand({ - aggregate: "timeseires_streaming_group_regular_collection", - pipeline: [{$_internalStreamingGroup: {_id: null, count: {$sum: 1}}}], - cursor: {}, -}), - 7026702); -assert.commandFailedWithCode(db.runCommand({ - aggregate: "timeseires_streaming_group_regular_collection", - pipeline: - [{$_internalStreamingGroup: {_id: null, count: {$sum: 1}, $monotonicIdFields: ["_id"]}}], - cursor: {}, -}), - 7026708); - -const runTest = function(pipeline, expectedMonotonicIdFields) { - const explain = assert.commandWorked(ts.explain().aggregate(pipeline)); - const streamingGroupStage = getAggPlanStage(explain, "$_internalStreamingGroup"); - assert.neq(streamingGroupStage, null); - assert.eq(streamingGroupStage.$_internalStreamingGroup.$monotonicIdFields, - expectedMonotonicIdFields); - - const found = ts.aggregate(pipeline).toArray(); - const expected = coll.aggregate(pipeline).toArray(); - assert.eq(expected, found); -}; - -runTest( - [ - {$sort: {time: 1}}, - { - $group: { - _id: { - symbol: "$meta.symbol", - minute: { - $subtract: [ - {$dateTrunc: {date: "$time", unit: "minute"}}, - {$dateTrunc: {date: new Date(startTime), unit: "minute"}}, - ] - } - }, - "average_price": {$avg: {$multiply: ["$price", "$amount"]}}, - "average_amount": {$avg: "$amount"} - } - }, - {$addFields: {"average_price": {$divide: ["$average_price", "$average_amount"]}}}, - {$sort: {_id: 1}} - ], - ["minute"]); - -runTest( - [ - {$sort: {time: 1}}, - { - $group: { - _id: {$dateTrunc: {date: "$time", unit: "minute"}}, - "average_price": {$avg: {$multiply: ["$price", "$amount"]}}, - "average_amount": {$avg: "$amount"} - } - }, - {$addFields: {"average_price": {$divide: ["$average_price", "$average_amount"]}}}, - {$sort: {_id: 1}} - ], - ["_id"]); - -runTest( - [ - {$sort: {time: 1}}, - { - $group: { - _id: "$time", - "average_price": {$avg: {$multiply: ["$price", "$amount"]}}, - "average_amount": {$avg: "$amount"} - } - }, - {$addFields: {"average_price": {$divide: ["$average_price", "$average_amount"]}}}, - {$sort: {_id: 1}} - ], - ["_id"]); -})(); diff --git a/jstests/core/timeseries/timeseries_union_with.js b/jstests/core/timeseries/timeseries_union_with.js index e92e4c4ca37..b78bbd6937c 100644 --- a/jstests/core/timeseries/timeseries_union_with.js +++ b/jstests/core/timeseries/timeseries_union_with.js @@ -4,11 +4,6 @@ * @tags: [ * does_not_support_transactions, * requires_timeseries, - * # This test depends on certain writes ending up in the same bucket. Stepdowns and tenant - * # migrations may result in writes splitting between two primaries, and thus different buckets. - * does_not_support_stepdowns, - * tenant_migration_incompatible, - * references_foreign_collection, * ] */ (function() { diff --git a/jstests/core/timeseries/timeseries_update_mixed_schema_bucket.js b/jstests/core/timeseries/timeseries_update_mixed_schema_bucket.js deleted file mode 100644 index 0211daa29d8..00000000000 --- a/jstests/core/timeseries/timeseries_update_mixed_schema_bucket.js +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Tests directly updating a time-series bucket to contain mixed schema. - * - * @tags: [ - * # $listCatalog does not include the tenant prefix in its results. - * command_not_supported_in_serverless, - * requires_timeseries, - * # $listCatalog not supported inside of a multi-document transaction - * does_not_support_transactions, - * # $listCatalog only exists since v6 - * multiversion_incompatible, - * ] - */ -(function() { -"use strict"; -load("jstests/core/timeseries/libs/timeseries.js"); // For 'TimeseriesTest'. - -TestData.skipEnforceTimeseriesBucketsAreAlwaysCompressedOnValidate = true; - -const testDB = db.getSiblingDB(jsTestName()); -const collName = "ts"; - -assert.commandWorkedOrFailedWithCode(testDB.runCommand({drop: collName}), - ErrorCodes.NamespaceNotFound); -assert.commandWorked( - testDB.createCollection(collName, {timeseries: {timeField: "t", metaField: "m"}})); -const coll = testDB[collName]; -const bucketsColl = testDB["system.buckets." + collName]; - -const bucket = { - _id: ObjectId("65a6eb806ffc9fa4280ecac4"), - control: { - version: NumberInt(1), - min: { - _id: ObjectId("65a6eba7e6d2e848e08c3750"), - t: ISODate("2024-01-16T20:48:00Z"), - a: 0, - }, - max: { - _id: ObjectId("65a6eba7e6d2e848e08c3751"), - t: ISODate("2024-01-16T20:48:39.448Z"), - a: 1, - }, - }, - meta: 0, - data: { - _id: { - 0: ObjectId("65a6eba7e6d2e848e08c3750"), - 1: ObjectId("65a6eba7e6d2e848e08c3751"), - }, - t: { - 0: ISODate("2024-01-16T20:48:39.448Z"), - 1: ISODate("2024-01-16T20:48:39.448Z"), - }, - a: { - 0: 0, - 1: 1, - }, - } -}; - -const update = function() { - return bucketsColl.update({_id: bucket._id}, - {$set: {"control.min.a": 1, "control.max.a": "a", "data.a.0": "a"}}); -}; - -assert.commandWorked(bucketsColl.insert(bucket)); -assert.commandFailedWithCode(update(), ErrorCodes.CannotInsertTimeseriesBucketsWithMixedSchema); -assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), false); -assert.commandWorked( - testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: true})); -assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), true); -assert.commandWorked(update()); -assert.commandWorked(bucketsColl.deleteOne({_id: bucket._id})); -assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), true); -assert.commandWorked( - testDB.runCommand({collMod: collName, timeseriesBucketsMayHaveMixedSchemaData: false})); -assert.eq(TimeseriesTest.bucketsMayHaveMixedSchemaData(bucketsColl), false); -})(); |