diff options
| author | Alyssa Wagenmaker <alyssa.wagenmaker@mongodb.com> | 2023-09-01 18:02:11 +0000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-09-01 18:43:21 +0000 |
| commit | 8e4b5670df9b9fe814e57cb5f3f8ee9407237b5a (patch) | |
| tree | 2c2f75187377c70d9772c5fac171470fc2b8f7b0 /src/mongo/db/exec | |
| parent | 69353845c8a545dc5a763ee199d9002e7cb95e6c (diff) | |
SERVER-79498 Introduce vectorSearch aggregation stager6.0.10-rc0r6.0.10
Co-authored-by: Chi-I Huang <chii.huang@mongodb.com>
Co-authored-by: Hima Sheth <hima.sheth@mongodb.com>
Diffstat (limited to 'src/mongo/db/exec')
8 files changed, 84 insertions, 12 deletions
diff --git a/src/mongo/db/exec/document_value/document.cpp b/src/mongo/db/exec/document_value/document.cpp index fd50895c9a1..4488af81f3b 100644 --- a/src/mongo/db/exec/document_value/document.cpp +++ b/src/mongo/db/exec/document_value/document.cpp @@ -90,7 +90,8 @@ const StringDataSet Document::allMetadataFieldNames{Document::metaFieldTextScore Document::metaFieldSearchHighlights, Document::metaFieldSearchSortValues, Document::metaFieldIndexKey, - Document::metaFieldSearchScoreDetails}; + Document::metaFieldSearchScoreDetails, + Document::metaFieldVectorSearchScore}; DocumentStorageIterator::DocumentStorageIterator(DocumentStorage* storage, BSONObjIterator bsonIt) : _bsonIt(std::move(bsonIt)), @@ -467,6 +468,8 @@ void DocumentStorage::loadLazyMetadata() const { _metadataFields.setSearchScoreDetails(elem.Obj()); } else if (fieldName == Document::metaFieldSearchSortValues) { _metadataFields.setSearchSortValues(elem.Obj()); + } else if (fieldName == Document::metaFieldVectorSearchScore) { + _metadataFields.setVectorSearchScore(elem.Double()); } } } @@ -537,6 +540,7 @@ constexpr StringData Document::metaFieldSearchScore; constexpr StringData Document::metaFieldSearchHighlights; constexpr StringData Document::metaFieldSearchScoreDetails; constexpr StringData Document::metaFieldSearchSortValues; +constexpr StringData Document::metaFieldVectorSearchScore; void Document::toBsonWithMetaData(BSONObjBuilder* builder) const { toBson(builder); @@ -567,6 +571,9 @@ void Document::toBsonWithMetaData(BSONObjBuilder* builder) const { if (metadata().hasSearchSortValues()) { builder->append(metaFieldSearchSortValues, metadata().getSearchSortValues()); } + if (metadata().hasVectorSearchScore()) { + builder->append(metaFieldVectorSearchScore, metadata().getVectorSearchScore()); + } } Document Document::fromBsonWithMetaData(const BSONObj& bson) { diff --git a/src/mongo/db/exec/document_value/document.h b/src/mongo/db/exec/document_value/document.h index ba4a50481bc..062778628a0 100644 --- a/src/mongo/db/exec/document_value/document.h +++ b/src/mongo/db/exec/document_value/document.h @@ -102,6 +102,7 @@ public: static constexpr StringData metaFieldSearchScoreDetails = "$searchScoreDetails"_sd; static constexpr StringData metaFieldSearchSortValues = "$searchSortValues"_sd; static constexpr StringData metaFieldIndexKey = "$indexKey"_sd; + static constexpr StringData metaFieldVectorSearchScore = "$vectorSearchScore"_sd; static const StringDataSet allMetadataFieldNames; diff --git a/src/mongo/db/exec/document_value/document_metadata_fields.cpp b/src/mongo/db/exec/document_value/document_metadata_fields.cpp index d33eee6c140..0532ed2e8cb 100644 --- a/src/mongo/db/exec/document_value/document_metadata_fields.cpp +++ b/src/mongo/db/exec/document_value/document_metadata_fields.cpp @@ -96,6 +96,9 @@ void DocumentMetadataFields::mergeWith(const DocumentMetadataFields& other) { if (!hasSearchSortValues() && other.hasSearchSortValues()) { setSearchSortValues(other.getSearchSortValues()); } + if (!hasVectorSearchScore() && other.hasVectorSearchScore()) { + setVectorSearchScore(other.getVectorSearchScore()); + } } void DocumentMetadataFields::copyFrom(const DocumentMetadataFields& other) { @@ -135,6 +138,9 @@ void DocumentMetadataFields::copyFrom(const DocumentMetadataFields& other) { if (other.hasSearchSortValues()) { setSearchSortValues(other.getSearchSortValues()); } + if (other.hasVectorSearchScore()) { + setVectorSearchScore(other.getVectorSearchScore()); + } } size_t DocumentMetadataFields::getApproximateSize() const { @@ -216,6 +222,10 @@ void DocumentMetadataFields::serializeForSorter(BufBuilder& buf) const { buf.appendNum(static_cast<char>(MetaType::kSearchSortValues + 1)); getSearchSortValues().appendSelfToBufBuilder(buf); } + if (hasVectorSearchScore()) { + buf.appendNum(static_cast<char>(MetaType::kVectorSearchScore + 1)); + buf.appendNum(getVectorSearchScore()); + } buf.appendNum(static_cast<char>(0)); } @@ -256,6 +266,8 @@ void DocumentMetadataFields::deserializeForSorter(BufReader& buf, DocumentMetada } else if (marker == static_cast<char>(MetaType::kSearchSortValues) + 1) { out->setSearchSortValues( BSONObj::deserializeForSorter(buf, BSONObj::SorterDeserializeSettings())); + } else if (marker == static_cast<char>(MetaType::kVectorSearchScore) + 1) { + out->setVectorSearchScore(buf.read<LittleEndian<double>>()); } else { uasserted(28744, "Unrecognized marker, unable to deserialize buffer"); } @@ -317,6 +329,8 @@ const char* DocumentMetadataFields::typeNameToDebugString(DocumentMetadataFields return "timeseries bucket max time"; case DocumentMetadataFields::kSearchSortValues: return "$search sort values"; + case DocumentMetadataFields::kVectorSearchScore: + return "$vectorSearch distance"; default: MONGO_UNREACHABLE; } diff --git a/src/mongo/db/exec/document_value/document_metadata_fields.h b/src/mongo/db/exec/document_value/document_metadata_fields.h index ede2f73229d..e8451677ea7 100644 --- a/src/mongo/db/exec/document_value/document_metadata_fields.h +++ b/src/mongo/db/exec/document_value/document_metadata_fields.h @@ -68,6 +68,7 @@ public: kTimeseriesBucketMinTime, kTimeseriesBucketMaxTime, kSearchSortValues, + kVectorSearchScore, // New fields must be added before the kNumFields sentinel. kNumFields @@ -330,6 +331,20 @@ public: _holder->searchSortValues = vals.getOwned(); } + bool hasVectorSearchScore() const { + return _holder && _holder->metaFields.test(MetaType::kVectorSearchScore); + } + + double getVectorSearchScore() const { + tassert(7828400, "vectorSearchScore must be present in metadata", hasVectorSearchScore()); + return _holder->vectorSearchScore; + } + + void setVectorSearchScore(double vectorSearchScore) { + _setCommon(MetaType::kVectorSearchScore); + _holder->vectorSearchScore = vectorSearchScore; + } + void serializeForSorter(BufBuilder& buf) const; bool isModified() const { @@ -376,6 +391,7 @@ private: Date_t timeseriesBucketMinTime; Date_t timeseriesBucketMaxTime; BSONObj searchSortValues; + double vectorSearchScore{0.0}; }; // Null until the first setter is called, at which point a MetadataHolder struct is allocated. diff --git a/src/mongo/db/exec/document_value/document_metadata_fields_test.cpp b/src/mongo/db/exec/document_value/document_metadata_fields_test.cpp index 4a76154f1f6..23ab2277c96 100644 --- a/src/mongo/db/exec/document_value/document_metadata_fields_test.cpp +++ b/src/mongo/db/exec/document_value/document_metadata_fields_test.cpp @@ -52,6 +52,7 @@ TEST(DocumentMetadataFieldsTest, AllMetadataRoundtripsThroughSerialization) { metadata.setSearchScoreDetails(BSON("scoreDetails" << "foo")); metadata.setSearchSortValues(BSON("a" << 1)); + metadata.setVectorSearchScore(7.6); BufBuilder builder; metadata.serializeForSorter(builder); @@ -72,6 +73,7 @@ TEST(DocumentMetadataFieldsTest, AllMetadataRoundtripsThroughSerialization) { BSON("scoreDetails" << "foo")); ASSERT_BSONOBJ_EQ(deserialized.getSearchSortValues(), BSON("a" << 1)); + ASSERT_EQ(deserialized.getVectorSearchScore(), 7.6); } TEST(DocumentMetadataFieldsTest, HasMethodsReturnFalseForEmptyMetadata) { @@ -87,6 +89,7 @@ TEST(DocumentMetadataFieldsTest, HasMethodsReturnFalseForEmptyMetadata) { ASSERT_FALSE(metadata.hasIndexKey()); ASSERT_FALSE(metadata.hasSearchScoreDetails()); ASSERT_FALSE(metadata.hasSearchSortValues()); + ASSERT_FALSE(metadata.hasVectorSearchScore()); } TEST(DocumentMetadataFieldsTest, HasMethodsReturnTrueForInitializedMetadata) { @@ -133,6 +136,10 @@ TEST(DocumentMetadataFieldsTest, HasMethodsReturnTrueForInitializedMetadata) { ASSERT_FALSE(metadata.hasSearchSortValues()); metadata.setSearchSortValues(BSON("a" << 1)); ASSERT_TRUE(metadata.hasSearchSortValues()); + + ASSERT_FALSE(metadata.hasVectorSearchScore()); + metadata.setVectorSearchScore(7.6); + ASSERT_TRUE(metadata.hasVectorSearchScore()); } TEST(DocumentMetadataFieldsTest, MoveConstructor) { @@ -148,6 +155,7 @@ TEST(DocumentMetadataFieldsTest, MoveConstructor) { metadata.setSearchScoreDetails(BSON("scoreDetails" << "foo")); metadata.setSearchSortValues(BSON("a" << 1)); + metadata.setVectorSearchScore(7.6); DocumentMetadataFields moveConstructed(std::move(metadata)); ASSERT_TRUE(moveConstructed); @@ -164,6 +172,7 @@ TEST(DocumentMetadataFieldsTest, MoveConstructor) { BSON("scoreDetails" << "foo")); ASSERT_BSONOBJ_EQ(moveConstructed.getSearchSortValues(), BSON("a" << 1)); + ASSERT_EQ(moveConstructed.getVectorSearchScore(), 7.6); ASSERT_FALSE(metadata); // NOLINT(bugprone-use-after-move) } @@ -181,6 +190,7 @@ TEST(DocumentMetadataFieldsTest, MoveAssignmentOperator) { metadata.setSearchScoreDetails(BSON("scoreDetails" << "foo")); metadata.setSearchSortValues(BSON("a" << 1)); + metadata.setVectorSearchScore(7.6); DocumentMetadataFields moveAssigned; moveAssigned.setTextScore(12.3); @@ -200,6 +210,7 @@ TEST(DocumentMetadataFieldsTest, MoveAssignmentOperator) { BSON("scoreDetails" << "foo")); ASSERT_BSONOBJ_EQ(moveAssigned.getSearchSortValues(), BSON("a" << 1)); + ASSERT_EQ(moveAssigned.getVectorSearchScore(), 7.6); ASSERT_FALSE(metadata); // NOLINT(bugprone-use-after-move) } @@ -254,6 +265,7 @@ TEST(DocumentMetadataFieldsTest, MergeWithOnlyCopiesMetadataThatDestinationDoesN ASSERT_FALSE(destination.hasIndexKey()); ASSERT_FALSE(destination.hasSearchScoreDetails()); ASSERT_FALSE(destination.hasSearchSortValues()); + ASSERT_FALSE(destination.hasVectorSearchScore()); } TEST(DocumentMetadataFieldsTest, CopyFromCopiesAllMetadataThatSourceHas) { @@ -280,6 +292,7 @@ TEST(DocumentMetadataFieldsTest, CopyFromCopiesAllMetadataThatSourceHas) { ASSERT_FALSE(destination.hasIndexKey()); ASSERT_FALSE(destination.hasSearchScoreDetails()); ASSERT_FALSE(destination.hasSearchSortValues()); + ASSERT_FALSE(destination.hasVectorSearchScore()); } TEST(DocumentMetadataFieldsTest, GetTimeseriesBucketMinTimeExists) { @@ -323,6 +336,7 @@ TEST(DocumentMetadataFieldsTest, MetadataIsMarkedModifiedOnSetMetadataField) { testFieldSetter([](DocumentMetadataFields& md) { md.setTimeseriesBucketMinTime(Date_t()); }); testFieldSetter([](DocumentMetadataFields& md) { md.setTimeseriesBucketMaxTime(Date_t()); }); testFieldSetter([](DocumentMetadataFields& md) { md.setSearchSortValues(BSON("a" << 1)); }); + testFieldSetter([](DocumentMetadataFields& md) { md.setVectorSearchScore(60.0); }); } TEST(DocumentMetadataFieldsTest, MetadataIsConstructedUnmodified) { diff --git a/src/mongo/db/exec/document_value/document_value_test.cpp b/src/mongo/db/exec/document_value/document_value_test.cpp index 1b5c833f88b..ecc550f6ee0 100644 --- a/src/mongo/db/exec/document_value/document_value_test.cpp +++ b/src/mongo/db/exec/document_value/document_value_test.cpp @@ -907,7 +907,7 @@ TEST(MetaFields, CopyMetadataFromCopiesAllMetadata) { << "h" << 1 << "$indexKey" << BSON("y" << 1) << "$searchScoreDetails" << BSON("scoreDetails" << "foo") - << "$searchSortValues" << BSON("a" << 1))); + << "$searchSortValues" << BSON("a" << 1) << "$vectorSearchScore" << 6.7)); MutableDocument destination{}; destination.copyMetaDataFrom(source); @@ -925,6 +925,7 @@ TEST(MetaFields, CopyMetadataFromCopiesAllMetadata) { BSON("scoreDetails" << "foo")); ASSERT_BSONOBJ_EQ(result.metadata().getSearchSortValues(), BSON("a" << 1)); + ASSERT_EQ(result.metadata().getVectorSearchScore(), 6.7); } class SerializationTest : public unittest::Test { @@ -945,6 +946,8 @@ protected: ASSERT_EQ(output.metadata().hasSearchScore(), input.metadata().hasSearchScore()); ASSERT_EQ(output.metadata().hasSearchHighlights(), input.metadata().hasSearchHighlights()); ASSERT_EQ(output.metadata().hasIndexKey(), input.metadata().hasIndexKey()); + ASSERT_EQ(output.metadata().hasVectorSearchScore(), + input.metadata().hasVectorSearchScore()); if (input.metadata().hasTextScore()) { ASSERT_EQ(output.metadata().getTextScore(), input.metadata().getTextScore()); } @@ -965,6 +968,10 @@ protected: ASSERT_BSONOBJ_EQ(output.metadata().getSearchScoreDetails(), input.metadata().getSearchScoreDetails()); } + if (input.metadata().hasVectorSearchScore()) { + ASSERT_EQ(output.metadata().getVectorSearchScore(), + input.metadata().getVectorSearchScore()); + } ASSERT(output.toBson().binaryEqual(input.toBson())); } @@ -979,6 +986,7 @@ TEST_F(SerializationTest, MetaSerializationNoVals) { << "def"_sd)); docBuilder.metadata().setSearchScoreDetails(BSON("scoreDetails" << "foo")); + docBuilder.metadata().setVectorSearchScore(40.0); assertRoundTrips(docBuilder.freeze()); } @@ -993,6 +1001,7 @@ TEST_F(SerializationTest, MetaSerializationWithVals) { docBuilder.metadata().setIndexKey(BSON("key" << 42)); docBuilder.metadata().setSearchScoreDetails(BSON("scoreDetails" << "foo")); + docBuilder.metadata().setVectorSearchScore(40.0); assertRoundTrips(docBuilder.freeze()); } @@ -1016,6 +1025,7 @@ TEST(MetaFields, ToAndFromBson) { docBuilder.metadata().setSearchScoreDetails(BSON("scoreDetails" << "foo")); docBuilder.metadata().setSearchSortValues(BSON("a" << 42)); + docBuilder.metadata().setVectorSearchScore(40.0); Document doc = docBuilder.freeze(); BSONObj obj = doc.toBsonWithMetaData(); ASSERT_EQ(10.0, obj[Document::metaFieldTextScore].Double()); @@ -1028,6 +1038,7 @@ TEST(MetaFields, ToAndFromBson) { BSON("scoreDetails" << "foo")); ASSERT_BSONOBJ_EQ(BSON("a" << 42), obj[Document::metaFieldSearchSortValues].Obj()); + ASSERT_EQ(40.0, obj[Document::metaFieldVectorSearchScore].Double()); Document fromBson = Document::fromBsonWithMetaData(obj); ASSERT_TRUE(fromBson.metadata().hasTextScore()); ASSERT_TRUE(fromBson.metadata().hasRandVal()); @@ -1037,6 +1048,7 @@ TEST(MetaFields, ToAndFromBson) { << "foo"), fromBson.metadata().getSearchScoreDetails()); ASSERT_BSONOBJ_EQ(BSON("a" << 42), fromBson.metadata().getSearchSortValues()); + ASSERT_EQ(40.0, fromBson.metadata().getVectorSearchScore()); } TEST(MetaFields, ToAndFromBsonTrivialConvertibility) { diff --git a/src/mongo/db/exec/exclusion_projection_executor_test.cpp b/src/mongo/db/exec/exclusion_projection_executor_test.cpp index 983c5e1995a..c3866ef865f 100644 --- a/src/mongo/db/exec/exclusion_projection_executor_test.cpp +++ b/src/mongo/db/exec/exclusion_projection_executor_test.cpp @@ -363,7 +363,8 @@ TEST(ExclusionProjectionExecutionTest, ShouldEvaluateMetaExpressions) { "i: {$meta: 'recordId'}, " "j: {$meta: 'indexKey'}, " "k: {$meta: 'sortKey'}, " - "l: {$meta: 'searchScoreDetails'}}")); + "l: {$meta: 'searchScoreDetails'}, " + "m: {$meta: 'vectorSearchScore'}}")); MutableDocument inputDocBuilder(Document{{"a", 1}, {"b", 2}}); inputDocBuilder.metadata().setTextScore(0.0); @@ -377,6 +378,7 @@ TEST(ExclusionProjectionExecutionTest, ShouldEvaluateMetaExpressions) { inputDocBuilder.metadata().setSortKey(Value{Document{{"bar", 8}}}, true); inputDocBuilder.metadata().setSearchScoreDetails(BSON("scoreDetails" << "foo")); + inputDocBuilder.metadata().setVectorSearchScore(9.0); Document inputDoc = inputDocBuilder.freeze(); auto result = exclusion->applyTransformation(inputDoc); @@ -384,7 +386,7 @@ TEST(ExclusionProjectionExecutionTest, ShouldEvaluateMetaExpressions) { ASSERT_DOCUMENT_EQ(result, Document{fromjson("{b: 2, c: 0.0, d: 1.0, e: 2.0, f: 'foo', g: 3.0, " "h: [4, 5], i: 6, j: {foo: 7}, k: [{bar: 8}]," - "l: {scoreDetails: 'foo'}}")}); + "l: {scoreDetails: 'foo'}, m: 9.0}")}); } TEST(ExclusionProjectionExecutionTest, ShouldAddMetaExpressionsToDependencies) { @@ -398,18 +400,20 @@ TEST(ExclusionProjectionExecutionTest, ShouldAddMetaExpressionsToDependencies) { "i: {$meta: 'recordId'}, " "j: {$meta: 'indexKey'}, " "k: {$meta: 'sortKey'}, " - "l: {$meta: 'searchScoreDetails'}}")); + "l: {$meta: 'searchScoreDetails'}}, " + "m: {$meta: 'vectorSearchScore'}")); DepsTracker deps; exclusion->addDependencies(&deps); ASSERT_EQ(deps.fields.size(), 0UL); - // We do not add the dependencies for searchScore, searchHighlights, or searchScoreDetails - // because those values are not stored in the collection (or in mongod at all). + // We do not add the dependencies for searchScore, searchHighlights, searchScoreDetails, or + // distance because those values are not stored in the collection (or in mongod at all). ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchScore]); ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchHighlights]); ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchScoreDetails]); + ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kVectorSearchScore]); ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kTextScore]); ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kRandVal]); diff --git a/src/mongo/db/exec/inclusion_projection_executor_test.cpp b/src/mongo/db/exec/inclusion_projection_executor_test.cpp index 53ebf3af4a3..b31899043ca 100644 --- a/src/mongo/db/exec/inclusion_projection_executor_test.cpp +++ b/src/mongo/db/exec/inclusion_projection_executor_test.cpp @@ -814,18 +814,20 @@ TEST_F(InclusionProjectionExecutionTestWithFallBackToDefault, "i: {$meta: 'recordId'}, " "j: {$meta: 'indexKey'}, " "k: {$meta: 'sortKey'}, " - "l: {$meta: 'searchScoreDetails'}}")); + "l: {$meta: 'searchScoreDetails'}}, " + "m: {$meta: 'vectorSearchScore'}")); DepsTracker deps; inclusion->addDependencies(&deps); ASSERT_EQ(deps.fields.size(), 2UL); - // We do not add the dependencies for searchScore, searchHighlights, or searchScoreDetails - // because those values are not stored in the collection (or in mongod at all). + // We do not add the dependencies for searchScore, searchHighlights, searchScoreDetails, or + // distance because those values are not stored in the collection (or in mongod at all). ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchScore]); ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchHighlights]); ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kSearchScoreDetails]); + ASSERT_FALSE(deps.metadataDeps()[DocumentMetadataFields::kVectorSearchScore]); ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kTextScore]); ASSERT_TRUE(deps.metadataDeps()[DocumentMetadataFields::kRandVal]); @@ -847,7 +849,8 @@ TEST_F(InclusionProjectionExecutionTestWithFallBackToDefault, ShouldEvaluateMeta "i: {$meta: 'recordId'}, " "j: {$meta: 'indexKey'}, " "k: {$meta: 'sortKey'}, " - "l: {$meta: 'searchScoreDetails'}}")); + "l: {$meta: 'searchScoreDetails'}, " + "m: {$meta: 'vectorSearchScore'}}")); MutableDocument inputDocBuilder(Document{{"a", 1}}); inputDocBuilder.metadata().setTextScore(0.0); @@ -861,6 +864,7 @@ TEST_F(InclusionProjectionExecutionTestWithFallBackToDefault, ShouldEvaluateMeta inputDocBuilder.metadata().setSortKey(Value{Document{{"bar", 8}}}, true); inputDocBuilder.metadata().setSearchScoreDetails(BSON("scoreDetails" << "foo")); + inputDocBuilder.metadata().setVectorSearchScore(9.0); Document inputDoc = inputDocBuilder.freeze(); auto result = inclusion->applyTransformation(inputDoc); @@ -868,7 +872,7 @@ TEST_F(InclusionProjectionExecutionTestWithFallBackToDefault, ShouldEvaluateMeta ASSERT_DOCUMENT_EQ(result, Document{fromjson("{a: 1, c: 0.0, d: 1.0, e: 2.0, f: 'foo', g: 3.0, " "h: [4, 5], i: 6, j: {foo: 7}, k: [{bar: 8}], " - "l: {scoreDetails: 'foo'}}")}); + "l: {scoreDetails: 'foo'}, m: 9.0}")}); } // |
