diff --git a/.github/workflows/generate-pr-description.yml b/.github/workflows/generate-pr-description.yml index 70261481efe0..b34908a32d4e 100644 --- a/.github/workflows/generate-pr-description.yml +++ b/.github/workflows/generate-pr-description.yml @@ -13,7 +13,7 @@ jobs: if: github.event.issue.pull_request && contains(github.event.comment.body, '/sk generate-pr-description') steps: - name: Get PR branch - uses: xt0rted/pull-request-comment-branch@v1 + uses: xt0rted/pull-request-comment-branch@v3 id: comment-branch - name: Set latest commit status as pending diff --git a/README.md b/README.md index 1d9cf8f3c6cf..cb6bedfd1832 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ feature parity between our currently supported languages. Java logo
- Using Semantic Kernel in Java + Using Semantic Kernel in Java
diff --git a/docs/decisions/0046-java-repository-separation.md b/docs/decisions/0046-java-repository-separation.md new file mode 100644 index 000000000000..48008bbd28e1 --- /dev/null +++ b/docs/decisions/0046-java-repository-separation.md @@ -0,0 +1,50 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +contact: John Oliver +date: 2024-06-18 +--- + +# Separate Java Repository To a Separate Code Base + +## Context and Problem Statement + +Managing multiple languages within a single repository provides some challenges with respect to how different languages and their build tools +manage repositories. Particularly with respect to how common build tooling for Java, like Apache Maven, interacts with repositories. Typically, +while doing a Maven release you want to be able to freeze your repository so that commits are not being added while +preparing a release. To achieve this in a shared repository we would effectively need to request all languages halt +merging pull requests while we are in this process. The Maven release process also interacts badly with the projects +desire for merges to be squashed which for the most part blocks a typical Maven release process that needs to push +multiple commits into a repository. + +Additionally, from a discoverability standpoint, in the original repository the majority of current pull requests, issues and activity are from +other languages. This has created some +confusion from users about if the semantic kernel repository is the correct repository for Java. Managing git history +when performing tasks such as looking +at diffs or compiling release notes is also significantly harder when the majority of commits and code are unrelated to Java. + +Also managing repository policies that are preferred by all languages is a challenge as we have to produce a more +complex build process to account for building multiple languages. If a user makes accidental changes to the repository outside their own language, +or make changes to the common files, require sign off from other languages, leading to delays as we +require review from users in other languages. Similarly common files such as GitHub Actions workflows, `.gitignore`, VS Code settings, `README.md`, `.editorconfig` etc, become +more complex as they have to simutaniously support multiple languages. + +In a community point of view, having a separate repo will foster community engagement, allowing developers to contribute, share ideas, and collaborate on the Java projects only. +Additionally, it enables transparent tracking of contributions, making it easy to identify top contributors and acknowledge their efforts. +Having a single repository will also provide valuable statistics on commits, pull requests, and other activities, helping maintainers monitor project progress and activity levels. + +## Decision Drivers + +- Allow project settings that are compatible with Java tooling +- Improve the communities' ability to discover and interact with the Java project +- Improve the ability for the community to observe changes to the Java project in isolation +- Simplify repository build/files to concentrate on a single language + +## Considered Options + +We have in the past run out of a separate branch within the [Semantic Kernel](https://github.co/microsoft/semantic-kernel) repository which solved +some of the issues however significantly hindered user discoverability as users expect to find the latest code on the main branch. + +## Decision Outcome + +Java repository has been moved to [semantic-kernel-java](https://github.com/microsoft/semantic-kernel-java) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 75bd307a175a..2e07233500c9 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -15,7 +15,7 @@ - + @@ -29,12 +29,12 @@ - + - + @@ -49,7 +49,7 @@ - + @@ -102,7 +102,7 @@ - + @@ -134,7 +134,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletion.cs b/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletion.cs index fe2f8e8c2e40..22fb6dbd82f5 100644 --- a/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletion.cs +++ b/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletion.cs @@ -107,6 +107,36 @@ public async Task ChatPromptWithInnerContentAsync() OutputInnerContent(replyInnerContent!); } + /// + /// Demonstrates how you can store the output of a chat completion request for use in the OpenAI model distillation or evals products. + /// + /// + /// This sample adds metadata to the chat completion request which allows the requests to be filtered in the OpenAI dashboard. + /// + [Fact] + public async Task ChatPromptStoreWithMetadataAsync() + { + Assert.NotNull(TestConfiguration.OpenAI.ChatModelId); + Assert.NotNull(TestConfiguration.OpenAI.ApiKey); + + StringBuilder chatPrompt = new(""" + You are a librarian, expert about books + Hi, I'm looking for book suggestions about Artificial Intelligence + """); + + var kernel = Kernel.CreateBuilder() + .AddOpenAIChatCompletion(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey) + .Build(); + + var functionResult = await kernel.InvokePromptAsync(chatPrompt.ToString(), + new(new OpenAIPromptExecutionSettings { Store = true, Metadata = new Dictionary() { { "concept", "chatcompletion" } } })); + + var messageContent = functionResult.GetValue(); // Retrieves underlying chat message content from FunctionResult. + var replyInnerContent = messageContent!.InnerContent as OpenAI.Chat.ChatCompletion; // Retrieves inner content from ChatMessageContent. + + OutputInnerContent(replyInnerContent!); + } + private async Task StartChatAsync(IChatCompletionService chatGPT) { Console.WriteLine("Chat content:"); diff --git a/dotnet/samples/Concepts/README.md b/dotnet/samples/Concepts/README.md index d53367f532d0..3d927b11baf7 100644 --- a/dotnet/samples/Concepts/README.md +++ b/dotnet/samples/Concepts/README.md @@ -39,8 +39,9 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom ### FunctionCalling - Examples on `Function Calling` with function call capable models -- [Gemini_FunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/FunctionCalling/Gemini_FunctionCalling.cs) - [FunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/FunctionCalling/FunctionCalling.cs) +- [Gemini_FunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/FunctionCalling/Gemini_FunctionCalling.cs) +- [AzureAIInference_FunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/FunctionCalling/AzureAIInference_FunctionCalling.cs) - [NexusRaven_HuggingFaceTextGeneration](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/FunctionCalling/NexusRaven_FunctionCalling.cs) - [MultipleFunctionsVsParameters](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/FunctionCalling/MultipleFunctionsVsParameters.cs) diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Settings/AzureOpenAIPromptExecutionSettingsTests.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Settings/AzureOpenAIPromptExecutionSettingsTests.cs index d8ff5b1e0d79..6b4b16c574af 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Settings/AzureOpenAIPromptExecutionSettingsTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI.UnitTests/Settings/AzureOpenAIPromptExecutionSettingsTests.cs @@ -36,6 +36,8 @@ public void ItCreatesOpenAIExecutionSettingsWithCorrectDefaults() Assert.Null(executionSettings.Logprobs); Assert.Null(executionSettings.AzureChatDataSource); Assert.Equal(maxTokensSettings, executionSettings.MaxTokens); + Assert.Null(executionSettings.Store); + Assert.Null(executionSettings.Metadata); } [Fact] @@ -54,6 +56,9 @@ public void ItUsesExistingOpenAIExecutionSettings() Logprobs = true, TopLogprobs = 5, TokenSelectionBiases = new Dictionary() { { 1, 2 }, { 3, 4 } }, + Seed = 123456, + Store = true, + Metadata = new Dictionary() { { "foo", "bar" } } }; // Act @@ -61,6 +66,14 @@ public void ItUsesExistingOpenAIExecutionSettings() // Assert Assert.Equal(actualSettings, executionSettings); + Assert.Equal(actualSettings, executionSettings); + Assert.Equal(actualSettings.MaxTokens, executionSettings.MaxTokens); + Assert.Equal(actualSettings.Logprobs, executionSettings.Logprobs); + Assert.Equal(actualSettings.TopLogprobs, executionSettings.TopLogprobs); + Assert.Equal(actualSettings.TokenSelectionBiases, executionSettings.TokenSelectionBiases); + Assert.Equal(actualSettings.Seed, executionSettings.Seed); + Assert.Equal(actualSettings.Store, executionSettings.Store); + Assert.Equal(actualSettings.Metadata, executionSettings.Metadata); } [Fact] @@ -71,7 +84,9 @@ public void ItCanUseOpenAIExecutionSettings() { ExtensionData = new Dictionary() { { "max_tokens", 1000 }, - { "temperature", 0 } + { "temperature", 0 }, + { "store", true }, + { "metadata", new Dictionary() { { "foo", "bar" } } } } }; @@ -82,6 +97,8 @@ public void ItCanUseOpenAIExecutionSettings() Assert.NotNull(executionSettings); Assert.Equal(1000, executionSettings.MaxTokens); Assert.Equal(0, executionSettings.Temperature); + Assert.True(executionSettings.Store); + Assert.Equal(new Dictionary() { { "foo", "bar" } }, executionSettings.Metadata); } [Fact] @@ -103,6 +120,8 @@ public void ItCreatesOpenAIExecutionSettingsFromExtraPropertiesSnakeCase() { "seed", 123456 }, { "logprobs", true }, { "top_logprobs", 5 }, + { "store", true }, + { "metadata", new Dictionary() { { "foo", "bar" } } } } }; @@ -131,7 +150,9 @@ public void ItCreatesOpenAIExecutionSettingsFromExtraPropertiesAsStrings() { "token_selection_biases", new Dictionary() { { "1", "2" }, { "3", "4" } } }, { "seed", 123456 }, { "logprobs", true }, - { "top_logprobs", 5 } + { "top_logprobs", 5 }, + { "store", true }, + { "metadata", new Dictionary() { { "foo", "bar" } } } } }; @@ -158,7 +179,9 @@ public void ItCreatesOpenAIExecutionSettingsFromJsonSnakeCase() "max_tokens": 128, "seed": 123456, "logprobs": true, - "top_logprobs": 5 + "top_logprobs": 5, + "store": true, + "metadata": { "foo": "bar" } } """; var actualSettings = JsonSerializer.Deserialize(json); @@ -217,7 +240,9 @@ public void PromptExecutionSettingsFreezeWorksAsExpected() "presence_penalty": 0.0, "frequency_penalty": 0.0, "stop_sequences": [ "DONE" ], - "token_selection_biases": { "1": 2, "3": 4 } + "token_selection_biases": { "1": 2, "3": 4 }, + "store": true, + "metadata": { "foo": "bar" } } """; var executionSettings = JsonSerializer.Deserialize(configPayload); @@ -232,6 +257,8 @@ public void PromptExecutionSettingsFreezeWorksAsExpected() Assert.Throws(() => executionSettings.TopP = 1); Assert.Throws(() => executionSettings.StopSequences?.Add("STOP")); Assert.Throws(() => executionSettings.TokenSelectionBiases?.Add(5, 6)); + Assert.Throws(() => executionSettings.Store = false); + Assert.Throws(() => executionSettings.Metadata?.Add("bar", "foo")); executionSettings!.Freeze(); // idempotent Assert.True(executionSettings.IsFrozen); @@ -267,7 +294,9 @@ public void ItCanCreateAzureOpenAIPromptExecutionSettingsFromOpenAIPromptExecuti Logprobs = true, Seed = 123456, TopLogprobs = 5, - ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions + ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions, + Store = true, + Metadata = new Dictionary() { { "foo", "bar" } } }; // Act @@ -307,5 +336,7 @@ private static void AssertExecutionSettings(AzureOpenAIPromptExecutionSettings e Assert.Equal(123456, executionSettings.Seed); Assert.Equal(true, executionSettings.Logprobs); Assert.Equal(5, executionSettings.TopLogprobs); + Assert.Equal(true, executionSettings.Store); + Assert.Equal(new Dictionary() { { "foo", "bar" } }, executionSettings.Metadata); } } diff --git a/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/AzureClientCore.ChatCompletion.cs b/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/AzureClientCore.ChatCompletion.cs index 63d46c7c77e2..bf7859815f1d 100644 --- a/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/AzureClientCore.ChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AzureOpenAI/Core/AzureClientCore.ChatCompletion.cs @@ -49,6 +49,7 @@ protected override ChatCompletionOptions CreateChatCompletionOptions( EndUserId = executionSettings.User, TopLogProbabilityCount = executionSettings.TopLogprobs, IncludeLogProbabilities = executionSettings.Logprobs, + StoredOutputEnabled = executionSettings.Store, }; var responseFormat = GetResponseFormat(executionSettings); @@ -90,6 +91,14 @@ protected override ChatCompletionOptions CreateChatCompletionOptions( } } + if (executionSettings.Metadata is not null) + { + foreach (var kvp in executionSettings.Metadata) + { + options.Metadata.Add(kvp.Key, kvp.Value); + } + } + if (toolCallingConfig.Options?.AllowParallelCalls is not null) { options.AllowParallelToolCalls = toolCallingConfig.Options.AllowParallelCalls; diff --git a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Settings/OpenAIPromptExecutionSettingsTests.cs b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Settings/OpenAIPromptExecutionSettingsTests.cs index 567c77babeea..90272b94717c 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Settings/OpenAIPromptExecutionSettingsTests.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Settings/OpenAIPromptExecutionSettingsTests.cs @@ -32,6 +32,8 @@ public void ItCreatesOpenAIExecutionSettingsWithCorrectDefaults() Assert.Null(executionSettings.TopLogprobs); Assert.Null(executionSettings.Logprobs); Assert.Equal(128, executionSettings.MaxTokens); + Assert.Null(executionSettings.Store); + Assert.Null(executionSettings.Metadata); } [Fact] @@ -44,12 +46,15 @@ public void ItUsesExistingOpenAIExecutionSettings() TopP = 0.7, FrequencyPenalty = 0.7, PresencePenalty = 0.7, - StopSequences = new string[] { "foo", "bar" }, + StopSequences = ["foo", "bar"], ChatSystemPrompt = "chat system prompt", MaxTokens = 128, Logprobs = true, TopLogprobs = 5, TokenSelectionBiases = new Dictionary() { { 1, 2 }, { 3, 4 } }, + Seed = 123456, + Store = true, + Metadata = new Dictionary() { { "foo", "bar" } } }; // Act @@ -58,7 +63,13 @@ public void ItUsesExistingOpenAIExecutionSettings() // Assert Assert.NotNull(executionSettings); Assert.Equal(actualSettings, executionSettings); - Assert.Equal(128, executionSettings.MaxTokens); + Assert.Equal(actualSettings.MaxTokens, executionSettings.MaxTokens); + Assert.Equal(actualSettings.Logprobs, executionSettings.Logprobs); + Assert.Equal(actualSettings.TopLogprobs, executionSettings.TopLogprobs); + Assert.Equal(actualSettings.TokenSelectionBiases, executionSettings.TokenSelectionBiases); + Assert.Equal(actualSettings.Seed, executionSettings.Seed); + Assert.Equal(actualSettings.Store, executionSettings.Store); + Assert.Equal(actualSettings.Metadata, executionSettings.Metadata); } [Fact] @@ -69,7 +80,9 @@ public void ItCanUseOpenAIExecutionSettings() { ExtensionData = new Dictionary() { { "max_tokens", 1000 }, - { "temperature", 0 } + { "temperature", 0 }, + { "store", true }, + { "metadata", new Dictionary() { { "foo", "bar" } } } } }; @@ -80,6 +93,8 @@ public void ItCanUseOpenAIExecutionSettings() Assert.NotNull(executionSettings); Assert.Equal(1000, executionSettings.MaxTokens); Assert.Equal(0, executionSettings.Temperature); + Assert.True(executionSettings.Store); + Assert.Equal(new Dictionary() { { "foo", "bar" } }, executionSettings.Metadata); } [Fact] @@ -102,6 +117,8 @@ public void ItCreatesOpenAIExecutionSettingsFromExtraPropertiesSnakeCase() { "seed", 123456 }, { "logprobs", true }, { "top_logprobs", 5 }, + { "store", true }, + { "metadata", new Dictionary() { { "foo", "bar" } } } } }; @@ -131,7 +148,9 @@ public void ItCreatesOpenAIExecutionSettingsFromExtraPropertiesAsStrings() { "token_selection_biases", new Dictionary() { { "1", "2" }, { "3", "4" } } }, { "seed", 123456 }, { "logprobs", true }, - { "top_logprobs", 5 } + { "top_logprobs", 5 }, + { "store", true }, + { "metadata", new Dictionary() { { "foo", "bar" } } } } }; @@ -159,7 +178,9 @@ public void ItCreatesOpenAIExecutionSettingsFromJsonSnakeCase() "max_tokens": 128, "seed": 123456, "logprobs": true, - "top_logprobs": 5 + "top_logprobs": 5, + "store": true, + "metadata": { "foo": "bar" } } """; var actualSettings = JsonSerializer.Deserialize(json); @@ -219,7 +240,12 @@ public void PromptExecutionSettingsFreezeWorksAsExpected() "presence_penalty": 0.0, "frequency_penalty": 0.0, "stop_sequences": [ "DONE" ], - "token_selection_biases": { "1": 2, "3": 4 } + "token_selection_biases": { "1": 2, "3": 4 }, + "seed": 123456, + "logprobs": true, + "top_logprobs": 5, + "store": true, + "metadata": { "foo": "bar" } } """; var executionSettings = JsonSerializer.Deserialize(configPayload); @@ -234,6 +260,11 @@ public void PromptExecutionSettingsFreezeWorksAsExpected() Assert.Throws(() => executionSettings.TopP = 1); Assert.Throws(() => executionSettings.StopSequences?.Add("STOP")); Assert.Throws(() => executionSettings.TokenSelectionBiases?.Add(5, 6)); + Assert.Throws(() => executionSettings.Seed = 654321); + Assert.Throws(() => executionSettings.Logprobs = false); + Assert.Throws(() => executionSettings.TopLogprobs = 10); + Assert.Throws(() => executionSettings.Store = false); + Assert.Throws(() => executionSettings.Metadata?.Add("bar", "baz")); executionSettings!.Freeze(); // idempotent Assert.True(executionSettings.IsFrozen); @@ -285,5 +316,7 @@ private static void AssertExecutionSettings(OpenAIPromptExecutionSettings execut Assert.Equal(123456, executionSettings.Seed); Assert.Equal(true, executionSettings.Logprobs); Assert.Equal(5, executionSettings.TopLogprobs); + Assert.Equal(true, executionSettings.Store); + Assert.Equal(new Dictionary() { { "foo", "bar" } }, executionSettings.Metadata); } } diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Core/ClientCore.ChatCompletion.cs b/dotnet/src/Connectors/Connectors.OpenAI/Core/ClientCore.ChatCompletion.cs index 9d03c3322964..b14e7b2f1c89 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Core/ClientCore.ChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Core/ClientCore.ChatCompletion.cs @@ -456,7 +456,8 @@ protected virtual ChatCompletionOptions CreateChatCompletionOptions( #pragma warning restore OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. EndUserId = executionSettings.User, TopLogProbabilityCount = executionSettings.TopLogprobs, - IncludeLogProbabilities = executionSettings.Logprobs + IncludeLogProbabilities = executionSettings.Logprobs, + StoredOutputEnabled = executionSettings.Store, }; var responseFormat = GetResponseFormat(executionSettings); @@ -496,6 +497,14 @@ protected virtual ChatCompletionOptions CreateChatCompletionOptions( options.AllowParallelToolCalls = toolCallingConfig.Options.AllowParallelCalls; } + if (executionSettings.Metadata is not null) + { + foreach (var kvp in executionSettings.Metadata) + { + options.Metadata.Add(kvp.Key, kvp.Value); + } + } + return options; } diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Settings/OpenAIPromptExecutionSettings.cs b/dotnet/src/Connectors/Connectors.OpenAI/Settings/OpenAIPromptExecutionSettings.cs index 3a5e632b7664..add62d564046 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Settings/OpenAIPromptExecutionSettings.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Settings/OpenAIPromptExecutionSettings.cs @@ -289,6 +289,40 @@ public int? TopLogprobs } } + /// + /// Developer-defined tags and values used for filtering completions in the OpenAI dashboard. + /// + [Experimental("SKEXP0010")] + [JsonPropertyName("metadata")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IDictionary? Metadata + { + get => this._metadata; + + set + { + this.ThrowIfFrozen(); + this._metadata = value; + } + } + + /// + /// Whether or not to store the output of this chat completion request for use in the OpenAI model distillation or evals products. + /// + [Experimental("SKEXP0010")] + [JsonPropertyName("store")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Store + { + get => this._store; + + set + { + this.ThrowIfFrozen(); + this._store = value; + } + } + /// public override void Freeze() { @@ -308,6 +342,11 @@ public override void Freeze() { this._tokenSelectionBiases = new ReadOnlyDictionary(this._tokenSelectionBiases); } + + if (this._metadata is not null) + { + this._metadata = new ReadOnlyDictionary(this._metadata); + } } /// @@ -372,7 +411,9 @@ public static OpenAIPromptExecutionSettings FromExecutionSettings(PromptExecutio User = this.User, ChatSystemPrompt = this.ChatSystemPrompt, Logprobs = this.Logprobs, - TopLogprobs = this.TopLogprobs + TopLogprobs = this.TopLogprobs, + Store = this.Store, + Metadata = this.Metadata is not null ? new Dictionary(this.Metadata) : null, }; } @@ -392,6 +433,8 @@ public static OpenAIPromptExecutionSettings FromExecutionSettings(PromptExecutio private string? _chatSystemPrompt; private bool? _logprobs; private int? _topLogprobs; + private bool? _store; + private IDictionary? _metadata; #endregion } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchHotel.cs new file mode 100644 index 000000000000..3f979fe2b828 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchHotel.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; +using Azure.Search.Documents.Indexes; +using Azure.Search.Documents.Indexes.Models; +using Microsoft.Extensions.VectorData; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.AzureAISearch; + +#pragma warning disable CS8618 + +public class AzureAISearchHotel +{ + [SimpleField(IsKey = true, IsFilterable = true)] + [VectorStoreRecordKey] + public string HotelId { get; set; } + + [SearchableField(IsFilterable = true, IsSortable = true)] + [VectorStoreRecordData(IsFilterable = true, IsFullTextSearchable = true)] + public string HotelName { get; set; } + + [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.EnLucene)] + [VectorStoreRecordData] + public string Description { get; set; } + + [VectorStoreRecordVector(1536)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } + + [SearchableField(IsFilterable = true, IsFacetable = true)] + [VectorStoreRecordData(IsFilterable = true)] +#pragma warning disable CA1819 // Properties should not return arrays + public string[] Tags { get; set; } +#pragma warning restore CA1819 // Properties should not return arrays + + [JsonPropertyName("parking_is_included")] + [SimpleField(IsFilterable = true, IsSortable = true, IsFacetable = true)] + [VectorStoreRecordData(IsFilterable = true)] + public bool? ParkingIncluded { get; set; } + + [SimpleField(IsFilterable = true, IsSortable = true, IsFacetable = true)] + [VectorStoreRecordData(IsFilterable = true)] + public DateTimeOffset? LastRenovationDate { get; set; } + + [SimpleField(IsFilterable = true, IsSortable = true, IsFacetable = true)] + [VectorStoreRecordData] + public double? Rating { get; set; } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchTextSearchTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchTextSearchTests.cs index fcac923a277b..115ae9aabff5 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchTextSearchTests.cs @@ -10,7 +10,6 @@ using SemanticKernel.IntegrationTests.Data; using SemanticKernel.IntegrationTests.TestSettings; using Xunit; -using static SemanticKernel.IntegrationTests.Connectors.Memory.AzureAISearch.AzureAISearchVectorStoreFixture; namespace SemanticKernel.IntegrationTests.Connectors.Memory.AzureAISearch; @@ -82,11 +81,11 @@ public override Task CreateTextSearchAsync() this.VectorStore = new AzureAISearchVectorStore(fixture.SearchIndexClient); } - var vectorSearch = this.VectorStore.GetCollection(fixture.TestIndexName); + var vectorSearch = this.VectorStore.GetCollection(fixture.TestIndexName); var stringMapper = new HotelTextSearchStringMapper(); var resultMapper = new HotelTextSearchResultMapper(); - var result = new VectorStoreTextSearch(vectorSearch, this.EmbeddingGenerator!, stringMapper, resultMapper); + var result = new VectorStoreTextSearch(vectorSearch, this.EmbeddingGenerator!, stringMapper, resultMapper); return Task.FromResult(result); } @@ -105,7 +104,7 @@ public override bool VerifySearchResults(object[] results, string query, TextSea foreach (var result in results) { Assert.NotNull(result); - Assert.IsType(result); + Assert.IsType(result); } return true; @@ -119,7 +118,7 @@ protected sealed class HotelTextSearchStringMapper : ITextSearchStringMapper /// public string MapFromResultToString(object result) { - if (result is Hotel hotel) + if (result is AzureAISearchHotel hotel) { return $"{hotel.HotelName} {hotel.Description}"; } @@ -135,7 +134,7 @@ protected sealed class HotelTextSearchResultMapper : ITextSearchResultMapper /// public TextSearchResult MapFromResultToTextSearchResult(object result) { - if (result is Hotel hotel) + if (result is AzureAISearchHotel hotel) { return new TextSearchResult(value: hotel.Description) { Name = hotel.HotelName, Link = $"id://{hotel.HotelId}" }; } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreFixture.cs index 5fa7869e4a3a..0c247faeea57 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreFixture.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreFixture.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text.Json.Serialization; using System.Text.RegularExpressions; using System.Threading.Tasks; using Azure; @@ -149,7 +148,7 @@ public static async Task CreateIndexAsync(string indexName, SearchIndexClient ad // Build the list of fields from the model, and then replace the DescriptionEmbedding field with a vector field, to work around // issue where the field is not recognized as an array on parsing on the server side when apply the VectorSearchFieldAttribute. FieldBuilder fieldBuilder = new(); - var searchFields = fieldBuilder.Build(typeof(Hotel)); + var searchFields = fieldBuilder.Build(typeof(AzureAISearchHotel)); var embeddingfield = searchFields.First(x => x.Name == "DescriptionEmbedding"); searchFields.Remove(embeddingfield); searchFields.Add(new VectorSearchField("DescriptionEmbedding", 1536, "my-vector-profile")); @@ -185,9 +184,9 @@ public static async Task UploadDocumentsAsync(SearchClient searchClient, ITextEm { var embedding = await embeddingGenerator.GenerateEmbeddingAsync("This is a great hotel"); - IndexDocumentsBatch batch = IndexDocumentsBatch.Create( + IndexDocumentsBatch batch = IndexDocumentsBatch.Create( IndexDocumentsAction.Upload( - new Hotel() + new AzureAISearchHotel() { HotelId = "BaseSet-1", HotelName = "Hotel 1", @@ -199,7 +198,7 @@ public static async Task UploadDocumentsAsync(SearchClient searchClient, ITextEm Rating = 3.6 }), IndexDocumentsAction.Upload( - new Hotel() + new AzureAISearchHotel() { HotelId = "BaseSet-2", HotelName = "Hotel 2", @@ -211,7 +210,7 @@ public static async Task UploadDocumentsAsync(SearchClient searchClient, ITextEm Rating = 3.60 }), IndexDocumentsAction.Upload( - new Hotel() + new AzureAISearchHotel() { HotelId = "BaseSet-3", HotelName = "Hotel 3", @@ -223,7 +222,7 @@ public static async Task UploadDocumentsAsync(SearchClient searchClient, ITextEm Rating = 4.80 }), IndexDocumentsAction.Upload( - new Hotel() + new AzureAISearchHotel() { HotelId = "BaseSet-4", HotelName = "Hotel 4", @@ -241,43 +240,4 @@ public static async Task UploadDocumentsAsync(SearchClient searchClient, ITextEm // Add some delay to allow time for the documents to get indexed and show up in search. await Task.Delay(5000); } - -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. - public class Hotel - { - [SimpleField(IsKey = true, IsFilterable = true)] - [VectorStoreRecordKey] - public string HotelId { get; set; } - - [SearchableField(IsFilterable = true, IsSortable = true)] - [VectorStoreRecordData(IsFilterable = true, IsFullTextSearchable = true)] - public string HotelName { get; set; } - - [SearchableField(AnalyzerName = LexicalAnalyzerName.Values.EnLucene)] - [VectorStoreRecordData] - public string Description { get; set; } - - [VectorStoreRecordVector(1536)] - public ReadOnlyMemory? DescriptionEmbedding { get; set; } - - [SearchableField(IsFilterable = true, IsFacetable = true)] - [VectorStoreRecordData(IsFilterable = true)] -#pragma warning disable CA1819 // Properties should not return arrays - public string[] Tags { get; set; } -#pragma warning restore CA1819 // Properties should not return arrays - - [JsonPropertyName("parking_is_included")] - [SimpleField(IsFilterable = true, IsSortable = true, IsFacetable = true)] - [VectorStoreRecordData(IsFilterable = true)] - public bool? ParkingIncluded { get; set; } - - [SimpleField(IsFilterable = true, IsSortable = true, IsFacetable = true)] - [VectorStoreRecordData(IsFilterable = true)] - public DateTimeOffset? LastRenovationDate { get; set; } - - [SimpleField(IsFilterable = true, IsSortable = true, IsFacetable = true)] - [VectorStoreRecordData] - public double? Rating { get; set; } - } -#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreRecordCollectionTests.cs index 9265efa90f02..e3a420a789f4 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreRecordCollectionTests.cs @@ -11,7 +11,6 @@ using Microsoft.SemanticKernel.Embeddings; using Xunit; using Xunit.Abstractions; -using static SemanticKernel.IntegrationTests.Connectors.Memory.AzureAISearch.AzureAISearchVectorStoreFixture; namespace SemanticKernel.IntegrationTests.Connectors.Memory.AzureAISearch; @@ -32,7 +31,7 @@ public async Task CollectionExistsReturnsCollectionStateAsync(bool expectedExist { // Arrange. var collectionName = expectedExists ? fixture.TestIndexName : "nonexistentcollection"; - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, collectionName); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, collectionName); // Act. var actual = await sut.CollectionExistsAsync(); @@ -49,11 +48,11 @@ public async Task ItCanCreateACollectionUpsertGetAndSearchAsync(bool useRecordDe // Arrange var hotel = await this.CreateTestHotelAsync("Upsert-1"); var testCollectionName = $"{fixture.TestIndexName}-createtest"; - var options = new AzureAISearchVectorStoreRecordCollectionOptions + var options = new AzureAISearchVectorStoreRecordCollectionOptions { VectorStoreRecordDefinition = useRecordDefinition ? fixture.VectorStoreRecordDefinition : null }; - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, testCollectionName, options); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, testCollectionName, options); await sut.DeleteCollectionAsync(); @@ -112,7 +111,7 @@ public async Task ItCanDeleteCollectionAsync() // Arrange var tempCollectionName = fixture.TestIndexName + "-delete"; await AzureAISearchVectorStoreFixture.CreateIndexAsync(tempCollectionName, fixture.SearchIndexClient); - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, tempCollectionName); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, tempCollectionName); // Act await sut.DeleteCollectionAsync(); @@ -127,11 +126,11 @@ public async Task ItCanDeleteCollectionAsync() public async Task ItCanUpsertDocumentToVectorStoreAsync(bool useRecordDefinition) { // Arrange - var options = new AzureAISearchVectorStoreRecordCollectionOptions + var options = new AzureAISearchVectorStoreRecordCollectionOptions { VectorStoreRecordDefinition = useRecordDefinition ? fixture.VectorStoreRecordDefinition : null }; - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName, options); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName, options); // Act var hotel = await this.CreateTestHotelAsync("Upsert-1"); @@ -161,7 +160,7 @@ public async Task ItCanUpsertDocumentToVectorStoreAsync(bool useRecordDefinition public async Task ItCanUpsertManyDocumentsToVectorStoreAsync() { // Arrange - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); // Act var results = sut.UpsertBatchAsync( @@ -195,11 +194,11 @@ await this.CreateTestHotelAsync("UpsertMany-3"), public async Task ItCanGetDocumentFromVectorStoreAsync(bool includeVectors, bool useRecordDefinition) { // Arrange - var options = new AzureAISearchVectorStoreRecordCollectionOptions + var options = new AzureAISearchVectorStoreRecordCollectionOptions { VectorStoreRecordDefinition = useRecordDefinition ? fixture.VectorStoreRecordDefinition : null }; - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName, options); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName, options); // Act var getResult = await sut.GetAsync("BaseSet-1", new GetRecordOptions { IncludeVectors = includeVectors }); @@ -232,7 +231,7 @@ public async Task ItCanGetDocumentFromVectorStoreAsync(bool includeVectors, bool public async Task ItCanGetManyDocumentsFromVectorStoreAsync() { // Arrange - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); // Act // Also include one non-existing key to test that the operation does not fail for these and returns only the found ones. @@ -256,11 +255,11 @@ public async Task ItCanGetManyDocumentsFromVectorStoreAsync() public async Task ItCanRemoveDocumentFromVectorStoreAsync(bool useRecordDefinition) { // Arrange - var options = new AzureAISearchVectorStoreRecordCollectionOptions + var options = new AzureAISearchVectorStoreRecordCollectionOptions { VectorStoreRecordDefinition = useRecordDefinition ? fixture.VectorStoreRecordDefinition : null }; - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); await sut.UpsertAsync(await this.CreateTestHotelAsync("Remove-1")); // Act @@ -276,7 +275,7 @@ public async Task ItCanRemoveDocumentFromVectorStoreAsync(bool useRecordDefiniti public async Task ItCanRemoveManyDocumentsFromVectorStoreAsync() { // Arrange - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); await sut.UpsertAsync(await this.CreateTestHotelAsync("RemoveMany-1")); await sut.UpsertAsync(await this.CreateTestHotelAsync("RemoveMany-2")); await sut.UpsertAsync(await this.CreateTestHotelAsync("RemoveMany-3")); @@ -295,7 +294,7 @@ public async Task ItCanRemoveManyDocumentsFromVectorStoreAsync() public async Task ItReturnsNullWhenGettingNonExistentRecordAsync() { // Arrange - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); // Act & Assert Assert.Null(await sut.GetAsync("BaseSet-5", new GetRecordOptions { IncludeVectors = true })); @@ -306,7 +305,7 @@ public async Task ItThrowsOperationExceptionForFailedConnectionAsync() { // Arrange var searchIndexClient = new SearchIndexClient(new Uri("https://localhost:12345"), new AzureKeyCredential("12345")); - var sut = new AzureAISearchVectorStoreRecordCollection(searchIndexClient, fixture.TestIndexName); + var sut = new AzureAISearchVectorStoreRecordCollection(searchIndexClient, fixture.TestIndexName); // Act & Assert await Assert.ThrowsAsync(async () => await sut.GetAsync("BaseSet-1", new GetRecordOptions { IncludeVectors = true })); @@ -317,7 +316,7 @@ public async Task ItThrowsOperationExceptionForFailedAuthenticationAsync() { // Arrange var searchIndexClient = new SearchIndexClient(new Uri(fixture.Config.ServiceUrl), new AzureKeyCredential("12345")); - var sut = new AzureAISearchVectorStoreRecordCollection(searchIndexClient, fixture.TestIndexName); + var sut = new AzureAISearchVectorStoreRecordCollection(searchIndexClient, fixture.TestIndexName); // Act & Assert await Assert.ThrowsAsync(async () => await sut.GetAsync("BaseSet-1", new GetRecordOptions { IncludeVectors = true })); @@ -327,8 +326,8 @@ public async Task ItThrowsOperationExceptionForFailedAuthenticationAsync() public async Task ItThrowsMappingExceptionForFailedMapperAsync() { // Arrange - var options = new AzureAISearchVectorStoreRecordCollectionOptions { JsonObjectCustomMapper = new FailingMapper() }; - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName, options); + var options = new AzureAISearchVectorStoreRecordCollectionOptions { JsonObjectCustomMapper = new FailingMapper() }; + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName, options); // Act & Assert await Assert.ThrowsAsync(async () => await sut.GetAsync("BaseSet-1", new GetRecordOptions { IncludeVectors = true })); @@ -340,7 +339,7 @@ public async Task ItThrowsMappingExceptionForFailedMapperAsync() public async Task ItCanSearchWithVectorAndFiltersAsync(string option, bool includeVectors) { // Arrange. - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); // Act. var filter = option == "equality" ? new VectorSearchFilter().EqualTo("HotelName", "Hotel 3") : new VectorSearchFilter().AnyTagEqualTo("Tags", "bar"); @@ -380,7 +379,7 @@ await fixture.EmbeddingGenerator.GenerateEmbeddingAsync("A great hotel"), public async Task ItCanSearchWithVectorizableTextAndFiltersAsync() { // Arrange. - var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); + var sut = new AzureAISearchVectorStoreRecordCollection(fixture.SearchIndexClient, fixture.TestIndexName); // Act. var filter = new VectorSearchFilter().EqualTo("HotelName", "Hotel 3"); @@ -452,7 +451,7 @@ public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync() Assert.Equal(genericMapperEmbedding, (ReadOnlyMemory)localGetResult.Vectors["DescriptionEmbedding"]!); } - private async Task CreateTestHotelAsync(string hotelId) => new() + private async Task CreateTestHotelAsync(string hotelId) => new() { HotelId = hotelId, HotelName = $"MyHotel {hotelId}", @@ -464,14 +463,14 @@ public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync() Rating = 3.6 }; - private sealed class FailingMapper : IVectorStoreRecordMapper + private sealed class FailingMapper : IVectorStoreRecordMapper { - public JsonObject MapFromDataToStorageModel(Hotel dataModel) + public JsonObject MapFromDataToStorageModel(AzureAISearchHotel dataModel) { throw new NotImplementedException(); } - public Hotel MapFromStorageToDataModel(JsonObject storageModel, StorageToDataModelMapperOptions options) + public AzureAISearchHotel MapFromStorageToDataModel(JsonObject storageModel, StorageToDataModelMapperOptions options) { throw new NotImplementedException(); } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreTests.cs index 7bda8cb0fff9..6afcc439faf0 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureAISearch/AzureAISearchVectorStoreTests.cs @@ -1,11 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel.Connectors.AzureAISearch; using Xunit; -using Xunit.Abstractions; namespace SemanticKernel.IntegrationTests.Connectors.Memory.AzureAISearch; @@ -14,32 +11,15 @@ namespace SemanticKernel.IntegrationTests.Connectors.Memory.AzureAISearch; /// Tests work with an Azure AI Search Instance. /// [Collection("AzureAISearchVectorStoreCollection")] -public class AzureAISearchVectorStoreTests(ITestOutputHelper output, AzureAISearchVectorStoreFixture fixture) +public class AzureAISearchVectorStoreTests(AzureAISearchVectorStoreFixture fixture) + : BaseVectorStoreTests(new AzureAISearchVectorStore(fixture.SearchIndexClient)) { // If null, all tests will be enabled private const string SkipReason = "Requires Azure AI Search Service instance up and running"; [Fact(Skip = SkipReason)] - public async Task ItCanGetAListOfExistingCollectionNamesAsync() + public override async Task ItCanGetAListOfExistingCollectionNamesAsync() { - // Arrange - var additionalCollectionName = fixture.TestIndexName + "-listnames"; - await AzureAISearchVectorStoreFixture.DeleteIndexIfExistsAsync(additionalCollectionName, fixture.SearchIndexClient); - await AzureAISearchVectorStoreFixture.CreateIndexAsync(additionalCollectionName, fixture.SearchIndexClient); - var sut = new AzureAISearchVectorStore(fixture.SearchIndexClient); - - // Act - var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); - - // Assert - Assert.Equal(2, collectionNames.Where(x => x.StartsWith(fixture.TestIndexName, StringComparison.InvariantCultureIgnoreCase)).Count()); - Assert.Contains(fixture.TestIndexName, collectionNames); - Assert.Contains(additionalCollectionName, collectionNames); - - // Output - output.WriteLine(string.Join(",", collectionNames)); - - // Cleanup - await AzureAISearchVectorStoreFixture.DeleteIndexIfExistsAsync(additionalCollectionName, fixture.SearchIndexClient); + await base.ItCanGetAListOfExistingCollectionNamesAsync(); } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBHotel.cs new file mode 100644 index 000000000000..7a8830ea2842 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBHotel.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; + +namespace SemanticKernel.IntegrationTests.Connectors.AzureCosmosDBMongoDB; + +#pragma warning disable CS8618 + +public class AzureCosmosDBMongoDBHotel +{ + /// The key of the record. + [VectorStoreRecordKey] + public string HotelId { get; init; } + + /// A string metadata field. + [VectorStoreRecordData(IsFilterable = true)] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreRecordData] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreRecordData] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] + public bool ParkingIncluded { get; set; } + + /// An array metadata field. + [VectorStoreRecordData] + public List Tags { get; set; } = []; + + /// A data field. + [VectorStoreRecordData] + public string Description { get; set; } + + /// A datetime metadata field. + [VectorStoreRecordData] + public DateTime Timestamp { get; set; } + + /// A vector field. + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs index 36ce5c0ca321..a56f8b41399c 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs @@ -81,47 +81,6 @@ public async Task DisposeAsync() } } -#pragma warning disable CS8618 - public record AzureCosmosDBMongoDBHotel() - { - /// The key of the record. - [VectorStoreRecordKey] - public string HotelId { get; init; } - - /// A string metadata field. - [VectorStoreRecordData(IsFilterable = true)] - public string? HotelName { get; set; } - - /// An int metadata field. - [VectorStoreRecordData] - public int HotelCode { get; set; } - - /// A float metadata field. - [VectorStoreRecordData] - public float? HotelRating { get; set; } - - /// A bool metadata field. - [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] - public bool ParkingIncluded { get; set; } - - /// An array metadata field. - [VectorStoreRecordData] - public List Tags { get; set; } = []; - - /// A data field. - [VectorStoreRecordData] - public string Description { get; set; } - - /// A datetime metadata field. - [VectorStoreRecordData] - public DateTime Timestamp { get; set; } - - /// A vector field. - [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)] - public ReadOnlyMemory? DescriptionEmbedding { get; set; } - } -#pragma warning restore CS8618 - #region private private static string GetConnectionString(IConfigurationRoot configuration) diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs index c936a92cf11c..c5929e0ecaa2 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs @@ -9,7 +9,6 @@ using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using Xunit; -using static SemanticKernel.IntegrationTests.Connectors.AzureCosmosDBMongoDB.AzureCosmosDBMongoDBVectorStoreFixture; namespace SemanticKernel.IntegrationTests.Connectors.AzureCosmosDBMongoDB; diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreTests.cs index 9be1378b7b86..9fcbcf81083a 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreTests.cs @@ -1,29 +1,21 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +using SemanticKernel.IntegrationTests.Connectors.Memory; using Xunit; namespace SemanticKernel.IntegrationTests.Connectors.AzureCosmosDBMongoDB; [Collection("AzureCosmosDBMongoDBVectorStoreCollection")] public class AzureCosmosDBMongoDBVectorStoreTests(AzureCosmosDBMongoDBVectorStoreFixture fixture) + : BaseVectorStoreTests(new AzureCosmosDBMongoDBVectorStore(fixture.MongoDatabase)) { private const string? SkipReason = "Azure CosmosDB MongoDB cluster is required"; [Fact(Skip = SkipReason)] - public async Task ItCanGetAListOfExistingCollectionNamesAsync() + public override async Task ItCanGetAListOfExistingCollectionNamesAsync() { - // Arrange - var sut = new AzureCosmosDBMongoDBVectorStore(fixture.MongoDatabase); - - // Act - var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); - - // Assert - Assert.Contains("sk-test-hotels", collectionNames); - Assert.Contains("sk-test-contacts", collectionNames); - Assert.Contains("sk-test-addresses", collectionNames); + await base.ItCanGetAListOfExistingCollectionNamesAsync(); } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreTests.cs index 938fe5c14caf..4d3899784f4a 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreTests.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Linq; using System.Threading.Tasks; -using Microsoft.Azure.Cosmos; using Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; using Xunit; @@ -13,23 +11,13 @@ namespace SemanticKernel.IntegrationTests.Connectors.Memory.AzureCosmosDBNoSQL; /// [Collection("AzureCosmosDBNoSQLVectorStoreCollection")] public sealed class AzureCosmosDBNoSQLVectorStoreTests(AzureCosmosDBNoSQLVectorStoreFixture fixture) + : BaseVectorStoreTests(new AzureCosmosDBNoSQLVectorStore(fixture.Database!)) { private const string? SkipReason = "Azure CosmosDB NoSQL cluster is required"; [Fact(Skip = SkipReason)] - public async Task ItCanGetAListOfExistingCollectionNamesAsync() + public override async Task ItCanGetAListOfExistingCollectionNamesAsync() { - // Arrange - var sut = new AzureCosmosDBNoSQLVectorStore(fixture.Database!); - - await fixture.Database!.CreateContainerIfNotExistsAsync(new ContainerProperties("list-names-1", "/id")); - await fixture.Database!.CreateContainerIfNotExistsAsync(new ContainerProperties("list-names-2", "/id")); - - // Act - var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); - - // Assert - Assert.Contains("list-names-1", collectionNames); - Assert.Contains("list-names-2", collectionNames); + await base.ItCanGetAListOfExistingCollectionNamesAsync(); } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/BaseVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/BaseVectorStoreTests.cs new file mode 100644 index 000000000000..1d7739fd427d --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/BaseVectorStoreTests.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Xunit; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory; + +/// +/// Base class for integration tests. +/// +public abstract class BaseVectorStoreTests(IVectorStore vectorStore) + where TKey : notnull +{ + [Fact] + public virtual async Task ItCanGetAListOfExistingCollectionNamesAsync() + { + // Arrange + var expectedCollectionNames = new List { "listcollectionnames1", "listcollectionnames2", "listcollectionnames3" }; + + foreach (var collectionName in expectedCollectionNames) + { + var collection = vectorStore.GetCollection(collectionName); + + await collection.CreateCollectionIfNotExistsAsync(); + } + + // Act + var actualCollectionNames = await vectorStore.ListCollectionNamesAsync().ToListAsync(); + + // Assert + var expected = expectedCollectionNames.Select(l => l.ToUpperInvariant()).ToList(); + var actual = actualCollectionNames.Select(l => l.ToUpperInvariant()).ToList(); + + expected.ForEach(item => Assert.Contains(item, actual)); + + // Cleanup + foreach (var collectionName in expectedCollectionNames) + { + var collection = vectorStore.GetCollection(collectionName); + + await collection.DeleteCollectionAsync(); + } + } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBHotel.cs new file mode 100644 index 000000000000..b3adb2e723a1 --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBHotel.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.VectorData; + +namespace SemanticKernel.IntegrationTests.Connectors.MongoDB; + +#pragma warning disable CS8618 + +public class MongoDBHotel +{ + /// The key of the record. + [VectorStoreRecordKey] + public string HotelId { get; init; } + + /// A string metadata field. + [VectorStoreRecordData(IsFilterable = true)] + public string? HotelName { get; set; } + + /// An int metadata field. + [VectorStoreRecordData] + public int HotelCode { get; set; } + + /// A float metadata field. + [VectorStoreRecordData] + public float? HotelRating { get; set; } + + /// A bool metadata field. + [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] + public bool ParkingIncluded { get; set; } + + /// An array metadata field. + [VectorStoreRecordData] + public List Tags { get; set; } = []; + + /// A data field. + [VectorStoreRecordData] + public string Description { get; set; } + + /// A datetime metadata field. + [VectorStoreRecordData] + public DateTime Timestamp { get; set; } + + /// A vector field. + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.IvfFlat)] + public ReadOnlyMemory? DescriptionEmbedding { get; set; } +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreFixture.cs index 6c037c70e11b..3d975dffbdf3 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreFixture.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreFixture.cs @@ -88,47 +88,6 @@ public async Task DisposeAsync() } } -#pragma warning disable CS8618 - public record MongoDBHotel() - { - /// The key of the record. - [VectorStoreRecordKey] - public string HotelId { get; init; } - - /// A string metadata field. - [VectorStoreRecordData(IsFilterable = true)] - public string? HotelName { get; set; } - - /// An int metadata field. - [VectorStoreRecordData] - public int HotelCode { get; set; } - - /// A float metadata field. - [VectorStoreRecordData] - public float? HotelRating { get; set; } - - /// A bool metadata field. - [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] - public bool ParkingIncluded { get; set; } - - /// An array metadata field. - [VectorStoreRecordData] - public List Tags { get; set; } = []; - - /// A data field. - [VectorStoreRecordData] - public string Description { get; set; } - - /// A datetime metadata field. - [VectorStoreRecordData] - public DateTime Timestamp { get; set; } - - /// A vector field. - [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.IvfFlat)] - public ReadOnlyMemory? DescriptionEmbedding { get; set; } - } -#pragma warning restore CS8618 - #region private private static async Task SetupMongoDBContainerAsync(DockerClient client) diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreRecordCollectionTests.cs index b0d6affb384f..11da55ba3329 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreRecordCollectionTests.cs @@ -9,7 +9,6 @@ using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using Xunit; -using static SemanticKernel.IntegrationTests.Connectors.MongoDB.MongoDBVectorStoreFixture; namespace SemanticKernel.IntegrationTests.Connectors.MongoDB; diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreTests.cs index e4d29d5925ce..cd0d7e374c4c 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreTests.cs @@ -1,30 +1,22 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel.Connectors.MongoDB; +using SemanticKernel.IntegrationTests.Connectors.Memory; using Xunit; namespace SemanticKernel.IntegrationTests.Connectors.MongoDB; [Collection("MongoDBVectorStoreCollection")] public class MongoDBVectorStoreTests(MongoDBVectorStoreFixture fixture) + : BaseVectorStoreTests(new MongoDBVectorStore(fixture.MongoDatabase)) { // If null, all tests will be enabled private const string? SkipReason = "The tests are for manual verification."; [Fact(Skip = SkipReason)] - public async Task ItCanGetAListOfExistingCollectionNamesAsync() + public override async Task ItCanGetAListOfExistingCollectionNamesAsync() { - // Arrange - var sut = new MongoDBVectorStore(fixture.MongoDatabase); - - // Act - var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); - - // Assert - Assert.Contains("sk-test-hotels", collectionNames); - Assert.Contains("sk-test-contacts", collectionNames); - Assert.Contains("sk-test-addresses", collectionNames); + await base.ItCanGetAListOfExistingCollectionNamesAsync(); } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs index f1f1c6e63937..2864bf28b793 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.VectorData; using Microsoft.SemanticKernel.Connectors.Pinecone; @@ -13,16 +12,15 @@ namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; [Collection("PineconeVectorStoreTests")] [PineconeApiKeySetCondition] -public class PineconeVectorStoreTests(PineconeVectorStoreFixture fixture) : IClassFixture +public class PineconeVectorStoreTests(PineconeVectorStoreFixture fixture) + : BaseVectorStoreTests(new PineconeVectorStore(fixture.Client)), IClassFixture { private PineconeVectorStoreFixture Fixture { get; } = fixture; [PineconeFact] - public async Task ListCollectionNamesAsync() + public override async Task ItCanGetAListOfExistingCollectionNamesAsync() { - var collectionNames = await this.Fixture.VectorStore.ListCollectionNamesAsync().ToListAsync(); - - Assert.Equal([this.Fixture.IndexName], collectionNames); + await base.ItCanGetAListOfExistingCollectionNamesAsync(); } [PineconeFact] diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreTests.cs index ae4a1313bfee..39551054e4bb 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreTests.cs @@ -1,15 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel.Connectors.Qdrant; using Xunit; -using Xunit.Abstractions; namespace SemanticKernel.IntegrationTests.Connectors.Memory.Qdrant; [Collection("QdrantVectorStoreCollection")] -public class QdrantVectorStoreTests(ITestOutputHelper output, QdrantVectorStoreFixture fixture) +public class QdrantVectorStoreTests(QdrantVectorStoreFixture fixture) + : BaseVectorStoreTests(new QdrantVectorStore(fixture.QdrantClient)) { [Fact] public async Task ItPassesSettingsFromVectorStoreToCollectionAsync() @@ -34,23 +33,4 @@ await directCollection.UpsertAsync(new QdrantVectorStoreFixture.HotelInfo DescriptionEmbedding = new float[1536], }); } - - [Fact] - public async Task ItCanGetAListOfExistingCollectionNamesAsync() - { - // Arrange - var sut = new QdrantVectorStore(fixture.QdrantClient); - - // Act - var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); - - // Assert - Assert.Equal(3, collectionNames.Count); - Assert.Contains("namedVectorsHotels", collectionNames); - Assert.Contains("singleVectorHotels", collectionNames); - Assert.Contains("singleVectorGuidIdHotels", collectionNames); - - // Output - output.WriteLine(string.Join(",", collectionNames)); - } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisHashSetVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisHashSetVectorStoreRecordCollectionTests.cs index 4fff25413c5c..6e60f8bb12f0 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisHashSetVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisHashSetVectorStoreRecordCollectionTests.cs @@ -10,7 +10,6 @@ using StackExchange.Redis; using Xunit; using Xunit.Abstractions; -using static SemanticKernel.IntegrationTests.Connectors.Memory.Redis.RedisVectorStoreFixture; namespace SemanticKernel.IntegrationTests.Connectors.Memory.Redis; @@ -33,7 +32,7 @@ public sealed class RedisHashSetVectorStoreRecordCollectionTests(ITestOutputHelp public async Task CollectionExistsReturnsCollectionStateAsync(string collectionName, bool expectedExists) { // Arrange. - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, collectionName); + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, collectionName); // Act. var actual = await sut.CollectionExistsAsync(); @@ -52,12 +51,12 @@ public async Task ItCanCreateACollectionUpsertGetAndSearchAsync(bool useRecordDe var collectionNamePostfix = useRecordDefinition ? "WithDefinition" : "WithType"; var testCollectionName = $"hashsetcreatetest{collectionNamePostfix}"; - var options = new RedisHashSetVectorStoreRecordCollectionOptions + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.BasicVectorStoreRecordDefinition : null }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, testCollectionName, options); + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, testCollectionName, options); // Act await sut.CreateCollectionAsync(); @@ -111,7 +110,7 @@ public async Task ItCanDeleteCollectionAsync() createParams.AddPrefix(tempCollectionName); await fixture.Database.FT().CreateAsync(tempCollectionName, createParams, schema); - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, tempCollectionName); + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, tempCollectionName); // Act await sut.DeleteCollectionAsync(); @@ -126,12 +125,12 @@ public async Task ItCanDeleteCollectionAsync() public async Task ItCanUpsertDocumentToVectorStoreAsync(bool useRecordDefinition) { // Arrange. - var options = new RedisHashSetVectorStoreRecordCollectionOptions + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.BasicVectorStoreRecordDefinition : null }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); var record = CreateTestHotel("HUpsert-2", 2); // Act. @@ -159,12 +158,12 @@ public async Task ItCanUpsertDocumentToVectorStoreAsync(bool useRecordDefinition public async Task ItCanUpsertManyDocumentsToVectorStoreAsync(bool useRecordDefinition) { // Arrange. - var options = new RedisHashSetVectorStoreRecordCollectionOptions + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.BasicVectorStoreRecordDefinition : null }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act. var results = sut.UpsertBatchAsync( @@ -198,12 +197,12 @@ public async Task ItCanUpsertManyDocumentsToVectorStoreAsync(bool useRecordDefin public async Task ItCanGetDocumentFromVectorStoreAsync(bool includeVectors, bool useRecordDefinition) { // Arrange. - var options = new RedisHashSetVectorStoreRecordCollectionOptions + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.BasicVectorStoreRecordDefinition : null }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act. var getResult = await sut.GetAsync("HBaseSet-1", new GetRecordOptions { IncludeVectors = includeVectors }); @@ -232,8 +231,8 @@ public async Task ItCanGetDocumentFromVectorStoreAsync(bool includeVectors, bool public async Task ItCanGetManyDocumentsFromVectorStoreAsync() { // Arrange - var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act // Also include one non-existing key to test that the operation does not fail for these and returns only the found ones. @@ -257,13 +256,13 @@ public async Task ItCanGetManyDocumentsFromVectorStoreAsync() public async Task ItCanRemoveDocumentFromVectorStoreAsync(bool useRecordDefinition) { // Arrange. - var options = new RedisHashSetVectorStoreRecordCollectionOptions + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.BasicVectorStoreRecordDefinition : null }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); - var record = new BasicFloat32Hotel + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var record = new RedisBasicFloat32Hotel { HotelId = "HRemove-1", HotelName = "Remove Test Hotel", @@ -287,8 +286,8 @@ public async Task ItCanRemoveDocumentFromVectorStoreAsync(bool useRecordDefiniti public async Task ItCanRemoveManyDocumentsFromVectorStoreAsync() { // Arrange - var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); await sut.UpsertAsync(CreateTestHotel("HRemoveMany-1", 1)); await sut.UpsertAsync(CreateTestHotel("HRemoveMany-2", 2)); await sut.UpsertAsync(CreateTestHotel("HRemoveMany-3", 3)); @@ -309,8 +308,8 @@ public async Task ItCanRemoveManyDocumentsFromVectorStoreAsync() public async Task ItCanSearchWithFloat32VectorAndFilterAsync(string filterType, bool includeVectors) { // Arrange - var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); var vector = new ReadOnlyMemory(new[] { 30f, 31f, 32f, 33f }); var filter = filterType == "equality" ? new VectorSearchFilter().EqualTo("HotelCode", 1) : new VectorSearchFilter().EqualTo("HotelName", "My Hotel 1"); @@ -348,14 +347,14 @@ public async Task ItCanSearchWithFloat32VectorAndFilterAsync(string filterType, public async Task ItCanSearchWithFloat32VectorAndTopSkipAsync() { // Arrange - var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName + "TopSkip", options); + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName + "TopSkip", options); await sut.CreateCollectionIfNotExistsAsync(); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "HTopSkip_1", HotelName = "1", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 1.0f]) }); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "HTopSkip_2", HotelName = "2", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 2.0f]) }); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "HTopSkip_3", HotelName = "3", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 3.0f]) }); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "HTopSkip_4", HotelName = "4", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 4.0f]) }); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "HTopSkip_5", HotelName = "5", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 5.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "HTopSkip_1", HotelName = "1", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 1.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "HTopSkip_2", HotelName = "2", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 2.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "HTopSkip_3", HotelName = "3", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 3.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "HTopSkip_4", HotelName = "4", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 4.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "HTopSkip_5", HotelName = "5", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 5.0f]) }); var vector = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 1.0f]); // Act @@ -379,12 +378,12 @@ public async Task ItCanSearchWithFloat32VectorAndTopSkipAsync() public async Task ItCanSearchWithFloat64VectorAsync(bool includeVectors) { // Arrange - var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName + "Float64", options); + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName + "Float64", options); await sut.CreateCollectionIfNotExistsAsync(); - await sut.UpsertAsync(new BasicFloat64Hotel { HotelId = "HFloat64_1", HotelName = "1", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0d, 1.1d, 1.2d, 1.3d]) }); - await sut.UpsertAsync(new BasicFloat64Hotel { HotelId = "HFloat64_2", HotelName = "2", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([2.0d, 2.1d, 2.2d, 2.3d]) }); - await sut.UpsertAsync(new BasicFloat64Hotel { HotelId = "HFloat64_3", HotelName = "3", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([3.0d, 3.1d, 3.2d, 3.3d]) }); + await sut.UpsertAsync(new RedisBasicFloat64Hotel { HotelId = "HFloat64_1", HotelName = "1", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0d, 1.1d, 1.2d, 1.3d]) }); + await sut.UpsertAsync(new RedisBasicFloat64Hotel { HotelId = "HFloat64_2", HotelName = "2", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([2.0d, 2.1d, 2.2d, 2.3d]) }); + await sut.UpsertAsync(new RedisBasicFloat64Hotel { HotelId = "HFloat64_3", HotelName = "3", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([3.0d, 3.1d, 3.2d, 3.3d]) }); var vector = new ReadOnlyMemory([2.0d, 2.1d, 2.2d, 2.3d]); @@ -418,8 +417,8 @@ public async Task ItCanSearchWithFloat64VectorAsync(bool includeVectors) public async Task ItReturnsNullWhenGettingNonExistentRecordAsync() { // Arrange - var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act & Assert Assert.Null(await sut.GetAsync("HBaseSet-5", new GetRecordOptions { IncludeVectors = true })); @@ -429,12 +428,12 @@ public async Task ItReturnsNullWhenGettingNonExistentRecordAsync() public async Task ItThrowsMappingExceptionForFailedMapperAsync() { // Arrange - var options = new RedisHashSetVectorStoreRecordCollectionOptions + var options = new RedisHashSetVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, HashEntriesCustomMapper = new FailingMapper() }; - var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var sut = new RedisHashSetVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act & Assert await Assert.ThrowsAsync(async () => await sut.GetAsync("HBaseSet-1", new GetRecordOptions { IncludeVectors = true })); @@ -493,9 +492,9 @@ public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync() Assert.Equal(new[] { 30f, 31f, 32f, 33f }, ((ReadOnlyMemory)localGetResult.Vectors["DescriptionEmbedding"]!).ToArray()); } - private static BasicFloat32Hotel CreateTestHotel(string hotelId, int hotelCode) + private static RedisBasicFloat32Hotel CreateTestHotel(string hotelId, int hotelCode) { - var record = new BasicFloat32Hotel + var record = new RedisBasicFloat32Hotel { HotelId = hotelId, HotelName = $"My Hotel {hotelCode}", @@ -508,14 +507,14 @@ private static BasicFloat32Hotel CreateTestHotel(string hotelId, int hotelCode) return record; } - private sealed class FailingMapper : IVectorStoreRecordMapper + private sealed class FailingMapper : IVectorStoreRecordMapper { - public (string Key, HashEntry[] HashEntries) MapFromDataToStorageModel(BasicFloat32Hotel dataModel) + public (string Key, HashEntry[] HashEntries) MapFromDataToStorageModel(RedisBasicFloat32Hotel dataModel) { throw new NotImplementedException(); } - public BasicFloat32Hotel MapFromStorageToDataModel((string Key, HashEntry[] HashEntries) storageModel, StorageToDataModelMapperOptions options) + public RedisBasicFloat32Hotel MapFromStorageToDataModel((string Key, HashEntry[] HashEntries) storageModel, StorageToDataModelMapperOptions options) { throw new NotImplementedException(); } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisHotel.cs new file mode 100644 index 000000000000..87dc5c2fb89b --- /dev/null +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisHotel.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json.Serialization; +using Microsoft.Extensions.VectorData; + +namespace SemanticKernel.IntegrationTests.Connectors.Memory.Redis; + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + +/// +/// A test model for the vector store that has complex properties as supported by JSON redis mode. +/// +public class RedisHotel +{ + [VectorStoreRecordKey] + public string HotelId { get; init; } + + [VectorStoreRecordData(IsFilterable = true)] + public string HotelName { get; init; } + + [VectorStoreRecordData(IsFilterable = true)] + public int HotelCode { get; init; } + + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; init; } + + [VectorStoreRecordVector(4)] + public ReadOnlyMemory? DescriptionEmbedding { get; init; } + +#pragma warning disable CA1819 // Properties should not return arrays + [VectorStoreRecordData(IsFilterable = true)] + public string[] Tags { get; init; } + + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string[] FTSTags { get; init; } +#pragma warning restore CA1819 // Properties should not return arrays + + [JsonPropertyName("parking_is_included")] + [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] + public bool ParkingIncluded { get; init; } + + [VectorStoreRecordData] + public DateTimeOffset LastRenovationDate { get; init; } + + [VectorStoreRecordData] + public double Rating { get; init; } + + [VectorStoreRecordData] + public RedisHotelAddress Address { get; init; } +} + +/// +/// A test model for the vector store to simulate a complex type. +/// +public class RedisHotelAddress +{ + public string City { get; init; } + public string Country { get; init; } +} + +/// +/// A test model for the vector store that only uses basic types as supported by HashSets Redis mode. +/// +public class RedisBasicHotel +{ + [VectorStoreRecordKey] + public string HotelId { get; init; } + + [VectorStoreRecordData(IsFilterable = true)] + public string HotelName { get; init; } + + [VectorStoreRecordData(IsFilterable = true)] + public int HotelCode { get; init; } + + [VectorStoreRecordData(IsFullTextSearchable = true)] + public string Description { get; init; } + + [VectorStoreRecordVector(4)] + public ReadOnlyMemory? DescriptionEmbedding { get; init; } + + [JsonPropertyName("parking_is_included")] + [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] + public bool ParkingIncluded { get; init; } + + [VectorStoreRecordData] + public double Rating { get; init; } +} + +/// +/// A test model for the vector store that only uses basic types as supported by HashSets Redis mode. +/// +public class RedisBasicFloat32Hotel : RedisBasicHotel +{ +} + +/// +/// A test model for the vector store that only uses basic types as supported by HashSets Redis mode. +/// +public class RedisBasicFloat64Hotel : RedisBasicHotel +{ +} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisJsonVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisJsonVectorStoreRecordCollectionTests.cs index 780a88067b61..7bb4ad04fa9f 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisJsonVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisJsonVectorStoreRecordCollectionTests.cs @@ -10,7 +10,6 @@ using NRedisStack.Search; using Xunit; using Xunit.Abstractions; -using static SemanticKernel.IntegrationTests.Connectors.Memory.Redis.RedisVectorStoreFixture; namespace SemanticKernel.IntegrationTests.Connectors.Memory.Redis; @@ -33,7 +32,7 @@ public sealed class RedisJsonVectorStoreRecordCollectionTests(ITestOutputHelper public async Task CollectionExistsReturnsCollectionStateAsync(string collectionName, bool expectedExists) { // Arrange. - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, collectionName); + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, collectionName); // Act. var actual = await sut.CollectionExistsAsync(); @@ -52,12 +51,12 @@ public async Task ItCanCreateACollectionUpsertGetAndSearchAsync(bool useRecordDe var collectionNamePostfix = useRecordDefinition ? "WithDefinition" : "WithType"; var testCollectionName = $"jsoncreatetest{collectionNamePostfix}"; - var options = new RedisJsonVectorStoreRecordCollectionOptions + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.VectorStoreRecordDefinition : null }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, testCollectionName, options); + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, testCollectionName, options); // Act await sut.CreateCollectionAsync(); @@ -120,7 +119,7 @@ public async Task ItCanDeleteCollectionAsync() createParams.AddPrefix(tempCollectionName); await fixture.Database.FT().CreateAsync(tempCollectionName, createParams, schema); - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, tempCollectionName); + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, tempCollectionName); // Act await sut.DeleteCollectionAsync(); @@ -135,13 +134,13 @@ public async Task ItCanDeleteCollectionAsync() public async Task ItCanUpsertDocumentToVectorStoreAsync(bool useRecordDefinition) { // Arrange. - var options = new RedisJsonVectorStoreRecordCollectionOptions + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.VectorStoreRecordDefinition : null }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); - Hotel record = CreateTestHotel("Upsert-2", 2); + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + RedisHotel record = CreateTestHotel("Upsert-2", 2); // Act. var upsertResult = await sut.UpsertAsync(record); @@ -173,12 +172,12 @@ public async Task ItCanUpsertDocumentToVectorStoreAsync(bool useRecordDefinition public async Task ItCanUpsertManyDocumentsToVectorStoreAsync(bool useRecordDefinition) { // Arrange. - var options = new RedisJsonVectorStoreRecordCollectionOptions + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.VectorStoreRecordDefinition : null }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act. var results = sut.UpsertBatchAsync( @@ -212,12 +211,12 @@ public async Task ItCanUpsertManyDocumentsToVectorStoreAsync(bool useRecordDefin public async Task ItCanGetDocumentFromVectorStoreAsync(bool includeVectors, bool useRecordDefinition) { // Arrange. - var options = new RedisJsonVectorStoreRecordCollectionOptions + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.VectorStoreRecordDefinition : null }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act. var getResult = await sut.GetAsync("BaseSet-1", new GetRecordOptions { IncludeVectors = includeVectors }); @@ -250,8 +249,8 @@ public async Task ItCanGetDocumentFromVectorStoreAsync(bool includeVectors, bool public async Task ItCanGetManyDocumentsFromVectorStoreAsync() { // Arrange - var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act // Also include one non-existing key to test that the operation does not fail for these and returns only the found ones. @@ -273,8 +272,8 @@ public async Task ItCanGetManyDocumentsFromVectorStoreAsync() public async Task ItFailsToGetDocumentsWithInvalidSchemaAsync() { // Arrange. - var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act & Assert. await Assert.ThrowsAsync(async () => await sut.GetAsync("BaseSet-4-Invalid", new GetRecordOptions { IncludeVectors = true })); @@ -286,14 +285,14 @@ public async Task ItFailsToGetDocumentsWithInvalidSchemaAsync() public async Task ItCanRemoveDocumentFromVectorStoreAsync(bool useRecordDefinition) { // Arrange. - var options = new RedisJsonVectorStoreRecordCollectionOptions + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, VectorStoreRecordDefinition = useRecordDefinition ? fixture.VectorStoreRecordDefinition : null }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); - var address = new HotelAddress { City = "Seattle", Country = "USA" }; - var record = new Hotel + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var address = new RedisHotelAddress { City = "Seattle", Country = "USA" }; + var record = new RedisHotel { HotelId = "Remove-1", HotelName = "Remove Test Hotel", @@ -317,8 +316,8 @@ public async Task ItCanRemoveDocumentFromVectorStoreAsync(bool useRecordDefiniti public async Task ItCanRemoveManyDocumentsFromVectorStoreAsync() { // Arrange - var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); await sut.UpsertAsync(CreateTestHotel("RemoveMany-1", 1)); await sut.UpsertAsync(CreateTestHotel("RemoveMany-2", 2)); await sut.UpsertAsync(CreateTestHotel("RemoveMany-3", 3)); @@ -339,8 +338,8 @@ public async Task ItCanRemoveManyDocumentsFromVectorStoreAsync() public async Task ItCanSearchWithFloat32VectorAndFilterAsync(string filterType) { // Arrange - var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); var vector = new ReadOnlyMemory(new[] { 30f, 31f, 32f, 33f }); var filter = filterType == "equality" ? new VectorSearchFilter().EqualTo("HotelCode", 1) : new VectorSearchFilter().AnyTagEqualTo("Tags", "pool"); @@ -372,14 +371,14 @@ public async Task ItCanSearchWithFloat32VectorAndFilterAsync(string filterType) public async Task ItCanSearchWithFloat32VectorAndTopSkipAsync() { // Arrange - var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName + "TopSkip", options); + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName + "TopSkip", options); await sut.CreateCollectionIfNotExistsAsync(); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "TopSkip_1", HotelName = "1", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 1.0f]) }); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "TopSkip_2", HotelName = "2", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 2.0f]) }); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "TopSkip_3", HotelName = "3", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 3.0f]) }); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "TopSkip_4", HotelName = "4", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 4.0f]) }); - await sut.UpsertAsync(new BasicFloat32Hotel { HotelId = "TopSkip_5", HotelName = "5", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 5.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "TopSkip_1", HotelName = "1", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 1.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "TopSkip_2", HotelName = "2", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 2.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "TopSkip_3", HotelName = "3", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 3.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "TopSkip_4", HotelName = "4", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 4.0f]) }); + await sut.UpsertAsync(new RedisBasicFloat32Hotel { HotelId = "TopSkip_5", HotelName = "5", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 5.0f]) }); var vector = new ReadOnlyMemory([1.0f, 1.0f, 1.0f, 1.0f]); // Act @@ -403,12 +402,12 @@ public async Task ItCanSearchWithFloat32VectorAndTopSkipAsync() public async Task ItCanSearchWithFloat64VectorAsync(bool includeVectors) { // Arrange - var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName + "Float64", options); + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName + "Float64", options); await sut.CreateCollectionIfNotExistsAsync(); - await sut.UpsertAsync(new BasicFloat64Hotel { HotelId = "Float64_1", HotelName = "1", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0d, 1.1d, 1.2d, 1.3d]) }); - await sut.UpsertAsync(new BasicFloat64Hotel { HotelId = "Float64_2", HotelName = "2", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([2.0d, 2.1d, 2.2d, 2.3d]) }); - await sut.UpsertAsync(new BasicFloat64Hotel { HotelId = "Float64_3", HotelName = "3", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([3.0d, 3.1d, 3.2d, 3.3d]) }); + await sut.UpsertAsync(new RedisBasicFloat64Hotel { HotelId = "Float64_1", HotelName = "1", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([1.0d, 1.1d, 1.2d, 1.3d]) }); + await sut.UpsertAsync(new RedisBasicFloat64Hotel { HotelId = "Float64_2", HotelName = "2", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([2.0d, 2.1d, 2.2d, 2.3d]) }); + await sut.UpsertAsync(new RedisBasicFloat64Hotel { HotelId = "Float64_3", HotelName = "3", Description = "Nice hotel", DescriptionEmbedding = new ReadOnlyMemory([3.0d, 3.1d, 3.2d, 3.3d]) }); var vector = new ReadOnlyMemory([2.0d, 2.1d, 2.2d, 2.3d]); @@ -438,8 +437,8 @@ public async Task ItCanSearchWithFloat64VectorAsync(bool includeVectors) public async Task ItReturnsNullWhenGettingNonExistentRecordAsync() { // Arrange - var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true }; + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act & Assert Assert.Null(await sut.GetAsync("BaseSet-5", new GetRecordOptions { IncludeVectors = true })); @@ -449,12 +448,12 @@ public async Task ItReturnsNullWhenGettingNonExistentRecordAsync() public async Task ItThrowsMappingExceptionForFailedMapperAsync() { // Arrange - var options = new RedisJsonVectorStoreRecordCollectionOptions + var options = new RedisJsonVectorStoreRecordCollectionOptions { PrefixCollectionNameToKeyNames = true, JsonNodeCustomMapper = new FailingMapper() }; - var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); + var sut = new RedisJsonVectorStoreRecordCollection(fixture.Database, TestCollectionName, options); // Act & Assert await Assert.ThrowsAsync(async () => await sut.GetAsync("BaseSet-1", new GetRecordOptions { IncludeVectors = true })); @@ -484,7 +483,7 @@ public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync() { "ParkingIncluded", true }, { "LastRenovationDate", new DateTimeOffset(1970, 1, 18, 0, 0, 0, TimeSpan.Zero) }, { "Rating", 3.6 }, - { "Address", new HotelAddress { City = "Seattle", Country = "USA" } }, + { "Address", new RedisHotelAddress { City = "Seattle", Country = "USA" } }, { "Description", "This is a generic mapper hotel" }, { "DescriptionEmbedding", new[] { 30f, 31f, 32f, 33f } } }, @@ -505,7 +504,7 @@ public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync() Assert.True((bool)baseSetGetResult.Data["ParkingIncluded"]!); Assert.Equal(new DateTimeOffset(1970, 1, 18, 0, 0, 0, TimeSpan.Zero), baseSetGetResult.Data["LastRenovationDate"]); Assert.Equal(3.6, baseSetGetResult.Data["Rating"]); - Assert.Equal("Seattle", ((HotelAddress)baseSetGetResult.Data["Address"]!).City); + Assert.Equal("Seattle", ((RedisHotelAddress)baseSetGetResult.Data["Address"]!).City); Assert.Equal("This is a great hotel.", baseSetGetResult.Data["Description"]); Assert.Equal(new[] { 30f, 31f, 32f, 33f }, ((ReadOnlyMemory)baseSetGetResult.Vectors["DescriptionEmbedding"]!).ToArray()); @@ -520,15 +519,15 @@ public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync() Assert.True((bool)localGetResult.Data["ParkingIncluded"]!); Assert.Equal(new DateTimeOffset(1970, 1, 18, 0, 0, 0, TimeSpan.Zero), localGetResult.Data["LastRenovationDate"]); Assert.Equal(3.6d, localGetResult.Data["Rating"]); - Assert.Equal("Seattle", ((HotelAddress)localGetResult.Data["Address"]!).City); + Assert.Equal("Seattle", ((RedisHotelAddress)localGetResult.Data["Address"]!).City); Assert.Equal("This is a generic mapper hotel", localGetResult.Data["Description"]); Assert.Equal(new[] { 30f, 31f, 32f, 33f }, ((ReadOnlyMemory)localGetResult.Vectors["DescriptionEmbedding"]!).ToArray()); } - private static Hotel CreateTestHotel(string hotelId, int hotelCode) + private static RedisHotel CreateTestHotel(string hotelId, int hotelCode) { - var address = new HotelAddress { City = "Seattle", Country = "USA" }; - var record = new Hotel + var address = new RedisHotelAddress { City = "Seattle", Country = "USA" }; + var record = new RedisHotel { HotelId = hotelId, HotelName = $"My Hotel {hotelCode}", @@ -545,14 +544,14 @@ private static Hotel CreateTestHotel(string hotelId, int hotelCode) return record; } - private sealed class FailingMapper : IVectorStoreRecordMapper + private sealed class FailingMapper : IVectorStoreRecordMapper { - public (string Key, JsonNode Node) MapFromDataToStorageModel(Hotel dataModel) + public (string Key, JsonNode Node) MapFromDataToStorageModel(RedisHotel dataModel) { throw new NotImplementedException(); } - public Hotel MapFromStorageToDataModel((string Key, JsonNode Node) storageModel, StorageToDataModelMapperOptions options) + public RedisHotel MapFromStorageToDataModel((string Key, JsonNode Node) storageModel, StorageToDataModelMapperOptions options) { throw new NotImplementedException(); } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisVectorStoreFixture.cs index 26ea2338001f..bec643a13d5b 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisVectorStoreFixture.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisVectorStoreFixture.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; using System.Runtime.InteropServices; -using System.Text.Json.Serialization; using System.Threading.Tasks; using Docker.DotNet; using Docker.DotNet.Models; @@ -17,6 +16,7 @@ namespace SemanticKernel.IntegrationTests.Connectors.Memory.Redis; #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + /// /// Does setup and teardown of redis docker container and associated test data. /// @@ -49,7 +49,7 @@ public RedisVectorStoreFixture() new VectorStoreRecordDataProperty("ParkingIncluded", typeof(bool)) { StoragePropertyName = "parking_is_included" }, new VectorStoreRecordDataProperty("LastRenovationDate", typeof(DateTimeOffset)), new VectorStoreRecordDataProperty("Rating", typeof(double)), - new VectorStoreRecordDataProperty("Address", typeof(HotelAddress)) + new VectorStoreRecordDataProperty("Address", typeof(RedisHotelAddress)) } }; this.BasicVectorStoreRecordDefinition = new VectorStoreRecordDefinition @@ -120,7 +120,7 @@ public async Task InitializeAsync() await this.Database.FT().CreateAsync("hashhotels", hashsetCreateParams, hashSchema); // Create some test data. - var address = new HotelAddress { City = "Seattle", Country = "USA" }; + var address = new RedisHotelAddress { City = "Seattle", Country = "USA" }; var embedding = new[] { 30f, 31f, 32f, 33f }; // Add JSON test data. @@ -234,98 +234,4 @@ await client.Containers.StartContainerAsync( return container.ID; } - - /// - /// A test model for the vector store that has complex properties as supported by JSON redis mode. - /// - public class Hotel - { - [VectorStoreRecordKey] - public string HotelId { get; init; } - - [VectorStoreRecordData(IsFilterable = true)] - public string HotelName { get; init; } - - [VectorStoreRecordData(IsFilterable = true)] - public int HotelCode { get; init; } - - [VectorStoreRecordData(IsFullTextSearchable = true)] - public string Description { get; init; } - - [VectorStoreRecordVector(4)] - public ReadOnlyMemory? DescriptionEmbedding { get; init; } - -#pragma warning disable CA1819 // Properties should not return arrays - [VectorStoreRecordData(IsFilterable = true)] - public string[] Tags { get; init; } - - [VectorStoreRecordData(IsFullTextSearchable = true)] - public string[] FTSTags { get; init; } -#pragma warning restore CA1819 // Properties should not return arrays - - [JsonPropertyName("parking_is_included")] - [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] - public bool ParkingIncluded { get; init; } - - [VectorStoreRecordData] - public DateTimeOffset LastRenovationDate { get; init; } - - [VectorStoreRecordData] - public double Rating { get; init; } - - [VectorStoreRecordData] - public HotelAddress Address { get; init; } - } - - /// - /// A test model for the vector store to simulate a complex type. - /// - public class HotelAddress - { - public string City { get; init; } - public string Country { get; init; } - } - - /// - /// A test model for the vector store that only uses basic types as supported by HashSets Redis mode. - /// - public class BasicHotel - { - [VectorStoreRecordKey] - public string HotelId { get; init; } - - [VectorStoreRecordData(IsFilterable = true)] - public string HotelName { get; init; } - - [VectorStoreRecordData(IsFilterable = true)] - public int HotelCode { get; init; } - - [VectorStoreRecordData(IsFullTextSearchable = true)] - public string Description { get; init; } - - [VectorStoreRecordVector(4)] - public ReadOnlyMemory? DescriptionEmbedding { get; init; } - - [JsonPropertyName("parking_is_included")] - [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] - public bool ParkingIncluded { get; init; } - - [VectorStoreRecordData] - public double Rating { get; init; } - } - - /// - /// A test model for the vector store that only uses basic types as supported by HashSets Redis mode. - /// - public class BasicFloat32Hotel : BasicHotel - { - } - - /// - /// A test model for the vector store that only uses basic types as supported by HashSets Redis mode. - /// - public class BasicFloat64Hotel : BasicHotel - { - } } -#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisVectorStoreTests.cs index 8e18522928eb..edde28dea285 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisVectorStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Redis/RedisVectorStoreTests.cs @@ -1,39 +1,25 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Linq; using System.Threading.Tasks; using Microsoft.SemanticKernel.Connectors.Redis; using Xunit; -using Xunit.Abstractions; namespace SemanticKernel.IntegrationTests.Connectors.Memory.Redis; /// /// Contains tests for the class. /// -/// Used to write to the test output stream. /// The test fixture. [Collection("RedisVectorStoreCollection")] -public class RedisVectorStoreTests(ITestOutputHelper output, RedisVectorStoreFixture fixture) +public class RedisVectorStoreTests(RedisVectorStoreFixture fixture) + : BaseVectorStoreTests(new RedisVectorStore(fixture.Database)) { // If null, all tests will be enabled - private const string SkipReason = "Requires Redis docker container up and running"; + private const string SkipReason = "This test is for manual verification"; [Fact(Skip = SkipReason)] - public async Task ItCanGetAListOfExistingCollectionNamesAsync() + public override async Task ItCanGetAListOfExistingCollectionNamesAsync() { - // Arrange - var sut = new RedisVectorStore(fixture.Database); - - // Act - var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); - - // Assert - Assert.Equal(2, collectionNames.Count); - Assert.Contains("jsonhotels", collectionNames); - Assert.Contains("hashhotels", collectionNames); - - // Output - output.WriteLine(string.Join(",", collectionNames)); + await base.ItCanGetAListOfExistingCollectionNamesAsync(); } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Sqlite/SqliteVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Sqlite/SqliteVectorStoreTests.cs index dc23b633b5b7..755f79195a93 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Sqlite/SqliteVectorStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Sqlite/SqliteVectorStoreTests.cs @@ -15,27 +15,14 @@ namespace SemanticKernel.IntegrationTests.Connectors.Memory.Sqlite; /// [Collection("SqliteVectorStoreCollection")] public sealed class SqliteVectorStoreTests(SqliteVectorStoreFixture fixture) + : BaseVectorStoreTests>(new SqliteVectorStore(fixture.Connection!)) { private const string? SkipReason = "SQLite vector search extension is required"; [Fact(Skip = SkipReason)] - public async Task ItCanGetAListOfExistingCollectionNamesAsync() + public override async Task ItCanGetAListOfExistingCollectionNamesAsync() { - // Arrange - var sut = new SqliteVectorStore(fixture.Connection!); - - var collection1 = fixture.GetCollection>("ListCollectionNames1"); - var collection2 = fixture.GetCollection>("ListCollectionNames2"); - - await collection1.CreateCollectionIfNotExistsAsync(); - await collection2.CreateCollectionIfNotExistsAsync(); - - // Act - var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); - - // Assert - Assert.Contains("ListCollectionNames1", collectionNames); - Assert.Contains("ListCollectionNames1", collectionNames); + await base.ItCanGetAListOfExistingCollectionNamesAsync(); } [Fact(Skip = SkipReason)] diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreTests.cs index 7de9413084ae..ce278486e9bc 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateVectorStoreTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Linq; -using System.Threading.Tasks; +using System; using Microsoft.SemanticKernel.Connectors.Weaviate; using Xunit; @@ -9,27 +8,5 @@ namespace SemanticKernel.IntegrationTests.Connectors.Memory.Weaviate; [Collection("WeaviateVectorStoreCollection")] public sealed class WeaviateVectorStoreTests(WeaviateVectorStoreFixture fixture) -{ - [Fact] - public async Task ItCanGetAListOfExistingCollectionNamesAsync() - { - // Arrange - var collection1 = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "Collection1"); - var collection2 = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "Collection2"); - var collection3 = new WeaviateVectorStoreRecordCollection(fixture.HttpClient!, "Collection3"); - - await collection1.CreateCollectionAsync(); - await collection2.CreateCollectionAsync(); - await collection3.CreateCollectionAsync(); - - var sut = new WeaviateVectorStore(fixture.HttpClient!); - - // Act - var collectionNames = await sut.ListCollectionNamesAsync().ToListAsync(); - - // Assert - Assert.Contains("Collection1", collectionNames); - Assert.Contains("Collection2", collectionNames); - Assert.Contains("Collection3", collectionNames); - } -} + : BaseVectorStoreTests(new WeaviateVectorStore(fixture.HttpClient!)) +{ } diff --git a/dotnet/src/IntegrationTests/testsettings.json b/dotnet/src/IntegrationTests/testsettings.json index c2396c7c0419..22c91e9affcc 100644 --- a/dotnet/src/IntegrationTests/testsettings.json +++ b/dotnet/src/IntegrationTests/testsettings.json @@ -95,7 +95,8 @@ "VectorSearchCollection": "dotnetMSKNearestTest.nearestSearch" }, "AzureCosmosDBNoSQL": { - "ConnectionString": "" + "ConnectionString": "", + "Endpoint": "" }, "AzureCosmosDBMongoDB": { "ConnectionString": "" diff --git a/dotnet/src/Plugins/Plugins.Core/CodeInterpreter/SessionsPythonPlugin.cs b/dotnet/src/Plugins/Plugins.Core/CodeInterpreter/SessionsPythonPlugin.cs index c8094de65201..bebd972fe350 100644 --- a/dotnet/src/Plugins/Plugins.Core/CodeInterpreter/SessionsPythonPlugin.cs +++ b/dotnet/src/Plugins/Plugins.Core/CodeInterpreter/SessionsPythonPlugin.cs @@ -113,6 +113,8 @@ public async Task ExecuteCodeAsync([Description("The valid Python code t var jsonElementResult = JsonSerializer.Deserialize(await response.Content.ReadAsStringAsync().ConfigureAwait(false)); return $""" + Status: + {jsonElementResult.GetProperty("status").GetRawText()} Result: {jsonElementResult.GetProperty("result").GetRawText()} Stdout: diff --git a/dotnet/src/Plugins/Plugins.UnitTests/Core/SessionsPythonPluginTests.cs b/dotnet/src/Plugins/Plugins.UnitTests/Core/SessionsPythonPluginTests.cs index 789cd85fc353..f9a5e9fc1bcb 100644 --- a/dotnet/src/Plugins/Plugins.UnitTests/Core/SessionsPythonPluginTests.cs +++ b/dotnet/src/Plugins/Plugins.UnitTests/Core/SessionsPythonPluginTests.cs @@ -70,6 +70,8 @@ public async Task ItShouldExecuteCodeAsync() Content = new StringContent(responseContent), }; var expectedResult = """ + Status: + "Success" Result: "" Stdout: diff --git a/python/samples/concepts/chat_completion/simple_chatbot_store_metadata.py b/python/samples/concepts/chat_completion/simple_chatbot_store_metadata.py new file mode 100644 index 000000000000..0c59ff9dc5ec --- /dev/null +++ b/python/samples/concepts/chat_completion/simple_chatbot_store_metadata.py @@ -0,0 +1,85 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio + +from samples.concepts.setup.chat_completion_services import ( + Services, + get_chat_completion_service_and_request_settings, +) +from semantic_kernel.contents import ChatHistory + +# This sample shows how to create a chatbot whose output can be stored for use with the OpenAI +# model distillation or evals products. +# This sample uses the following two main components: +# - a ChatCompletionService: This component is responsible for generating responses to user messages. +# - a ChatHistory: This component is responsible for keeping track of the chat history. +# The chatbot in this sample is called Mosscap, who is an expert in basketball. + +# To learn more about OpenAI distillation, see: https://platform.openai.com/docs/guides/distillation +# To learn more about OpenAI evals, see: https://platform.openai.com/docs/guides/evals + + +# You can select from the following chat completion services: +# - Services.OPENAI +# Please make sure you have configured your environment correctly for the selected chat completion service. +chat_completion_service, request_settings = get_chat_completion_service_and_request_settings(Services.OPENAI) + +# This is the system message that gives the chatbot its personality. +system_message = """ +You are a chat bot whose expertise is basketball. +Your name is Mosscap and you have one goal: to answer questions about basketball. +""" + +# Create a chat history object with the system message. +chat_history = ChatHistory(system_message=system_message) +# Configure the store and metadata settings for the chat completion service. +request_settings.store = True +request_settings.metadata = {"chatbot": "Mosscap"} + + +async def chat() -> bool: + try: + user_input = input("User:> ") + except KeyboardInterrupt: + print("\n\nExiting chat...") + return False + except EOFError: + print("\n\nExiting chat...") + return False + + if user_input == "exit": + print("\n\nExiting chat...") + return False + + # Add the user message to the chat history so that the chatbot can respond to it. + chat_history.add_user_message(user_input) + + # Get the chat message content from the chat completion service. + response = await chat_completion_service.get_chat_message_content( + chat_history=chat_history, + settings=request_settings, + ) + if response: + print(f"Mosscap:> {response}") + + # Add the chat message to the chat history to keep track of the conversation. + chat_history.add_message(response) + + return True + + +async def main() -> None: + # Start the chat loop. The chat loop will continue until the user types "exit". + chatting = True + while chatting: + chatting = await chat() + + # Sample output: + # User:> Who has the most career points in NBA history? + # Mosscap:> As of October 2023, the all-time leader in total regular-season scoring in the history of the National + # Basketball Association (N.B.A.) is Kareem Abdul-Jabbar, who scored 38,387 total regular-seasonPoints + # during his illustrious 20-year playing Career. + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_prompt_execution_settings.py b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_prompt_execution_settings.py index 1ff6c993ea24..12451d35296f 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_prompt_execution_settings.py +++ b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_prompt_execution_settings.py @@ -33,6 +33,8 @@ class OpenAIPromptExecutionSettings(PromptExecutionSettings): temperature: Annotated[float | None, Field(ge=0.0, le=2.0)] = None top_p: Annotated[float | None, Field(ge=0.0, le=1.0)] = None user: str | None = None + store: bool | None = None + metadata: dict[str, str] | None = None class OpenAITextPromptExecutionSettings(OpenAIPromptExecutionSettings):