diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index ff2e7b41618b..a6fed7b35ec4 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -13,7 +13,7 @@ - + @@ -40,7 +40,7 @@ - + @@ -69,7 +69,7 @@ - + @@ -77,7 +77,7 @@ - + diff --git a/dotnet/samples/Demos/VectorStoreRAG/DataLoader.cs b/dotnet/samples/Demos/VectorStoreRAG/DataLoader.cs index ea798c6a881d..84241ce5d3f2 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/DataLoader.cs +++ b/dotnet/samples/Demos/VectorStoreRAG/DataLoader.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Net; using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Embeddings; using UglyToad.PdfPig; using UglyToad.PdfPig.Content; @@ -21,14 +23,14 @@ internal sealed class DataLoader( ITextEmbeddingGenerationService textEmbeddingGenerationService) : IDataLoader where TKey : notnull { /// - public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken) + public async Task LoadPdf(string pdfPath, int batchSize, int betweenBatchDelayInMs, CancellationToken cancellationToken) { // Create the collection if it doesn't exist. await vectorStoreRecordCollection.CreateCollectionIfNotExistsAsync(cancellationToken).ConfigureAwait(false); // Load the paragraphs from the PDF file and split them into batches. var sections = LoadParagraphs(pdfPath, cancellationToken); - var batches = sections.Chunk(10); + var batches = sections.Chunk(batchSize); // Process each batch of paragraphs. foreach (var batch in batches) @@ -40,7 +42,7 @@ public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken) Text = section.ParagraphText, ReferenceDescription = $"{new FileInfo(pdfPath).Name}#page={section.PageNumber}", ReferenceLink = $"{new Uri(new FileInfo(pdfPath).FullName).AbsoluteUri}#page={section.PageNumber}", - TextEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(section.ParagraphText, cancellationToken: cancellationToken).ConfigureAwait(false) + TextEmbedding = await GenerateEmbeddingsWithRetryAsync(textEmbeddingGenerationService, section.ParagraphText, cancellationToken: cancellationToken).ConfigureAwait(false) }); // Upsert the records into the vector store. @@ -50,6 +52,8 @@ public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken) { Console.WriteLine($"Upserted record '{key}' into VectorDB"); } + + await Task.Delay(betweenBatchDelayInMs, cancellationToken).ConfigureAwait(false); } } @@ -83,4 +87,39 @@ public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken) } } } + + /// + /// Add a simple retry mechanism to embedding generation. + /// + /// The embedding generation service. + /// The text to generate the embedding for. + /// The to monitor for cancellation requests. + /// The generated embedding. + private static async Task> GenerateEmbeddingsWithRetryAsync(ITextEmbeddingGenerationService textEmbeddingGenerationService, string text, CancellationToken cancellationToken) + { + var tries = 0; + + while (true) + { + try + { + return await textEmbeddingGenerationService.GenerateEmbeddingAsync(text, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (HttpOperationException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests) + { + tries++; + + if (tries < 3) + { + Console.WriteLine($"Failed to generate embedding. Error: {ex}"); + Console.WriteLine("Retrying embedding generation..."); + await Task.Delay(10_000, cancellationToken).ConfigureAwait(false); + } + else + { + throw; + } + } + } + } } diff --git a/dotnet/samples/Demos/VectorStoreRAG/IDataLoader.cs b/dotnet/samples/Demos/VectorStoreRAG/IDataLoader.cs index abd2eed7e8ef..2a2ecac04e20 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/IDataLoader.cs +++ b/dotnet/samples/Demos/VectorStoreRAG/IDataLoader.cs @@ -11,7 +11,9 @@ internal interface IDataLoader /// Load the text from a PDF file into the data store. /// /// The pdf file to load. + /// Maximum number of parallel threads to generate embeddings and upload records. + /// The number of milliseconds to delay between batches to avoid throttling. /// The to monitor for cancellation requests. /// An async task that completes when the loading is complete. - Task LoadPdf(string pdfPath, CancellationToken cancellationToken); + Task LoadPdf(string pdfPath, int batchSize, int betweenBatchDelayInMs, CancellationToken cancellationToken); } diff --git a/dotnet/samples/Demos/VectorStoreRAG/Options/RagConfig.cs b/dotnet/samples/Demos/VectorStoreRAG/Options/RagConfig.cs index b7a919d890bc..4a524a777b05 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/Options/RagConfig.cs +++ b/dotnet/samples/Demos/VectorStoreRAG/Options/RagConfig.cs @@ -22,4 +22,10 @@ internal sealed class RagConfig [Required] public string CollectionName { get; set; } = string.Empty; + + [Required] + public int DataLoadingBatchSize { get; set; } = 2; + + [Required] + public int DataLoadingBetweenBatchDelayInMilliseconds { get; set; } = 0; } diff --git a/dotnet/samples/Demos/VectorStoreRAG/Program.cs b/dotnet/samples/Demos/VectorStoreRAG/Program.cs index 39fc7522f506..f420c35a7baf 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/Program.cs +++ b/dotnet/samples/Demos/VectorStoreRAG/Program.cs @@ -13,13 +13,17 @@ HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); -builder.Configuration - .AddUserSecrets(); - +// Configure configuration and load the application configuration. +builder.Configuration.AddUserSecrets(); builder.Services.Configure(builder.Configuration.GetSection(RagConfig.ConfigSectionName)); - var appConfig = new ApplicationConfig(builder.Configuration); +// Create a cancellation token and source to pass to the application service to allow them +// to request a graceful application shutdown. +CancellationTokenSource appShutdownCancellationTokenSource = new(); +CancellationToken appShutdownCancellationToken = appShutdownCancellationTokenSource.Token; +builder.Services.AddKeyedSingleton("AppShutdown", appShutdownCancellationTokenSource); + // Register the kernel with the dependency injection container // and add Chat Completion and Text Embedding Generation services. var kernelBuilder = builder.Services.AddKernel() @@ -54,6 +58,10 @@ appConfig.AzureCosmosDBNoSQLConfig.ConnectionString, appConfig.AzureCosmosDBNoSQLConfig.DatabaseName); break; + case "InMemory": + kernelBuilder.AddInMemoryVectorStoreRecordCollection>( + appConfig.RagConfig.CollectionName); + break; case "Qdrant": kernelBuilder.AddQdrantVectorStoreRecordCollection>( appConfig.RagConfig.CollectionName, @@ -84,6 +92,7 @@ case "AzureAISearch": case "AzureCosmosDBMongoDB": case "AzureCosmosDBNoSQL": + case "InMemory": case "Redis": RegisterServices(builder, kernelBuilder, appConfig); break; @@ -97,7 +106,7 @@ // Build and run the host. using IHost host = builder.Build(); -await host.RunAsync().ConfigureAwait(false); +await host.RunAsync(appShutdownCancellationToken).ConfigureAwait(false); static void RegisterServices(HostApplicationBuilder builder, IKernelBuilder kernelBuilder, ApplicationConfig vectorStoreRagConfig) where TKey : notnull diff --git a/dotnet/samples/Demos/VectorStoreRAG/RAGChatService.cs b/dotnet/samples/Demos/VectorStoreRAG/RAGChatService.cs index 0b0bf875c1c0..6b3af97d8824 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/RAGChatService.cs +++ b/dotnet/samples/Demos/VectorStoreRAG/RAGChatService.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Options; using Microsoft.SemanticKernel; @@ -17,11 +18,13 @@ namespace VectorStoreRAG; /// Used to search the vector store. /// Used to make requests to the LLM. /// The configuration options for the application. +/// Used to gracefully shut down the entire application when cancelled. internal sealed class RAGChatService( IDataLoader dataLoader, VectorStoreTextSearch> vectorStoreTextSearch, Kernel kernel, - IOptions ragConfigOptions) : IHostedService + IOptions ragConfigOptions, + [FromKeyedServices("AppShutdown")] CancellationTokenSource appShutdownCancellationTokenSource) : IHostedService { private Task? _dataLoaded; private Task? _chatLoop; @@ -83,6 +86,9 @@ private async Task ChatLoopAsync(CancellationToken cancellationToken) Console.WriteLine("PDF loading complete\n"); + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine("Assistant > Press enter with no prompt to exit."); + // Add a search plugin to the kernel which we will use in the template below // to do a vector search for related information to the user query. kernel.Plugins.Add(vectorStoreTextSearch.CreateWithGetTextSearchResults("SearchPlugin")); @@ -99,6 +105,13 @@ private async Task ChatLoopAsync(CancellationToken cancellationToken) Console.Write("User > "); var question = Console.ReadLine(); + // Exit the application if the user didn't type anything. + if (string.IsNullOrWhiteSpace(question)) + { + appShutdownCancellationTokenSource.Cancel(); + break; + } + // Invoke the LLM with a template that uses the search plugin to // 1. get related information to the user query from the vector store // 2. add the information to the LLM prompt. @@ -158,7 +171,11 @@ private async Task LoadDataAsync(CancellationToken cancellationToken) foreach (var pdfFilePath in ragConfigOptions.Value.PdfFilePaths ?? []) { Console.WriteLine($"Loading PDF into vector store: {pdfFilePath}"); - await dataLoader.LoadPdf(pdfFilePath, cancellationToken).ConfigureAwait(false); + await dataLoader.LoadPdf( + pdfFilePath, + ragConfigOptions.Value.DataLoadingBatchSize, + ragConfigOptions.Value.DataLoadingBetweenBatchDelayInMilliseconds, + cancellationToken).ConfigureAwait(false); } } catch (Exception ex) diff --git a/dotnet/samples/Demos/VectorStoreRAG/README.md b/dotnet/samples/Demos/VectorStoreRAG/README.md index bd7abc41ae30..dc5a1a5a2317 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/README.md +++ b/dotnet/samples/Demos/VectorStoreRAG/README.md @@ -11,12 +11,15 @@ The sample can be configured in various ways: 1. AzureAISearch 1. AzureCosmosDBMongoDB 1. AzureCosmosDBNoSQL + 1. InMemory 1. Qdrant 1. Redis 1. Weaviate 1. You can choose whether to load data into the vector store by setting the `Rag:BuildCollection` configuration setting in the `appsettings.json` file to `true`. If you set this to `false`, the sample will assume that data was already loaded previously and it will go straight into the chat experience. 1. You can choose the name of the collection to use by setting the `Rag:CollectionName` configuration setting in the `appsettings.json` file. 1. You can choose the pdf file to load into the vector store by setting the `Rag:PdfFilePaths` array in the `appsettings.json` file. +1. You can choose the number of records to process per batch when loading data into the vector store by setting the `Rag:DataLoadingBatchSize` configuration setting in the `appsettings.json` file. +1. You can choose the number of milliseconds to wait between batches when loading data into the vector store by setting the `Rag:DataLoadingBetweenBatchDelayInMilliseconds` configuration setting in the `appsettings.json` file. ## Dependency Setup @@ -45,6 +48,7 @@ For Azure OpenAI, you need to add the following secrets: ```cli dotnet user-secrets set "AIServices:AzureOpenAI:Endpoint" "https://.openai.azure.com" +dotnet user-secrets set "AIServices:AzureOpenAI:ChatDeploymentName" "" ``` Note that the code doesn't use an API Key to communicate with Azure Open AI, but rather an `AzureCliCredential` so no api key secret is required. @@ -55,6 +59,7 @@ For Azure OpenAI Embeddings, you need to add the following secrets: ```cli dotnet user-secrets set "AIServices:AzureOpenAIEmbeddings:Endpoint" "https://.openai.azure.com" +dotnet user-secrets set "AIServices:AzureOpenAIEmbeddings:DeploymentName" "" ``` Note that the code doesn't use an API Key to communicate with Azure Open AI, but rather an `AzureCliCredential` so no api key secret is required. diff --git a/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj b/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj index fd80cbe8c047..7b1557a8005c 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj +++ b/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj @@ -21,6 +21,7 @@ + diff --git a/dotnet/samples/Demos/VectorStoreRAG/appsettings.json b/dotnet/samples/Demos/VectorStoreRAG/appsettings.json index 47a1daa9d4d4..be581ad34d9d 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/appsettings.json +++ b/dotnet/samples/Demos/VectorStoreRAG/appsettings.json @@ -43,7 +43,9 @@ "Rag": { "BuildCollection": true, "PdfFilePaths": [ "sourcedocument.pdf" ], - "VectorStoreType": "Qdrant", - "CollectionName": "pdfcontent" + "VectorStoreType": "InMemory", + "CollectionName": "pdfcontent", + "DataLoadingBatchSize": 2, + "DataLoadingBetweenBatchDelayInMilliseconds": 1000 } } diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs index ae2a5ad0da43..7fe5e3875fb8 100644 --- a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs @@ -39,6 +39,6 @@ public class AzureCosmosDBMongoDBHotelModel(string hotelId) public string? Description { get; set; } /// A vector field. - [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)] + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs index 00f6272a9c53..b9d74ac4667e 100644 --- a/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs @@ -828,11 +828,11 @@ private sealed class VectorSearchModel [VectorStoreRecordData] public string? HotelName { get; set; } - [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance, StoragePropertyName = "test_embedding_1")] + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat, StoragePropertyName = "test_embedding_1")] public ReadOnlyMemory TestEmbedding1 { get; set; } [BsonElement("test_embedding_2")] - [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)] + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)] public ReadOnlyMemory TestEmbedding2 { get; set; } } #pragma warning restore CA1812 diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLHotel.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLHotel.cs index 45a2a6036297..df06e97d3846 100644 --- a/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLHotel.cs +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLHotel.cs @@ -39,6 +39,6 @@ public class AzureCosmosDBNoSQLHotel(string hotelId) /// A vector field. [JsonPropertyName("description_embedding")] - [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)] + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } diff --git a/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLVectorStoreRecordCollectionTests.cs index 5410ed752e04..9cb9ac03c681 100644 --- a/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLVectorStoreRecordCollectionTests.cs @@ -701,16 +701,16 @@ private sealed class TestIndexingModel [VectorStoreRecordKey] public string? Id { get; set; } - [VectorStoreRecordVector(Dimensions: 1, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)] + [VectorStoreRecordVector(Dimensions: 1, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)] public ReadOnlyMemory? DescriptionEmbedding1 { get; set; } - [VectorStoreRecordVector(Dimensions: 2, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)] + [VectorStoreRecordVector(Dimensions: 2, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)] public ReadOnlyMemory? DescriptionEmbedding2 { get; set; } - [VectorStoreRecordVector(Dimensions: 3, IndexKind: IndexKind.QuantizedFlat, DistanceFunction: DistanceFunction.DotProductSimilarity)] + [VectorStoreRecordVector(Dimensions: 3, DistanceFunction: DistanceFunction.DotProductSimilarity, IndexKind: IndexKind.QuantizedFlat)] public ReadOnlyMemory? DescriptionEmbedding3 { get; set; } - [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.DiskAnn, DistanceFunction: DistanceFunction.EuclideanDistance)] + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.EuclideanDistance, IndexKind: IndexKind.DiskAnn)] public ReadOnlyMemory? DescriptionEmbedding4 { get; set; } [VectorStoreRecordData(IsFilterable = true)] diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/Connectors.Memory.AzureAISearch.csproj b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/Connectors.Memory.AzureAISearch.csproj index 1b8b979b91f2..585e946ccbd0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/Connectors.Memory.AzureAISearch.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/Connectors.Memory.AzureAISearch.csproj @@ -4,7 +4,7 @@ Microsoft.SemanticKernel.Connectors.AzureAISearch Microsoft.SemanticKernel.Connectors.AzureAISearch net8.0;netstandard2.0 - alpha + preview $(NoWarn);NU5104 diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/Connectors.Memory.AzureCosmosDBMongoDB.csproj b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/Connectors.Memory.AzureCosmosDBMongoDB.csproj index 9ce9d24d1aed..9a2d68d24132 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/Connectors.Memory.AzureCosmosDBMongoDB.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/Connectors.Memory.AzureCosmosDBMongoDB.csproj @@ -6,7 +6,7 @@ $(AssemblyName) net8.0;netstandard2.0 $(NoWarn);NU5104;SKEXP0001,SKEXP0010 - alpha + preview diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/Connectors.Memory.AzureCosmosDBNoSQL.csproj b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/Connectors.Memory.AzureCosmosDBNoSQL.csproj index 32fd24223c81..9e065f2d7db8 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/Connectors.Memory.AzureCosmosDBNoSQL.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/Connectors.Memory.AzureCosmosDBNoSQL.csproj @@ -6,7 +6,7 @@ $(AssemblyName) net8.0;netstandard2.0 $(NoWarn);NU5104;SKEXP0001,SKEXP0010 - alpha + preview diff --git a/dotnet/src/Connectors/Connectors.Memory.InMemory/Connectors.Memory.InMemory.csproj b/dotnet/src/Connectors/Connectors.Memory.InMemory/Connectors.Memory.InMemory.csproj index 01877d8cd681..1815446a8f80 100644 --- a/dotnet/src/Connectors/Connectors.Memory.InMemory/Connectors.Memory.InMemory.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.InMemory/Connectors.Memory.InMemory.csproj @@ -4,7 +4,7 @@ Microsoft.SemanticKernel.Connectors.InMemory $(AssemblyName) net8.0;netstandard2.0 - alpha + preview diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj index 69b47fe172f0..c9c647171150 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj @@ -5,7 +5,7 @@ Microsoft.SemanticKernel.Connectors.Pinecone $(AssemblyName) net8.0;netstandard2.0 - alpha + preview diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Connectors.Memory.Qdrant.csproj b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Connectors.Memory.Qdrant.csproj index 322a58d22400..93425c69fbe9 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/Connectors.Memory.Qdrant.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/Connectors.Memory.Qdrant.csproj @@ -5,7 +5,7 @@ Microsoft.SemanticKernel.Connectors.Qdrant $(AssemblyName) net8.0;netstandard2.0 - alpha + preview diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/Connectors.Memory.Redis.csproj b/dotnet/src/Connectors/Connectors.Memory.Redis/Connectors.Memory.Redis.csproj index 52f2ea6f159c..3f2fd4360fe8 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/Connectors.Memory.Redis.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/Connectors.Memory.Redis.csproj @@ -5,7 +5,7 @@ Microsoft.SemanticKernel.Connectors.Redis $(AssemblyName) net8.0;netstandard2.0 - alpha + preview diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj index 06901c892c22..26b63c694dff 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/Connectors.Memory.Weaviate.csproj @@ -5,7 +5,7 @@ Microsoft.SemanticKernel.Connectors.Weaviate $(AssemblyName) net8.0;netstandard2.0 - alpha + preview diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs index cf6e6ab7b27c..9ce781b39b8b 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs @@ -44,6 +44,6 @@ public sealed record WeaviateHotel public DateTimeOffset Timestamp { get; set; } /// A vector field. - [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Hnsw, DistanceFunction: DistanceFunction.CosineDistance)] + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.Hnsw)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/RecordAttributes/VectorStoreRecordVectorAttribute.cs b/dotnet/src/Connectors/VectorData.Abstractions/RecordAttributes/VectorStoreRecordVectorAttribute.cs index 83c5d8ba1edc..e86a0883574c 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/RecordAttributes/VectorStoreRecordVectorAttribute.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/RecordAttributes/VectorStoreRecordVectorAttribute.cs @@ -33,13 +33,24 @@ public VectorStoreRecordVectorAttribute(int Dimensions) /// Initializes a new instance of the class. /// /// The number of dimensions that the vector has. - /// The kind of index to use. /// The distance function to use when comparing vectors. - public VectorStoreRecordVectorAttribute(int Dimensions, string? IndexKind, string? DistanceFunction) + public VectorStoreRecordVectorAttribute(int Dimensions, string? DistanceFunction) + { + this.Dimensions = Dimensions; + this.DistanceFunction = DistanceFunction; + } + + /// + /// Initializes a new instance of the class. + /// + /// The number of dimensions that the vector has. + /// The distance function to use when comparing vectors. + /// The kind of index to use. + public VectorStoreRecordVectorAttribute(int Dimensions, string? DistanceFunction, string? IndexKind) { this.Dimensions = Dimensions; - this.IndexKind = IndexKind; this.DistanceFunction = DistanceFunction; + this.IndexKind = IndexKind; } /// diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs index c38672d2da62..36ce5c0ca321 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreFixture.cs @@ -117,7 +117,7 @@ public record AzureCosmosDBMongoDBHotel() public DateTime Timestamp { get; set; } /// A vector field. - [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)] + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } #pragma warning restore CS8618 diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLHotel.cs index b7de210f7d18..e7d353486504 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLHotel.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/AzureCosmosDBNoSQL/AzureCosmosDBNoSQLHotel.cs @@ -45,6 +45,6 @@ public record AzureCosmosDBNoSQLHotel() public DateTimeOffset Timestamp { get; set; } /// A vector field. - [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)] + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs index a9dbc72e3862..7e640ea968e1 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs @@ -59,6 +59,6 @@ public record PineconeAllTypes() [VectorStoreRecordData] public IEnumerable Enumerable { get; set; } - [VectorStoreRecordVector(Dimensions: 8, IndexKind: null, DistanceFunction: DistanceFunction.DotProductSimilarity)] + [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)] public ReadOnlyMemory? Embedding { get; set; } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs index af688fcf02a5..3603f2b5ef04 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs @@ -34,6 +34,6 @@ public record PineconeHotel() [VectorStoreRecordData] public string Description { get; set; } - [VectorStoreRecordVector(Dimensions: 8, IndexKind: null, DistanceFunction: DistanceFunction.DotProductSimilarity)] + [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)] public ReadOnlyMemory DescriptionEmbedding { get; set; } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs index 5949533b4680..532d840309ba 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs @@ -555,7 +555,7 @@ private sealed record PineconeRecordWithUnsupportedMetric [VectorStoreRecordData] public string? Name { get; set; } - [VectorStoreRecordVector(Dimensions: 5, IndexKind: null, DistanceFunction: "just eyeball it")] + [VectorStoreRecordVector(Dimensions: 5, DistanceFunction: "just eyeball it")] public ReadOnlyMemory Embedding { get; set; } } #pragma warning restore CA1812 diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreFixture.cs index af4f94299005..c101ea570597 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreFixture.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Qdrant/QdrantVectorStoreFixture.cs @@ -335,7 +335,7 @@ public record HotelInfo() public string Description { get; set; } /// A vector field. - [VectorStoreRecordVector(VectorDimensions, IndexKind.Hnsw, DistanceFunction.ManhattanDistance)] + [VectorStoreRecordVector(VectorDimensions, DistanceFunction.ManhattanDistance, IndexKind.Hnsw)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } @@ -358,7 +358,7 @@ public record HotelInfoWithGuidId() public string Description { get; set; } /// A vector field. - [VectorStoreRecordVector(VectorDimensions, IndexKind.Hnsw, DistanceFunction.ManhattanDistance)] + [VectorStoreRecordVector(VectorDimensions, DistanceFunction.ManhattanDistance, IndexKind.Hnsw)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } } diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs index b06b8a71d729..bfcd78c9a51c 100644 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs +++ b/dotnet/src/IntegrationTests/Connectors/Memory/Weaviate/WeaviateHotel.cs @@ -44,6 +44,6 @@ public sealed record WeaviateHotel public DateTimeOffset Timestamp { get; set; } /// A vector field. - [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Hnsw, DistanceFunction: DistanceFunction.CosineDistance)] + [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.Hnsw)] public ReadOnlyMemory? DescriptionEmbedding { get; set; } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreRecordPropertyReaderTests.cs b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreRecordPropertyReaderTests.cs index f5e4a1eb044d..a3b26a313b10 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreRecordPropertyReaderTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreRecordPropertyReaderTests.cs @@ -552,7 +552,7 @@ private sealed class MultiPropsModel [JsonPropertyName("json_data2")] public string Data2 { get; set; } = string.Empty; - [VectorStoreRecordVector(4, IndexKind.Flat, DistanceFunction.DotProductSimilarity)] + [VectorStoreRecordVector(4, DistanceFunction.DotProductSimilarity, IndexKind.Flat)] public ReadOnlyMemory Vector1 { get; set; } [VectorStoreRecordVector(StoragePropertyName = "storage_vector2")]