Merge branch 'main' into vector-store-vectorless-search

dotnet/Directory.Packages.props

@@ -13,7 +13,7 @@
     <PackageVersion Include="Azure.Identity" Version="1.12.0" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.3.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.6.0" />
-    <PackageVersion Include="Handlebars.Net.Helpers" Version="2.4.5" />
+    <PackageVersion Include="Handlebars.Net.Helpers" Version="2.4.6" />
     <PackageVersion Include="Markdig" Version="0.37.0" />
     <PackageVersion Include="Handlebars.Net" Version="2.1.6" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.11.67" />
@@ -77,7 +77,7 @@
     <PackageVersion Include="Docker.DotNet" Version="3.125.15" />
     <!-- Plugins -->
     <PackageVersion Include="DocumentFormat.OpenXml" Version="3.1.0" />
-    <PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
+    <PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.10" />
     <PackageVersion Include="DuckDB.NET.Data.Full" Version="1.1.1" />
     <PackageVersion Include="DuckDB.NET.Data" Version="1.1.1" />
     <PackageVersion Include="MongoDB.Driver" Version="2.28.0" />

dotnet/samples/Demos/VectorStoreRAG/DataLoader.cs

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System.Net;
 using Microsoft.Extensions.VectorData;
+using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.Embeddings;
 using UglyToad.PdfPig;
 using UglyToad.PdfPig.Content;
@@ -21,14 +23,14 @@ internal sealed class DataLoader<TKey>(
     ITextEmbeddingGenerationService textEmbeddingGenerationService) : IDataLoader where TKey : notnull
 {
     /// <inheritdoc/>
-    public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken)
+    public async Task LoadPdf(string pdfPath, int batchSize, int betweenBatchDelayInMs, CancellationToken cancellationToken)
     {
         // Create the collection if it doesn't exist.
         await vectorStoreRecordCollection.CreateCollectionIfNotExistsAsync(cancellationToken).ConfigureAwait(false);
 
         // Load the paragraphs from the PDF file and split them into batches.
         var sections = LoadParagraphs(pdfPath, cancellationToken);
-        var batches = sections.Chunk(10);
+        var batches = sections.Chunk(batchSize);
 
         // Process each batch of paragraphs.
         foreach (var batch in batches)
@@ -40,7 +42,7 @@ public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken)
                 Text = section.ParagraphText,
                 ReferenceDescription = $"{new FileInfo(pdfPath).Name}#page={section.PageNumber}",
                 ReferenceLink = $"{new Uri(new FileInfo(pdfPath).FullName).AbsoluteUri}#page={section.PageNumber}",
-                TextEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(section.ParagraphText, cancellationToken: cancellationToken).ConfigureAwait(false)
+                TextEmbedding = await GenerateEmbeddingsWithRetryAsync(textEmbeddingGenerationService, section.ParagraphText, cancellationToken: cancellationToken).ConfigureAwait(false)
             });
 
             // Upsert the records into the vector store.
@@ -50,6 +52,8 @@ public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken)
             {
                 Console.WriteLine($"Upserted record '{key}' into VectorDB");
             }
+
+            await Task.Delay(betweenBatchDelayInMs, cancellationToken).ConfigureAwait(false);
         }
     }
 
@@ -83,4 +87,39 @@ public async Task LoadPdf(string pdfPath, CancellationToken cancellationToken)
             }
         }
     }
+
+    /// <summary>
+    /// Add a simple retry mechanism to embedding generation.
+    /// </summary>
+    /// <param name="textEmbeddingGenerationService">The embedding generation service.</param>
+    /// <param name="text">The text to generate the embedding for.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
+    /// <returns>The generated embedding.</returns>
+    private static async Task<ReadOnlyMemory<float>> GenerateEmbeddingsWithRetryAsync(ITextEmbeddingGenerationService textEmbeddingGenerationService, string text, CancellationToken cancellationToken)
+    {
+        var tries = 0;
+
+        while (true)
+        {
+            try
+            {
+                return await textEmbeddingGenerationService.GenerateEmbeddingAsync(text, cancellationToken: cancellationToken).ConfigureAwait(false);
+            }
+            catch (HttpOperationException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests)
+            {
+                tries++;
+
+                if (tries < 3)
+                {
+                    Console.WriteLine($"Failed to generate embedding. Error: {ex}");
+                    Console.WriteLine("Retrying embedding generation...");
+                    await Task.Delay(10_000, cancellationToken).ConfigureAwait(false);
+                }
+                else
+                {
+                    throw;
+                }
+            }
+        }
+    }
 }

dotnet/samples/Demos/VectorStoreRAG/IDataLoader.cs

@@ -11,7 +11,9 @@ internal interface IDataLoader
     /// Load the text from a PDF file into the data store.
     /// </summary>
     /// <param name="pdfPath">The pdf file to load.</param>
+    /// <param name="batchSize">Maximum number of parallel threads to generate embeddings and upload records.</param>
+    /// <param name="betweenBatchDelayInMs">The number of milliseconds to delay between batches to avoid throttling.</param>
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests.</param>
     /// <returns>An async task that completes when the loading is complete.</returns>
-    Task LoadPdf(string pdfPath, CancellationToken cancellationToken);
+    Task LoadPdf(string pdfPath, int batchSize, int betweenBatchDelayInMs, CancellationToken cancellationToken);
 }

dotnet/samples/Demos/VectorStoreRAG/Options/RagConfig.cs

@@ -22,4 +22,10 @@ internal sealed class RagConfig
 
     [Required]
     public string CollectionName { get; set; } = string.Empty;
+
+    [Required]
+    public int DataLoadingBatchSize { get; set; } = 2;
+
+    [Required]
+    public int DataLoadingBetweenBatchDelayInMilliseconds { get; set; } = 0;
 }

dotnet/samples/Demos/VectorStoreRAG/Program.cs

@@ -13,13 +13,17 @@
 
 HostApplicationBuilder builder = Host.CreateApplicationBuilder(args);
 
-builder.Configuration
-    .AddUserSecrets<Program>();
-
+// Configure configuration and load the application configuration.
+builder.Configuration.AddUserSecrets<Program>();
 builder.Services.Configure<RagConfig>(builder.Configuration.GetSection(RagConfig.ConfigSectionName));
-
 var appConfig = new ApplicationConfig(builder.Configuration);
 
+// Create a cancellation token and source to pass to the application service to allow them
+// to request a graceful application shutdown.
+CancellationTokenSource appShutdownCancellationTokenSource = new();
+CancellationToken appShutdownCancellationToken = appShutdownCancellationTokenSource.Token;
+builder.Services.AddKeyedSingleton("AppShutdown", appShutdownCancellationTokenSource);
+
 // Register the kernel with the dependency injection container
 // and add Chat Completion and Text Embedding Generation services.
 var kernelBuilder = builder.Services.AddKernel()
@@ -54,6 +58,10 @@
             appConfig.AzureCosmosDBNoSQLConfig.ConnectionString,
             appConfig.AzureCosmosDBNoSQLConfig.DatabaseName);
         break;
+    case "InMemory":
+        kernelBuilder.AddInMemoryVectorStoreRecordCollection<string, TextSnippet<string>>(
+            appConfig.RagConfig.CollectionName);
+        break;
     case "Qdrant":
         kernelBuilder.AddQdrantVectorStoreRecordCollection<Guid, TextSnippet<Guid>>(
             appConfig.RagConfig.CollectionName,
@@ -84,6 +92,7 @@
     case "AzureAISearch":
     case "AzureCosmosDBMongoDB":
     case "AzureCosmosDBNoSQL":
+    case "InMemory":
     case "Redis":
         RegisterServices<string>(builder, kernelBuilder, appConfig);
         break;
@@ -97,7 +106,7 @@
 
 // Build and run the host.
 using IHost host = builder.Build();
-await host.RunAsync().ConfigureAwait(false);
+await host.RunAsync(appShutdownCancellationToken).ConfigureAwait(false);
 
 static void RegisterServices<TKey>(HostApplicationBuilder builder, IKernelBuilder kernelBuilder, ApplicationConfig vectorStoreRagConfig)
     where TKey : notnull

dotnet/samples/Demos/VectorStoreRAG/RAGChatService.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
 using Microsoft.SemanticKernel;
@@ -17,11 +18,13 @@ namespace VectorStoreRAG;
 /// <param name="vectorStoreTextSearch">Used to search the vector store.</param>
 /// <param name="kernel">Used to make requests to the LLM.</param>
 /// <param name="ragConfigOptions">The configuration options for the application.</param>
+/// <param name="appShutdownCancellationTokenSource">Used to gracefully shut down the entire application when cancelled.</param>
 internal sealed class RAGChatService<TKey>(
     IDataLoader dataLoader,
     VectorStoreTextSearch<TextSnippet<TKey>> vectorStoreTextSearch,
     Kernel kernel,
-    IOptions<RagConfig> ragConfigOptions) : IHostedService
+    IOptions<RagConfig> ragConfigOptions,
+    [FromKeyedServices("AppShutdown")] CancellationTokenSource appShutdownCancellationTokenSource) : IHostedService
 {
     private Task? _dataLoaded;
     private Task? _chatLoop;
@@ -83,6 +86,9 @@ private async Task ChatLoopAsync(CancellationToken cancellationToken)
 
         Console.WriteLine("PDF loading complete\n");
 
+        Console.ForegroundColor = ConsoleColor.Green;
+        Console.WriteLine("Assistant > Press enter with no prompt to exit.");
+
         // Add a search plugin to the kernel which we will use in the template below
         // to do a vector search for related information to the user query.
         kernel.Plugins.Add(vectorStoreTextSearch.CreateWithGetTextSearchResults("SearchPlugin"));
@@ -99,6 +105,13 @@ private async Task ChatLoopAsync(CancellationToken cancellationToken)
             Console.Write("User > ");
             var question = Console.ReadLine();
 
+            // Exit the application if the user didn't type anything.
+            if (string.IsNullOrWhiteSpace(question))
+            {
+                appShutdownCancellationTokenSource.Cancel();
+                break;
+            }
+
             // Invoke the LLM with a template that uses the search plugin to
             // 1. get related information to the user query from the vector store
             // 2. add the information to the LLM prompt.
@@ -158,7 +171,11 @@ private async Task LoadDataAsync(CancellationToken cancellationToken)
             foreach (var pdfFilePath in ragConfigOptions.Value.PdfFilePaths ?? [])
             {
                 Console.WriteLine($"Loading PDF into vector store: {pdfFilePath}");
-                await dataLoader.LoadPdf(pdfFilePath, cancellationToken).ConfigureAwait(false);
+                await dataLoader.LoadPdf(
+                    pdfFilePath,
+                    ragConfigOptions.Value.DataLoadingBatchSize,
+                    ragConfigOptions.Value.DataLoadingBetweenBatchDelayInMilliseconds,
+                    cancellationToken).ConfigureAwait(false);
             }
         }
         catch (Exception ex)

dotnet/samples/Demos/VectorStoreRAG/README.md

@@ -11,12 +11,15 @@ The sample can be configured in various ways:
    1. AzureAISearch
    1. AzureCosmosDBMongoDB
    1. AzureCosmosDBNoSQL
+   1. InMemory
    1. Qdrant
    1. Redis
    1. Weaviate
 1. You can choose whether to load data into the vector store by setting the `Rag:BuildCollection` configuration setting in the `appsettings.json` file to `true`. If you set this to `false`, the sample will assume that data was already loaded previously and it will go straight into the chat experience.
 1. You can choose the name of the collection to use by setting the `Rag:CollectionName` configuration setting in the `appsettings.json` file.
 1. You can choose the pdf file to load into the vector store by setting the `Rag:PdfFilePaths` array in the `appsettings.json` file.
+1. You can choose the number of records to process per batch when loading data into the vector store by setting the `Rag:DataLoadingBatchSize` configuration setting in the `appsettings.json` file.
+1. You can choose the number of milliseconds to wait between batches when loading data into the vector store by setting the `Rag:DataLoadingBetweenBatchDelayInMilliseconds` configuration setting in the `appsettings.json` file.
 
 ## Dependency Setup
 
@@ -45,6 +48,7 @@ For Azure OpenAI, you need to add the following secrets:
 
 ```cli
 dotnet user-secrets set "AIServices:AzureOpenAI:Endpoint" "https://<yourservice>.openai.azure.com"
+dotnet user-secrets set "AIServices:AzureOpenAI:ChatDeploymentName" "<your deployment name>"
 ```
 
 Note that the code doesn't use an API Key to communicate with Azure Open AI, but rather an `AzureCliCredential` so no api key secret is required.
@@ -55,6 +59,7 @@ For Azure OpenAI Embeddings, you need to add the following secrets:
 
 ```cli
 dotnet user-secrets set "AIServices:AzureOpenAIEmbeddings:Endpoint" "https://<yourservice>.openai.azure.com"
+dotnet user-secrets set "AIServices:AzureOpenAIEmbeddings:DeploymentName" "<your deployment name>"
 ```
 
 Note that the code doesn't use an API Key to communicate with Azure Open AI, but rather an `AzureCliCredential` so no api key secret is required.

dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj

@@ -21,6 +21,7 @@
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.AzureAISearch\Connectors.Memory.AzureAISearch.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.AzureCosmosDBMongoDB\Connectors.Memory.AzureCosmosDBMongoDB.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.AzureCosmosDBNoSQL\Connectors.Memory.AzureCosmosDBNoSQL.csproj" />
+    <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.InMemory\Connectors.Memory.InMemory.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.Qdrant\Connectors.Memory.Qdrant.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj" />
     <ProjectReference Include="..\..\..\src\Connectors\Connectors.Memory.Weaviate\Connectors.Memory.Weaviate.csproj" />

dotnet/samples/Demos/VectorStoreRAG/appsettings.json

@@ -43,7 +43,9 @@
   "Rag": {
     "BuildCollection": true,
     "PdfFilePaths": [ "sourcedocument.pdf" ],
-    "VectorStoreType": "Qdrant",
-    "CollectionName": "pdfcontent"
+    "VectorStoreType": "InMemory",
+    "CollectionName": "pdfcontent",
+    "DataLoadingBatchSize": 2,
+    "DataLoadingBetweenBatchDelayInMilliseconds": 1000
   }
 }

dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBHotelModel.cs

@@ -39,6 +39,6 @@ public class AzureCosmosDBMongoDBHotelModel(string hotelId)
     public string? Description { get; set; }
 
     /// <summary>A vector field.</summary>
-    [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)]
+    [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)]
     public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
 }

dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs

@@ -828,11 +828,11 @@ private sealed class VectorSearchModel
         [VectorStoreRecordData]
         public string? HotelName { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance, StoragePropertyName = "test_embedding_1")]
+        [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat, StoragePropertyName = "test_embedding_1")]
         public ReadOnlyMemory<float> TestEmbedding1 { get; set; }
 
         [BsonElement("test_embedding_2")]
-        [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.IvfFlat, DistanceFunction: DistanceFunction.CosineDistance)]
+        [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.IvfFlat)]
         public ReadOnlyMemory<float> TestEmbedding2 { get; set; }
     }
 #pragma warning restore CA1812

dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLHotel.cs

@@ -39,6 +39,6 @@ public class AzureCosmosDBNoSQLHotel(string hotelId)
 
     /// <summary>A vector field.</summary>
     [JsonPropertyName("description_embedding")]
-    [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)]
+    [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)]
     public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
 }

dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLVectorStoreRecordCollectionTests.cs

@@ -701,16 +701,16 @@ private sealed class TestIndexingModel
         [VectorStoreRecordKey]
         public string? Id { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 1, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)]
+        [VectorStoreRecordVector(Dimensions: 1, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)]
         public ReadOnlyMemory<Half>? DescriptionEmbedding1 { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 2, IndexKind: IndexKind.Flat, DistanceFunction: DistanceFunction.CosineSimilarity)]
+        [VectorStoreRecordVector(Dimensions: 2, DistanceFunction: DistanceFunction.CosineSimilarity, IndexKind: IndexKind.Flat)]
         public ReadOnlyMemory<float>? DescriptionEmbedding2 { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 3, IndexKind: IndexKind.QuantizedFlat, DistanceFunction: DistanceFunction.DotProductSimilarity)]
+        [VectorStoreRecordVector(Dimensions: 3, DistanceFunction: DistanceFunction.DotProductSimilarity, IndexKind: IndexKind.QuantizedFlat)]
         public ReadOnlyMemory<byte>? DescriptionEmbedding3 { get; set; }
 
-        [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.DiskAnn, DistanceFunction: DistanceFunction.EuclideanDistance)]
+        [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.EuclideanDistance, IndexKind: IndexKind.DiskAnn)]
         public ReadOnlyMemory<sbyte>? DescriptionEmbedding4 { get; set; }
 
         [VectorStoreRecordData(IsFilterable = true)]

dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateHotel.cs

@@ -44,6 +44,6 @@ public sealed record WeaviateHotel
     public DateTimeOffset Timestamp { get; set; }
 
     /// <summary>A vector field.</summary>
-    [VectorStoreRecordVector(Dimensions: 4, IndexKind: IndexKind.Hnsw, DistanceFunction: DistanceFunction.CosineDistance)]
+    [VectorStoreRecordVector(Dimensions: 4, DistanceFunction: DistanceFunction.CosineDistance, IndexKind: IndexKind.Hnsw)]
     public ReadOnlyMemory<float>? DescriptionEmbedding { get; set; }
 }

dotnet/src/Connectors/VectorData.Abstractions/RecordAttributes/VectorStoreRecordVectorAttribute.cs

@@ -33,13 +33,24 @@ public VectorStoreRecordVectorAttribute(int Dimensions)
     /// Initializes a new instance of the <see cref="VectorStoreRecordVectorAttribute"/> class.
     /// </summary>
     /// <param name="Dimensions">The number of dimensions that the vector has.</param>
-    /// <param name="IndexKind">The kind of index to use.</param>
     /// <param name="DistanceFunction">The distance function to use when comparing vectors.</param>
-    public VectorStoreRecordVectorAttribute(int Dimensions, string? IndexKind, string? DistanceFunction)
+    public VectorStoreRecordVectorAttribute(int Dimensions, string? DistanceFunction)
+    {
+        this.Dimensions = Dimensions;
+        this.DistanceFunction = DistanceFunction;
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="VectorStoreRecordVectorAttribute"/> class.
+    /// </summary>
+    /// <param name="Dimensions">The number of dimensions that the vector has.</param>
+    /// <param name="DistanceFunction">The distance function to use when comparing vectors.</param>
+    /// <param name="IndexKind">The kind of index to use.</param>
+    public VectorStoreRecordVectorAttribute(int Dimensions, string? DistanceFunction, string? IndexKind)
     {
         this.Dimensions = Dimensions;
-        this.IndexKind = IndexKind;
         this.DistanceFunction = DistanceFunction;
+        this.IndexKind = IndexKind;
     }
 
     /// <summary>