Switch to upgrading the index flag data type to support unique attrib…

…utes. Implement migration for old values, and add an example DB from the app that is in the old format, so that we can verify that upgrading to the new format works as expected.

src/NexusMods.MnemonicDB.Abstractions/Attribute.cs

@@ -123,7 +123,7 @@ private void AssertTag(ValueTag tag)
     /// <summary>
     /// Reads the high level value from the given span
     /// </summary>
-    public TValueType ReadValue(ReadOnlySpan<byte> span, ValueTag tag, AttributeResolver resolver)
+    public virtual TValueType ReadValue(ReadOnlySpan<byte> span, ValueTag tag, AttributeResolver resolver)
     {
         AssertTag(tag);
         return FromLowLevel(TSerializer.Read(span), resolver);

src/NexusMods.MnemonicDB.Abstractions/Attributes/IndexedFlagsAttribute.cs

@@ -0,0 +1,46 @@
+using System;
+using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.MnemonicDB.Abstractions.Attributes;
+
+/// <summary>
+/// An attribute that defines the indexing status of an attribute, this is stored as a custom attribute type because
+/// older DBs store data in a different format and a conversion on read is required.
+/// </summary>
+public class IndexedFlagsAttribute : ScalarAttribute<IndexedFlags, byte, UInt8Serializer>
+{
+    /// <inheritdoc />
+    public IndexedFlagsAttribute(string ns, string name) : 
+        base(ns, name)
+    {
+        DefaultValue = IndexedFlags.None;
+    }
+
+    /// <summary>
+    /// This performs the proper conversion from the low level representation to the high level representation. Taking
+    /// into consideration old attribute formats
+    /// </summary>
+    public override IndexedFlags ReadValue(ReadOnlySpan<byte> span, ValueTag tag, AttributeResolver resolver)
+    {
+        // Old format, an attribute's existance means it's indexed
+        if (tag == ValueTag.Null)
+            return IndexedFlags.Indexed;
+        if (tag == ValueTag.UInt8)
+            return (IndexedFlags)span[0];
+        return IndexedFlags.None;
+    }
+
+    /// <inheritdoc />
+    protected override byte ToLowLevel(IndexedFlags value)
+    {
+        return (byte)value;
+    }
+
+    /// <inheritdoc />
+    protected override IndexedFlags FromLowLevel(byte value, AttributeResolver resolver)
+    {
+        throw new NotSupportedException("This method should never be called.");
+    }
+}

src/NexusMods.MnemonicDB.Abstractions/Attributes/ScalarAttribute.cs

@@ -66,7 +66,10 @@ public bool TryGetValue<T>(T entity, [NotNullWhen(true)] out TValue? value)
     public TValue Get<T>(T entity, IndexSegment segment)
         where T : IHasEntityIdAndDb
     {
-        if (TryGetValue(entity, segment, out var value)) return value;
+        if (TryGetValue(entity, segment, out var value)) 
+            return value;
+        if (DefaultValue.HasValue) 
+            return DefaultValue.Value;
         return ThrowKeyNotfoundException(entity.Id);
     }
 

src/NexusMods.MnemonicDB.Abstractions/BuiltInEntities/AttributeDefinition.cs

@@ -29,7 +29,7 @@ public partial class AttributeDefinition : IModelDefinition
     /// <summary>
     /// True if the attribute is indexed.
     /// </summary>
-    public static readonly EnumByteAttribute<IndexedFlags> Indexed = new(Namespace, nameof(Indexed)) { DefaultValue = IndexedFlags.None };
+    public static readonly IndexedFlagsAttribute Indexed = new(Namespace, nameof(Indexed));
 
     /// <summary>
     /// This attribute is optional.

src/NexusMods.MnemonicDB/InternalTxFunctions/SimpleMigration.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Threading;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
@@ -15,6 +16,7 @@ internal class SimpleMigration : AInternalFn
     private readonly IAttribute[] _declaredAttributes;
     private ulong _tempId = PartitionId.Temp.MakeEntityId(1).Value;
 
+    private static string[] InternalNamespaces = ["NexusMods.MnemonicDB.DatomStore", "NexusMods.MnemonicDB.Transactions"]; 
     public SimpleMigration(IAttribute[] attributes)
     {
         _declaredAttributes = attributes;
@@ -38,6 +40,10 @@ public override void Execute(DatomStore store)
         var madeChanges = false;
         foreach (var attribute in _declaredAttributes)
         {
+            // Internal transactions are migrated elsewhere
+            if (InternalNamespaces.Contains(attribute.Id.Namespace))
+                continue;
+
             if (!cache.TryGetAttributeId(attribute.Id, out var aid))
             {
                 madeChanges = true;

tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs

@@ -25,11 +25,14 @@ public class AMnemonicDBTest : IDisposable
     protected IConnection Connection;
     protected ILogger Logger;
     private readonly IAnalyzer[] _analyzers;
+
+    protected TemporaryFileManager TemporaryFileManager;
 
 
     protected AMnemonicDBTest(IServiceProvider provider)
     {
         Provider = provider;
+        TemporaryFileManager = provider.GetRequiredService<TemporaryFileManager>();
         _attributes = provider.GetRequiredService<IEnumerable<IAttribute>>().ToArray();
 
         Config = new DatomStoreSettings
@@ -52,6 +55,7 @@ protected AMnemonicDBTest(IServiceProvider provider)
         Logger = provider.GetRequiredService<ILogger<AMnemonicDBTest>>();
     }
 
+
     protected async Task LoadDatamodel(RelativePath name)
     {
         var fullPath = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory).Combine("Resources").Combine(name);

tests/NexusMods.MnemonicDB.Tests/MigrationTests.cs

@@ -1,5 +1,14 @@
+using System.IO.Compression;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using NexusMods.Hashing.xxHash3;
+using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
+using NexusMods.MnemonicDB.Abstractions.DatomIterators;
+using NexusMods.MnemonicDB.Abstractions.Query;
+using NexusMods.MnemonicDB.Storage;
+using NexusMods.MnemonicDB.Storage.RocksDbBackend;
 using NexusMods.MnemonicDB.TestModel;
 using NexusMods.MnemonicDB.TestModel.Attributes;
 using NexusMods.Paths;
@@ -91,4 +100,37 @@ public async Task CanRemoveIndex()
         Action act = () => Connection.Db.Datoms(Mod.Source, new Uri("http://mod0.com")).ToArray();
         act.Should().Throw<InvalidOperationException>();
     }
+
+    [Theory]
+    [InlineData("SDV.2_5_2025.rocksdb.zip")]
+    public async Task CanOpenOlderDBs(string fileName)
+    {
+        var path = FileSystem.Shared.GetKnownPath(KnownPath.EntryDirectory) / "Resources/Databases" / fileName;
+
+        await using var extractedFolder = TemporaryFileManager.CreateFolder();
+
+        ZipFile.ExtractToDirectory(path.ToString(), extractedFolder.Path.ToString());
+
+        var settings = new DatomStoreSettings()
+        {
+            Path = extractedFolder.Path / "MnemonicDB.rocksdb",
+        };
+        using var backend = new Backend();
+        using var store = new DatomStore(Provider.GetRequiredService<ILogger<DatomStore>>(), settings, backend);
+        var connection = new Connection(Provider.GetRequiredService<ILogger<Connection>>(), store, Provider, [], false);
+
+        var db = connection.Db;
+        var attrs = db.Datoms(AttributeDefinition.UniqueId);
+        var datoms = new List<Datom>();
+        foreach (var attr in attrs)
+            datoms.AddRange(db.Datoms(attr.E));
+
+        var cache = connection.AttributeCache;
+        foreach (var attr in AttributeDefinition.All(db))
+        {
+            attr.Indexed.Should().Be(cache.GetIndexedFlags(cache.GetAttributeId(attr.UniqueId)), "The indexed flags are backwards compatible");
+        }
+
+        return;
+    }
 }

tests/NexusMods.MnemonicDB.Tests/NexusMods.MnemonicDB.Tests.csproj

@@ -18,6 +18,9 @@
         <DependentUpon>DbTests.cs</DependentUpon>
       </None>
       <None Remove="DbTests.CanGetCommitUpdates_update_datom_0.received.txt" />
+      <None Update="Resources\Databases\SDV.2_5_2025.rocksdb.zip">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
     </ItemGroup>
 
     <ItemGroup>
@@ -38,4 +41,8 @@
       <PackageReference Update="GitHubActionsTestLogger"/>
     </ItemGroup>
 
+    <ItemGroup>
+      <Folder Include="Resources\Databases\" />
+    </ItemGroup>
+
 </Project>

tests/NexusMods.MnemonicDB.Tests/Startup.cs

@@ -12,6 +12,7 @@ public class Startup
     public void ConfigureServices(IServiceCollection services)
     {
         services.AddTestModel()
+            .AddSingleton<TemporaryFileManager>()
             .AddFileSystem()
             .AddLogging(builder => builder.AddXunitOutput().SetMinimumLevel(LogLevel.Debug))
             .AddMnemonicDBStorage();