Merge pull request #125 from Nexus-Mods/really-fix-unique-ids

Really fix unique ids

src/NexusMods.MnemonicDB.Abstractions/Attribute.cs

@@ -2,6 +2,7 @@
 using System.Buffers;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
 using NexusMods.MnemonicDB.Abstractions.DatomIterators;
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using NexusMods.MnemonicDB.Abstractions.Internals;
@@ -60,6 +61,22 @@ protected Attribute(
 
     /// <inheritdoc />
     public bool IsUnique { get; init; }
+
+    /// <summary>
+    /// Returns the indexed flags for this attribute
+    /// </summary>
+    public IndexedFlags IndexedFlags
+    {
+        get
+        {
+            if (IsUnique)
+                return IndexedFlags.Unique;
+            if (IsIndexed)
+                return IndexedFlags.Indexed;
+            return IndexedFlags.None;
+        }
+
+    }
 
     /// <inheritdoc />
     public bool NoHistory { get; init; }
@@ -106,7 +123,7 @@ private void AssertTag(ValueTag tag)
     /// <summary>
     /// Reads the high level value from the given span
     /// </summary>
-    public TValueType ReadValue(ReadOnlySpan<byte> span, ValueTag tag, AttributeResolver resolver)
+    public virtual TValueType ReadValue(ReadOnlySpan<byte> span, ValueTag tag, AttributeResolver resolver)
     {
         AssertTag(tag);
         return FromLowLevel(TSerializer.Read(span), resolver);

src/NexusMods.MnemonicDB.Abstractions/AttributeCache.cs

@@ -17,8 +17,7 @@ public sealed class AttributeCache
     private Dictionary<Symbol, AttributeId> _attributeIdsBySymbol = new();
     private BitArray _isCardinalityMany;
     private BitArray _isReference;
-    private BitArray _isIndexed;
-    private BitArray _isUnique;
+    private IndexedFlags[] _indexedFlags;
     private Symbol[] _symbols;
     private ValueTag[] _valueTags;
     private BitArray _isNoHistory;
@@ -31,17 +30,15 @@ public AttributeCache()
         var maxId = AttributeDefinition.HardcodedIds.Values.Max() + 1;
         _isCardinalityMany = new BitArray(maxId);
         _isReference = new BitArray(maxId);
-        _isIndexed = new BitArray(maxId);
-        _isUnique = new BitArray(maxId);
+        _indexedFlags = new IndexedFlags[maxId];
         _isNoHistory = new BitArray(maxId);
         _symbols = new Symbol[maxId];
         _valueTags = new ValueTag[maxId];
 
         foreach (var kv in AttributeDefinition.HardcodedIds)
         {
             _attributeIdsBySymbol[kv.Key.Id] = AttributeId.From(kv.Value);
-            _isIndexed[kv.Value] = kv.Key.IsIndexed;
-            _isUnique[kv.Value] = kv.Key.IsUnique;
+            _indexedFlags[kv.Value] = kv.Key.IndexedFlags;
             _symbols[kv.Value] = kv.Key.Id;
             _valueTags[kv.Value] = kv.Key.LowLevelType;
         }
@@ -84,22 +81,22 @@ public void Reset(IDb db)
         _isReference = newIsReference;
 
         var isIndexed = db.Datoms(AttributeDefinition.Indexed);
-        var newIsIndexed = new BitArray(maxIndex);
+        var newIsIndexed = new IndexedFlags[maxIndex];
         foreach (var datom in isIndexed)
         {
+            // Older DBs use a null to mark indexed attributes
+            var flags = datom.Prefix.ValueTag switch
+            {
+                ValueTag.Null => IndexedFlags.Indexed,
+                ValueTag.UInt8 => (IndexedFlags)datom.ValueSpan[0],
+                _ => IndexedFlags.None
+            };
+
             var id = datom.E.Value;
-            newIsIndexed[(int)id] = true;
+            newIsIndexed[(int)id] = flags;
         }
-        _isIndexed = newIsIndexed;
+        _indexedFlags = newIsIndexed;
 
-        var isUnique = db.Datoms(AttributeDefinition.Unique);
-        var newIsUnique = new BitArray(maxIndex);
-        foreach (var datom in isUnique)
-        {
-            var id = datom.E.Value;
-            newIsUnique[(int)id] = true;
-        }
-        _isUnique = newIsUnique;
 
         var isNoHistory = db.Datoms(AttributeDefinition.NoHistory);
         var newIsNoHistory = new BitArray(maxIndex);
@@ -146,7 +143,7 @@ public bool IsReference(AttributeId attrId)
     /// </summary>
     public bool IsIndexed(AttributeId attrId)
     {
-        return _isIndexed[attrId.Value];
+        return _indexedFlags[attrId.Value].HasFlag(IndexedFlags.Indexed);
     }
 
     /// <summary>
@@ -211,6 +208,14 @@ public ValueTag GetValueTag(AttributeId aid)
     /// </summary>
     public bool IsUnique(AttributeId attrId)
     {
-        return _isUnique[attrId.Value];
+        return _indexedFlags[attrId.Value].HasFlag(IndexedFlags.Unique);
+    }
+
+    /// <summary>
+    /// Get the indexed flags for the given attribute id
+    /// </summary>
+    public IndexedFlags GetIndexedFlags(AttributeId aid)
+    {
+        return _indexedFlags[aid.Value];
     }
 }

src/NexusMods.MnemonicDB.Abstractions/Attributes/IndexedFlagsAttribute.cs

@@ -0,0 +1,46 @@
+using System;
+using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
+using NexusMods.MnemonicDB.Abstractions.ElementComparers;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.MnemonicDB.Abstractions.Attributes;
+
+/// <summary>
+/// An attribute that defines the indexing status of an attribute, this is stored as a custom attribute type because
+/// older DBs store data in a different format and a conversion on read is required.
+/// </summary>
+public class IndexedFlagsAttribute : ScalarAttribute<IndexedFlags, byte, UInt8Serializer>
+{
+    /// <inheritdoc />
+    public IndexedFlagsAttribute(string ns, string name) : 
+        base(ns, name)
+    {
+        DefaultValue = IndexedFlags.None;
+    }
+
+    /// <summary>
+    /// This performs the proper conversion from the low level representation to the high level representation. Taking
+    /// into consideration old attribute formats
+    /// </summary>
+    public override IndexedFlags ReadValue(ReadOnlySpan<byte> span, ValueTag tag, AttributeResolver resolver)
+    {
+        // Old format, an attribute's existance means it's indexed
+        if (tag == ValueTag.Null)
+            return IndexedFlags.Indexed;
+        if (tag == ValueTag.UInt8)
+            return (IndexedFlags)span[0];
+        return IndexedFlags.None;
+    }
+
+    /// <inheritdoc />
+    protected override byte ToLowLevel(IndexedFlags value)
+    {
+        return (byte)value;
+    }
+
+    /// <inheritdoc />
+    protected override IndexedFlags FromLowLevel(byte value, AttributeResolver resolver)
+    {
+        throw new NotSupportedException("This method should never be called.");
+    }
+}

src/NexusMods.MnemonicDB.Abstractions/Attributes/ScalarAttribute.cs

@@ -66,7 +66,10 @@ public bool TryGetValue<T>(T entity, [NotNullWhen(true)] out TValue? value)
     public TValue Get<T>(T entity, IndexSegment segment)
         where T : IHasEntityIdAndDb
     {
-        if (TryGetValue(entity, segment, out var value)) return value;
+        if (TryGetValue(entity, segment, out var value)) 
+            return value;
+        if (DefaultValue.HasValue) 
+            return DefaultValue.Value;
         return ThrowKeyNotfoundException(entity.Id);
     }
 

src/NexusMods.MnemonicDB.Abstractions/BuiltInEntities/AttributeDefinition.cs

@@ -29,13 +29,8 @@ public partial class AttributeDefinition : IModelDefinition
     /// <summary>
     /// True if the attribute is indexed.
     /// </summary>
-    public static readonly MarkerAttribute Indexed = new(Namespace, nameof(Indexed));
+    public static readonly IndexedFlagsAttribute Indexed = new(Namespace, nameof(Indexed));
 
-    /// <summary>
-    /// True if the attribute is unique, that this attr/value pair can only exist on one entity at a time
-    /// </summary>
-    public static readonly MarkerAttribute Unique = new(Namespace, nameof(Unique));
-
     /// <summary>
     /// This attribute is optional.
     /// </summary>
@@ -63,22 +58,22 @@ public static void Insert(ITransaction tx, IAttribute attribute, ushort id = 0)
     {
         if (id == 0)
             id = HardcodedIds[attribute];
-        var eid = EntityId.From(id);
+        var eid = EntityId.From((ushort)id);
         tx.Add(eid, UniqueId, attribute.Id);
         tx.Add(eid, ValueType, attribute.LowLevelType);
         tx.Add(eid, Cardinality, attribute.Cardinalty);
         if (attribute.IsIndexed)
-            tx.Add(eid, Indexed, Null.Instance);
-        if (attribute.IsUnique)
-            tx.Add(eid, Unique, Null.Instance);
+            tx.Add(eid, Indexed, IndexedFlags.None);
         if (attribute.NoHistory)
             tx.Add(eid, NoHistory, Null.Instance);
         if (attribute.DeclaredOptional)
             tx.Add(eid, Optional, Null.Instance);
     }
 
     /// <summary>
-    /// Hardcoded ids for the initial attributes
+    /// Hardcoded ids for the initial attributes. Negative numbers are assigned the next available id and may differ
+    /// from database to database. These IDs can't be hardcoded because they were added after some databases were created,
+    /// and so we can't be certain that we have a specific ID for them.
     /// </summary>
     public static readonly Dictionary<IAttribute, ushort> HardcodedIds = new()
     {
@@ -90,7 +85,6 @@ public static void Insert(ITransaction tx, IAttribute attribute, ushort id = 0)
         { Cardinality, 6 },
         { Documentation, 7 },
         { Transaction.Timestamp, 8},
-        { Unique, 9 }
     };
 
     /// <summary>
@@ -102,7 +96,6 @@ public static void AddInitial(ITransaction tx)
         Insert(tx, ValueType);
         Insert(tx, Documentation);
         Insert(tx, Indexed);
-        Insert(tx, Unique);
         Insert(tx, Optional);
         Insert(tx, NoHistory);
         Insert(tx, Cardinality);

src/NexusMods.MnemonicDB.Abstractions/BuiltInEntities/IndexedFlags.cs

@@ -0,0 +1,24 @@
+using System;
+
+namespace NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
+
+/// <summary>
+/// The indexed flags for an attribute
+/// </summary>
+[Flags]
+public enum IndexedFlags : byte
+{
+    /// <summary>
+    /// Attribute is not indexed
+    /// </summary>
+    None = 0b0000,
+    /// <summary>
+    /// The attribute values are indexed
+    /// </summary>
+    Indexed = 0b0001,
+
+    /// <summary>
+    /// The attribute values have a unique index
+    /// </summary>
+    Unique = 0b0011,
+}

src/NexusMods.MnemonicDB.Abstractions/IAttribute.cs

@@ -2,6 +2,7 @@
 using System.Buffers;
 using System.Numerics;
 using System.Runtime.CompilerServices;
+using NexusMods.MnemonicDB.Abstractions.BuiltInEntities;
 using NexusMods.MnemonicDB.Abstractions.ElementComparers;
 using NexusMods.MnemonicDB.Abstractions.Internals;
 using NexusMods.MnemonicDB.Abstractions.Models;
@@ -45,6 +46,11 @@ public interface IAttribute
     /// be thought of as making sure that .Datoms(Attribute, Value) for this attribute never returns more than one datom.
     /// </summary>
     bool IsUnique { get; }
+
+    /// <summary>
+    /// The indexed flags for the attribute
+    /// </summary>
+    IndexedFlags IndexedFlags { get; }
 
     /// <summary>
     ///   True if the attribute has no history, false if it does.

src/NexusMods.MnemonicDB.Abstractions/ITransaction.cs

@@ -20,7 +20,7 @@ public interface ITransaction : IDisposable
     ///     Gets the temporary id for the transaction
     /// </summary>
     public TxId ThisTxId { get; }
-
+    
     /// <summary>
     ///     Gets a temporary id for a new entity in the given partition
     /// </summary>