Issue #63. Fix cross reference logic

lib/schema.ex

@@ -114,7 +114,7 @@ defmodule Schema do
 
   @spec data_type?(binary(), binary() | list(binary())) :: boolean()
   def data_type?(type, type), do: true
-  
+
   def data_type?(type, base_type) when is_binary(base_type) do
     types = Map.get(Repo.data_types(), :attributes)
 
@@ -153,6 +153,9 @@ defmodule Schema do
     |> apply_profiles(profiles, MapSet.size(profiles))
   end
 
+  @spec all_classes() :: map()
+  def all_classes(), do: Repo.all_classes()
+
   @doc """
     Returns a single event class.
   """
@@ -430,7 +433,7 @@ defmodule Schema do
   # ----------------------------#
 
   def enrich(data, enum_text, observables) do
-  	Schema.Helper.enrich(data, enum_text, observables)
+    Schema.Helper.enrich(data, enum_text, observables)
   end
 
   # -------------------------------#

lib/schema/cache.ex

@@ -19,27 +19,29 @@ defmodule Schema.Cache do
 
   require Logger
 
-  @enforce_keys [:version, :profiles, :dictionary, :categories, :base_event, :classes, :objects]
-  defstruct ~w[version profiles dictionary base_event categories classes objects]a
-
-  @spec new(map()) :: __MODULE__.t()
-  def new(version) do
-    %__MODULE__{
-      version: version,
-      profiles: Map.new(),
-      dictionary: Map.new(),
-      categories: Map.new(),
-      base_event: Map.new(),
-      classes: Map.new(),
-      objects: Map.new()
-    }
-  end
+  @enforce_keys [
+    :version,
+    :profiles,
+    :categories,
+    :dictionary,
+    :base_event,
+    :classes,
+    :all_classes,
+    :objects
+  ]
+  defstruct ~w[version profiles dictionary base_event categories classes all_classes objects]a
 
   @type t() :: %__MODULE__{}
   @type class_t() :: map()
   @type object_t() :: map()
   @type category_t() :: map()
   @type dictionary_t() :: map()
+  @type link_t() :: %{
+          group: :common | :class | :object,
+          type: String.t(),
+          caption: String.t(),
+          attribute_keys: nil | MapSet.t(String.t())
+        }
 
   @ocsf_deprecated :"@deprecated"
 
@@ -53,7 +55,7 @@ defmodule Schema.Cache do
     categories = JsonReader.read_categories() |> update_categories()
     dictionary = JsonReader.read_dictionary() |> update_dictionary()
 
-    {base_event, classes} = read_classes(categories[:attributes])
+    {base_event, classes, all_classes} = read_classes(categories[:attributes])
     objects = read_objects()
 
     dictionary = Utils.update_dictionary(dictionary, base_event, classes, objects)
@@ -97,13 +99,16 @@ defmodule Schema.Cache do
       )
     end
 
-    new(version)
-    |> set_profiles(profiles)
-    |> set_categories(categories)
-    |> set_dictionary(dictionary)
-    |> set_base_event(base_event)
-    |> set_classes(classes)
-    |> set_objects(objects)
+    %__MODULE__{
+      version: version,
+      profiles: profiles,
+      categories: categories,
+      dictionary: dictionary,
+      base_event: base_event,
+      classes: classes,
+      all_classes: all_classes,
+      objects: objects
+    }
   end
 
   @doc """
@@ -141,6 +146,9 @@ defmodule Schema.Cache do
   @spec classes(__MODULE__.t()) :: map()
   def classes(%__MODULE__{classes: classes}), do: classes
 
+  @spec all_classes(__MODULE__.t()) :: map()
+  def all_classes(%__MODULE__{all_classes: all_classes}), do: all_classes
+
   @spec export_classes(__MODULE__.t()) :: map()
   def export_classes(%__MODULE__{classes: classes, dictionary: dictionary}) do
     Enum.into(classes, Map.new(), fn {name, class} ->
@@ -321,13 +329,29 @@ defmodule Schema.Cache do
       |> Enum.into(%{}, fn class -> attribute_source(class) end)
       |> extend_type()
 
+    resolved = resolve_extends(classes)
+
     classes =
-      resolve_extends(classes)
+      resolved
       # remove intermediate classes
       |> Stream.filter(fn {key, class} -> Map.has_key?(class, :uid) or key == :base_event end)
       |> Enum.into(%{}, fn class -> enrich_class(class, categories) end)
 
-    {Map.get(classes, :base_event), classes}
+    # all_classes has just enough info to interrogate the complete class hierarchy,
+    # removing most details. It can be used to get the caption and parent (extends) of
+    # any class, including hidden ones (classes without a uid)
+    all_classes =
+      Enum.map(
+        resolved,
+        fn {class_name, class_info} ->
+          {class_name,
+           Map.take(class_info, [:name, :caption, :extends])
+           |> Map.put(:hidden?, class_name != :base_event && !Map.has_key?(class_info, :uid))}
+        end
+      )
+      |> Enum.into(%{})
+
+    {Map.get(classes, :base_event), classes, all_classes}
   end
 
   defp read_objects() do
@@ -666,30 +690,6 @@ defmodule Schema.Cache do
     end
   end
 
-  defp set_profiles(%__MODULE__{} = schema, profiles) do
-    struct(schema, profiles: profiles)
-  end
-
-  defp set_dictionary(%__MODULE__{} = schema, dictionary) do
-    struct(schema, dictionary: dictionary)
-  end
-
-  defp set_categories(%__MODULE__{} = schema, categories) do
-    struct(schema, categories: categories)
-  end
-
-  defp set_base_event(%__MODULE__{} = schema, base_event) do
-    struct(schema, base_event: base_event)
-  end
-
-  defp set_classes(%__MODULE__{} = schema, classes) do
-    struct(schema, classes: classes)
-  end
-
-  defp set_objects(%__MODULE__{} = schema, objects) do
-    struct(schema, objects: objects)
-  end
-
   defp update_observables(objects, dictionary) do
     if Map.has_key?(objects, :observable) do
       observable_types = get_in(dictionary, [:types, :attributes]) |> observables()
@@ -869,10 +869,10 @@ defmodule Schema.Cache do
     end
   end
 
-  defp update_linked_profiles(name, links, object, classes) do
-    Enum.reduce(links, classes, fn {type, key, _}, acc ->
-      if type == name do
-        Map.update!(acc, String.to_atom(key), fn class ->
+  defp update_linked_profiles(group, links, object, classes) do
+    Enum.reduce(links, classes, fn link, acc ->
+      if link[:group] == group do
+        Map.update!(acc, String.to_atom(link[:type]), fn class ->
           Map.put(class, :profiles, merge(class[:profiles], object[:profiles]))
         end)
       else

lib/schema/profiles.ex

@@ -48,21 +48,21 @@ defmodule Schema.Profiles do
   @doc """
     Checks classes or objects if all profile attributes are defined.
   """
-  def sanity_check(type, maps, profiles) do
+  def sanity_check(group, maps, profiles) do
     profiles =
       Enum.reduce(maps, profiles, fn {name, map}, acc ->
-        check_profiles(type, name, map, map[:profiles], acc)
+        check_profiles(group, name, map, map[:profiles], acc)
       end)
 
     {maps, profiles}
   end
 
   # Checks if all profile attributes are defined in the given attribute set.
-  defp check_profiles(_type, _name, _map, nil, all_profiles) do
+  defp check_profiles(_group, _name, _map, nil, all_profiles) do
     all_profiles
   end
 
-  defp check_profiles(type, name, map, profiles, all_profiles) do
+  defp check_profiles(group, name, map, profiles, all_profiles) do
     Enum.reduce(profiles, all_profiles, fn p, acc ->
       case acc[p] do
         nil ->
@@ -71,7 +71,7 @@ defmodule Schema.Profiles do
 
         profile ->
           check_profile(name, profile, map[:attributes])
-          link = {type, Atom.to_string(name), map[:caption]}
+          link = %{group: group, type: Atom.to_string(name), caption: map[:caption]}
           profile = Map.update(profile, :_links, [link], fn links -> [link | links] end)
           Map.put(acc, p, profile)
       end

lib/schema/repo.ex

@@ -119,6 +119,11 @@ defmodule Schema.Repo do
     Agent.get(__MODULE__, fn schema -> Cache.classes(schema) |> filter(extensions) end)
   end
 
+  @spec all_classes() :: map()
+  def all_classes() do
+    Agent.get(__MODULE__, fn schema -> Cache.all_classes(schema) end)
+  end
+
   @spec export_classes() :: map()
   def export_classes() do
     Agent.get(__MODULE__, fn schema -> Cache.export_classes(schema) end)
@@ -248,8 +253,8 @@ defmodule Schema.Repo do
   defp remove_extension_links(nil, _extensions), do: []
 
   defp remove_extension_links(links, extensions) do
-    Enum.filter(links, fn {_, key, _} ->
-      [ext | rest] = String.split(key, "/")
+    Enum.filter(links, fn link ->
+      [ext | rest] = String.split(link[:type], "/")
       rest == [] or MapSet.member?(extensions, ext)
     end)
   end

lib/schema/utils.ex

@@ -83,8 +83,32 @@ defmodule Schema.Utils do
     Enum.filter(dictionary, fn {_name, map} -> Map.get(map, :object_type) == name end)
     |> Enum.map(fn {_, map} -> Map.get(map, :_links) end)
     |> List.flatten()
-    |> Stream.filter(fn links -> links != nil end)
-    |> Stream.uniq()
+    |> Enum.filter(fn links -> links != nil end)
+    # We need to de-duplicate by group and type, and merge the attribute_keys sets for each
+    # First group_by
+    |> Enum.group_by(fn link -> {link[:group], link[:type]} end)
+    # Next use reduce to merge each group
+    |> Enum.reduce(
+      [],
+      fn {_group, group_links}, acc ->
+        group_link =
+          Enum.reduce(
+            group_links,
+            fn link, link_acc ->
+              Map.update(
+                link_acc,
+                :attribute_keys,
+                MapSet.new(),
+                fn attribute_keys ->
+                  MapSet.union(attribute_keys, link[:attribute_keys])
+                end
+              )
+            end
+          )
+
+        [group_link | acc]
+      end
+    )
     |> Enum.to_list()
   end
 
@@ -158,7 +182,7 @@ defmodule Schema.Utils do
   # Adds attribute's used-by links to the dictionary.
   defp add_common_links(dict, class) do
     Map.update!(dict, :attributes, fn attributes ->
-      link = {:common, class[:name], class[:caption]}
+      link = %{group: :common, type: class[:name], caption: class[:caption]}
 
       update_attributes(
         class,
@@ -177,7 +201,7 @@ defmodule Schema.Utils do
           _ -> Atom.to_string(name)
         end
 
-      link = {:class, type, class[:caption] || "*No name*"}
+      link = %{group: :class, type: type, caption: class[:caption] || "*No name*"}
 
       update_attributes(
         class,
@@ -190,14 +214,13 @@ defmodule Schema.Utils do
 
   defp update_dictionary_links(item, link) do
     Map.update(item, :_links, [link], fn links ->
-      [{_, id, _} | _] = links
-      if id > 0, do: [link | links], else: links
+      [link | links]
     end)
   end
 
   defp add_object_links(dict, {name, obj}) do
     Map.update!(dict, :attributes, fn dictionary ->
-      link = {:object, Atom.to_string(name), obj[:caption] || "*No name*"}
+      link = %{group: :object, type: Atom.to_string(name), caption: obj[:caption] || "*No name*"}
       update_attributes(obj, dictionary, link, &update_object_links/2)
     end)
   end
@@ -212,26 +235,39 @@ defmodule Schema.Utils do
     name = item[:caption]
     attributes = item[:attributes]
 
-    Enum.reduce(attributes, dictionary, fn {k, v}, acc ->
-      case find_entity(acc, item, k) do
+    Enum.reduce(attributes, dictionary, fn {attribute_key, attribute_map}, acc ->
+      link =
+        Map.update(
+          link,
+          :attribute_keys,
+          MapSet.new([attribute_key]),
+          fn attribute_keys ->
+            MapSet.put(attribute_keys, attribute_key)
+          end
+        )
+
+      case find_entity(acc, item, attribute_key) do
         {_, nil} ->
-          case String.split(Atom.to_string(v[:_source]), "/") do
+          case String.split(Atom.to_string(attribute_map[:_source]), "/") do
             [ext, _] ->
-              ext_key = String.to_atom("#{ext}/#{k}")
+              ext_key = String.to_atom("#{ext}/#{attribute_key}")
 
               data =
                 case Map.get(acc, ext_key) do
                   nil ->
-                    update_links.(v, link)
+                    update_links.(attribute_map, link)
 
                   attr ->
-                    deep_merge(attr, v) |> update_links.(link)
+                    deep_merge(attr, attribute_map) |> update_links.(link)
                 end
 
               Map.put(acc, ext_key, data)
 
             _ ->
-              Logger.warning("'#{name}' uses undefined attribute: #{k}: #{inspect(v)}")
+              Logger.warning(
+                "'#{name}' uses undefined attribute: #{attribute_key}: #{inspect(attribute_map)}"
+              )
+
               acc
           end