diff --git a/lib/schema.ex b/lib/schema.ex index 24fc239..8a96086 100644 --- a/lib/schema.ex +++ b/lib/schema.ex @@ -114,7 +114,7 @@ defmodule Schema do @spec data_type?(binary(), binary() | list(binary())) :: boolean() def data_type?(type, type), do: true - + def data_type?(type, base_type) when is_binary(base_type) do types = Map.get(Repo.data_types(), :attributes) @@ -153,6 +153,9 @@ defmodule Schema do |> apply_profiles(profiles, MapSet.size(profiles)) end + @spec all_classes() :: map() + def all_classes(), do: Repo.all_classes() + @doc """ Returns a single event class. """ @@ -430,7 +433,7 @@ defmodule Schema do # ----------------------------# def enrich(data, enum_text, observables) do - Schema.Helper.enrich(data, enum_text, observables) + Schema.Helper.enrich(data, enum_text, observables) end # -------------------------------# diff --git a/lib/schema/cache.ex b/lib/schema/cache.ex index 4473b50..b1725d6 100644 --- a/lib/schema/cache.ex +++ b/lib/schema/cache.ex @@ -19,27 +19,29 @@ defmodule Schema.Cache do require Logger - @enforce_keys [:version, :profiles, :dictionary, :categories, :base_event, :classes, :objects] - defstruct ~w[version profiles dictionary base_event categories classes objects]a - - @spec new(map()) :: __MODULE__.t() - def new(version) do - %__MODULE__{ - version: version, - profiles: Map.new(), - dictionary: Map.new(), - categories: Map.new(), - base_event: Map.new(), - classes: Map.new(), - objects: Map.new() - } - end + @enforce_keys [ + :version, + :profiles, + :categories, + :dictionary, + :base_event, + :classes, + :all_classes, + :objects + ] + defstruct ~w[version profiles dictionary base_event categories classes all_classes objects]a @type t() :: %__MODULE__{} @type class_t() :: map() @type object_t() :: map() @type category_t() :: map() @type dictionary_t() :: map() + @type link_t() :: %{ + group: :common | :class | :object, + type: String.t(), + caption: String.t(), + attribute_keys: nil | MapSet.t(String.t()) + } @ocsf_deprecated :"@deprecated" @@ -53,7 +55,7 @@ defmodule Schema.Cache do categories = JsonReader.read_categories() |> update_categories() dictionary = JsonReader.read_dictionary() |> update_dictionary() - {base_event, classes} = read_classes(categories[:attributes]) + {base_event, classes, all_classes} = read_classes(categories[:attributes]) objects = read_objects() dictionary = Utils.update_dictionary(dictionary, base_event, classes, objects) @@ -97,13 +99,16 @@ defmodule Schema.Cache do ) end - new(version) - |> set_profiles(profiles) - |> set_categories(categories) - |> set_dictionary(dictionary) - |> set_base_event(base_event) - |> set_classes(classes) - |> set_objects(objects) + %__MODULE__{ + version: version, + profiles: profiles, + categories: categories, + dictionary: dictionary, + base_event: base_event, + classes: classes, + all_classes: all_classes, + objects: objects + } end @doc """ @@ -141,6 +146,9 @@ defmodule Schema.Cache do @spec classes(__MODULE__.t()) :: map() def classes(%__MODULE__{classes: classes}), do: classes + @spec all_classes(__MODULE__.t()) :: map() + def all_classes(%__MODULE__{all_classes: all_classes}), do: all_classes + @spec export_classes(__MODULE__.t()) :: map() def export_classes(%__MODULE__{classes: classes, dictionary: dictionary}) do Enum.into(classes, Map.new(), fn {name, class} -> @@ -321,13 +329,29 @@ defmodule Schema.Cache do |> Enum.into(%{}, fn class -> attribute_source(class) end) |> extend_type() + resolved = resolve_extends(classes) + classes = - resolve_extends(classes) + resolved # remove intermediate classes |> Stream.filter(fn {key, class} -> Map.has_key?(class, :uid) or key == :base_event end) |> Enum.into(%{}, fn class -> enrich_class(class, categories) end) - {Map.get(classes, :base_event), classes} + # all_classes has just enough info to interrogate the complete class hierarchy, + # removing most details. It can be used to get the caption and parent (extends) of + # any class, including hidden ones (classes without a uid) + all_classes = + Enum.map( + resolved, + fn {class_name, class_info} -> + {class_name, + Map.take(class_info, [:name, :caption, :extends]) + |> Map.put(:hidden?, class_name != :base_event && !Map.has_key?(class_info, :uid))} + end + ) + |> Enum.into(%{}) + + {Map.get(classes, :base_event), classes, all_classes} end defp read_objects() do @@ -666,30 +690,6 @@ defmodule Schema.Cache do end end - defp set_profiles(%__MODULE__{} = schema, profiles) do - struct(schema, profiles: profiles) - end - - defp set_dictionary(%__MODULE__{} = schema, dictionary) do - struct(schema, dictionary: dictionary) - end - - defp set_categories(%__MODULE__{} = schema, categories) do - struct(schema, categories: categories) - end - - defp set_base_event(%__MODULE__{} = schema, base_event) do - struct(schema, base_event: base_event) - end - - defp set_classes(%__MODULE__{} = schema, classes) do - struct(schema, classes: classes) - end - - defp set_objects(%__MODULE__{} = schema, objects) do - struct(schema, objects: objects) - end - defp update_observables(objects, dictionary) do if Map.has_key?(objects, :observable) do observable_types = get_in(dictionary, [:types, :attributes]) |> observables() @@ -869,10 +869,10 @@ defmodule Schema.Cache do end end - defp update_linked_profiles(name, links, object, classes) do - Enum.reduce(links, classes, fn {type, key, _}, acc -> - if type == name do - Map.update!(acc, String.to_atom(key), fn class -> + defp update_linked_profiles(group, links, object, classes) do + Enum.reduce(links, classes, fn link, acc -> + if link[:group] == group do + Map.update!(acc, String.to_atom(link[:type]), fn class -> Map.put(class, :profiles, merge(class[:profiles], object[:profiles])) end) else diff --git a/lib/schema/profiles.ex b/lib/schema/profiles.ex index a402a5e..e7fa380 100644 --- a/lib/schema/profiles.ex +++ b/lib/schema/profiles.ex @@ -48,21 +48,21 @@ defmodule Schema.Profiles do @doc """ Checks classes or objects if all profile attributes are defined. """ - def sanity_check(type, maps, profiles) do + def sanity_check(group, maps, profiles) do profiles = Enum.reduce(maps, profiles, fn {name, map}, acc -> - check_profiles(type, name, map, map[:profiles], acc) + check_profiles(group, name, map, map[:profiles], acc) end) {maps, profiles} end # Checks if all profile attributes are defined in the given attribute set. - defp check_profiles(_type, _name, _map, nil, all_profiles) do + defp check_profiles(_group, _name, _map, nil, all_profiles) do all_profiles end - defp check_profiles(type, name, map, profiles, all_profiles) do + defp check_profiles(group, name, map, profiles, all_profiles) do Enum.reduce(profiles, all_profiles, fn p, acc -> case acc[p] do nil -> @@ -71,7 +71,7 @@ defmodule Schema.Profiles do profile -> check_profile(name, profile, map[:attributes]) - link = {type, Atom.to_string(name), map[:caption]} + link = %{group: group, type: Atom.to_string(name), caption: map[:caption]} profile = Map.update(profile, :_links, [link], fn links -> [link | links] end) Map.put(acc, p, profile) end diff --git a/lib/schema/repo.ex b/lib/schema/repo.ex index b528c77..e3add6f 100644 --- a/lib/schema/repo.ex +++ b/lib/schema/repo.ex @@ -119,6 +119,11 @@ defmodule Schema.Repo do Agent.get(__MODULE__, fn schema -> Cache.classes(schema) |> filter(extensions) end) end + @spec all_classes() :: map() + def all_classes() do + Agent.get(__MODULE__, fn schema -> Cache.all_classes(schema) end) + end + @spec export_classes() :: map() def export_classes() do Agent.get(__MODULE__, fn schema -> Cache.export_classes(schema) end) @@ -248,8 +253,8 @@ defmodule Schema.Repo do defp remove_extension_links(nil, _extensions), do: [] defp remove_extension_links(links, extensions) do - Enum.filter(links, fn {_, key, _} -> - [ext | rest] = String.split(key, "/") + Enum.filter(links, fn link -> + [ext | rest] = String.split(link[:type], "/") rest == [] or MapSet.member?(extensions, ext) end) end diff --git a/lib/schema/utils.ex b/lib/schema/utils.ex index ebdc7a8..ae8a8bb 100644 --- a/lib/schema/utils.ex +++ b/lib/schema/utils.ex @@ -83,8 +83,32 @@ defmodule Schema.Utils do Enum.filter(dictionary, fn {_name, map} -> Map.get(map, :object_type) == name end) |> Enum.map(fn {_, map} -> Map.get(map, :_links) end) |> List.flatten() - |> Stream.filter(fn links -> links != nil end) - |> Stream.uniq() + |> Enum.filter(fn links -> links != nil end) + # We need to de-duplicate by group and type, and merge the attribute_keys sets for each + # First group_by + |> Enum.group_by(fn link -> {link[:group], link[:type]} end) + # Next use reduce to merge each group + |> Enum.reduce( + [], + fn {_group, group_links}, acc -> + group_link = + Enum.reduce( + group_links, + fn link, link_acc -> + Map.update( + link_acc, + :attribute_keys, + MapSet.new(), + fn attribute_keys -> + MapSet.union(attribute_keys, link[:attribute_keys]) + end + ) + end + ) + + [group_link | acc] + end + ) |> Enum.to_list() end @@ -158,7 +182,7 @@ defmodule Schema.Utils do # Adds attribute's used-by links to the dictionary. defp add_common_links(dict, class) do Map.update!(dict, :attributes, fn attributes -> - link = {:common, class[:name], class[:caption]} + link = %{group: :common, type: class[:name], caption: class[:caption]} update_attributes( class, @@ -177,7 +201,7 @@ defmodule Schema.Utils do _ -> Atom.to_string(name) end - link = {:class, type, class[:caption] || "*No name*"} + link = %{group: :class, type: type, caption: class[:caption] || "*No name*"} update_attributes( class, @@ -190,14 +214,13 @@ defmodule Schema.Utils do defp update_dictionary_links(item, link) do Map.update(item, :_links, [link], fn links -> - [{_, id, _} | _] = links - if id > 0, do: [link | links], else: links + [link | links] end) end defp add_object_links(dict, {name, obj}) do Map.update!(dict, :attributes, fn dictionary -> - link = {:object, Atom.to_string(name), obj[:caption] || "*No name*"} + link = %{group: :object, type: Atom.to_string(name), caption: obj[:caption] || "*No name*"} update_attributes(obj, dictionary, link, &update_object_links/2) end) end @@ -212,26 +235,39 @@ defmodule Schema.Utils do name = item[:caption] attributes = item[:attributes] - Enum.reduce(attributes, dictionary, fn {k, v}, acc -> - case find_entity(acc, item, k) do + Enum.reduce(attributes, dictionary, fn {attribute_key, attribute_map}, acc -> + link = + Map.update( + link, + :attribute_keys, + MapSet.new([attribute_key]), + fn attribute_keys -> + MapSet.put(attribute_keys, attribute_key) + end + ) + + case find_entity(acc, item, attribute_key) do {_, nil} -> - case String.split(Atom.to_string(v[:_source]), "/") do + case String.split(Atom.to_string(attribute_map[:_source]), "/") do [ext, _] -> - ext_key = String.to_atom("#{ext}/#{k}") + ext_key = String.to_atom("#{ext}/#{attribute_key}") data = case Map.get(acc, ext_key) do nil -> - update_links.(v, link) + update_links.(attribute_map, link) attr -> - deep_merge(attr, v) |> update_links.(link) + deep_merge(attr, attribute_map) |> update_links.(link) end Map.put(acc, ext_key, data) _ -> - Logger.warning("'#{name}' uses undefined attribute: #{k}: #{inspect(v)}") + Logger.warning( + "'#{name}' uses undefined attribute: #{attribute_key}: #{inspect(attribute_map)}" + ) + acc end diff --git a/lib/schema_web/controllers/schema_controller.ex b/lib/schema_web/controllers/schema_controller.ex index 0961587..3e9db0a 100644 --- a/lib/schema_web/controllers/schema_controller.ex +++ b/lib/schema_web/controllers/schema_controller.ex @@ -81,7 +81,7 @@ defmodule SchemaWeb.SchemaController do category(:string, "Class category", required: true) category_name(:string, "Class category caption", required: true) profiles(:array, "Class profiles", items: %PhoenixSwagger.Schema{type: :string}) - uid(:integer, "Class unique indentifier", required: true) + uid(:integer, "Class unique identifier", required: true) end example([ @@ -168,32 +168,40 @@ defmodule SchemaWeb.SchemaController do @spec versions(Plug.Conn.t(), any) :: Plug.Conn.t() def versions(conn, _params) do - url = Application.get_env(:schema_server, SchemaWeb.Endpoint)[:url] # The :url key is meant to be set for production, but isn't set for local development - base_url = if url == nil do - "#{conn.scheme}://#{conn.host}:#{conn.port}" - else - "#{conn.scheme}://#{Keyword.fetch!(url, :host)}:#{Keyword.fetch!(url, :port)}" - end + base_url = + if url == nil do + "#{conn.scheme}://#{conn.host}:#{conn.port}" + else + "#{conn.scheme}://#{Keyword.fetch!(url, :host)}:#{Keyword.fetch!(url, :port)}" + end - available_versions = Schemas.versions() - |> Enum.map(fn {version, _} -> version end) + available_versions = + Schemas.versions() + |> Enum.map(fn {version, _} -> version end) - default_version = %{:version => Schema.version(), :url => "#{base_url}/#{Schema.version()}/api"} + default_version = %{ + :version => Schema.version(), + :url => "#{base_url}/#{Schema.version()}/api" + } - versions_response = case available_versions do - [] -> - # If there is no response, we only provide a single schema - %{:versions => [default_version], :default => default_version} + versions_response = + case available_versions do + [] -> + # If there is no response, we only provide a single schema + %{:versions => [default_version], :default => default_version} - [_head | _tail] -> - available_versions_objects = available_versions - |> Enum.map(fn version -> %{:version => version, :url => "#{base_url}/#{version}/api"} end) - %{:versions => available_versions_objects, :default => default_version} + [_head | _tail] -> + available_versions_objects = + available_versions + |> Enum.map(fn version -> + %{:version => version, :url => "#{base_url}/#{version}/api"} + end) - end + %{:versions => available_versions_objects, :default => default_version} + end send_json_resp(conn, versions_response) end @@ -256,6 +264,7 @@ defmodule SchemaWeb.SchemaController do Enum.into(get_profiles(params), %{}, fn {k, v} -> {k, Schema.delete_links(v)} end) + send_json_resp(conn, profiles) end @@ -271,7 +280,7 @@ defmodule SchemaWeb.SchemaController do @doc """ Get a profile by name. get /api/profiles/:name - get /api/profiles/:extention/:name + get /api/profiles/:extension/:name """ swagger_path :profile do get("/api/profiles/{name}") @@ -294,13 +303,15 @@ defmodule SchemaWeb.SchemaController do @spec profile(Plug.Conn.t(), map) :: Plug.Conn.t() def profile(conn, %{"id" => id} = params) do - name = case params["extension"] do - nil -> id - extension -> "#{extension}/#{id}" - end + name = + case params["extension"] do + nil -> id + extension -> "#{extension}/#{id}" + end try do data = Schema.profiles() + case Map.get(data, name) do nil -> send_json_resp(conn, 404, %{error: "Profile #{name} not found"}) @@ -553,7 +564,7 @@ defmodule SchemaWeb.SchemaController do @doc """ Get an object by name. get /api/objects/:name - get /api/objects/:extention/:name + get /api/objects/:extension/:name """ swagger_path :object do get("/api/objects/{name}") @@ -650,7 +661,7 @@ defmodule SchemaWeb.SchemaController do swagger_path :export_schema do get("/export/schema") summary("Export schema") - description("Get OCSF schema defintions, including data types, objects, and classes.") + description("Get OCSF schema definitions, including data types, objects, and classes.") produces("application/json") tag("Schema Export") @@ -900,7 +911,9 @@ defmodule SchemaWeb.SchemaController do default: false ) - data(:body, PhoenixSwagger.Schema.ref(:Event), "The event data to be enriched.", required: true) + data(:body, PhoenixSwagger.Schema.ref(:Event), "The event data to be enriched.", + required: true + ) end response(200, "Success") @@ -978,7 +991,9 @@ defmodule SchemaWeb.SchemaController do allowEmptyValue: true ) - data(:body, PhoenixSwagger.Schema.ref(:Event), "The event data to be translated", required: true) + data(:body, PhoenixSwagger.Schema.ref(:Event), "The event data to be translated", + required: true + ) end response(200, "Success") @@ -1026,7 +1041,9 @@ defmodule SchemaWeb.SchemaController do tag("Tools") parameters do - data(:body, PhoenixSwagger.Schema.ref(:Event), "The event data to be validated", required: true) + data(:body, PhoenixSwagger.Schema.ref(:Event), "The event data to be validated", + required: true + ) end response(200, "Success") @@ -1082,7 +1099,7 @@ defmodule SchemaWeb.SchemaController do @doc """ Returns randomly generated event sample data for the given name. get /sample/classes/:name - get /sample/classes/:extention/:name + get /sample/classes/:extension/:name """ swagger_path :sample_class do get("/sample/classes/{name}") @@ -1144,7 +1161,7 @@ defmodule SchemaWeb.SchemaController do @doc """ Returns randomly generated object sample data for the given name. get /sample/objects/:name - get /sample/objects/:extention/:name + get /sample/objects/:extension/:name """ swagger_path :sample_object do get("/sample/objects/{name}") diff --git a/lib/schema_web/templates/layout/app.html.eex b/lib/schema_web/templates/layout/app.html.eex index f6969a7..e499ff3 100644 --- a/lib/schema_web/templates/layout/app.html.eex +++ b/lib/schema_web/templates/layout/app.html.eex @@ -114,7 +114,7 @@ limitations under the License. OCSF Schema