diff --git a/specification/_doc_ids/table.csv b/specification/_doc_ids/table.csv index d62eed207..ca10e4db2 100644 --- a/specification/_doc_ids/table.csv +++ b/specification/_doc_ids/table.csv @@ -78,6 +78,7 @@ cluster-stats,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/c cluster-update-settings,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/cluster-update-settings.html cluster,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/cluster.html common-options,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/common-options.html +community-id-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/community-id-processor.html connector-sync-job-cancel,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/cancel-connector-sync-job-api.html connector-sync-job-delete,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/delete-connector-sync-job-api.html connector-sync-job-get,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/get-connector-sync-job-api.html @@ -157,6 +158,7 @@ fail-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/ field-and-document-access-control,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/field-and-document-access-control.html field-usage-stats,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/field-usage-stats.html find-structure,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/find-structure.html +fingerprint-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/fingerprint-processor.html foreach-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/foreach-processor.html fuzziness,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/common-options.html#fuzziness gap-policy,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/search-aggregations-pipeline.html#gap-policy @@ -329,6 +331,7 @@ modules-scripting,https://www.elastic.co/guide/en/elasticsearch/reference/{branc modules-snapshots,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/modules-snapshots.html monitor-elasticsearch-cluster,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/monitor-elasticsearch-cluster.html multi-fields,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/multi-fields.html +network-direction-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/network-direction-processor.html node-roles,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/modules-node.html#node-roles paginate-search-results,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/paginate-search-results.html painless-contexts,https://www.elastic.co/guide/en/elasticsearch/painless/{branch}/painless-contexts.html @@ -407,6 +410,7 @@ query-dsl,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/query realtime,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/docs-get.html#realtime redact-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/redact-processor.html regexp-syntax,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/regexp-syntax.html +registered-domain-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/registered-domain-processor.html remove-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/remove-processor.html rename-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/rename-processor.html reroute-processor,https://www.elastic.co/guide/en/elasticsearch/reference/{branch}/reroute-processor.html diff --git a/specification/ingest/_types/Processors.ts b/specification/ingest/_types/Processors.ts index 0ee0ad802..33402841a 100644 --- a/specification/ingest/_types/Processors.ts +++ b/specification/ingest/_types/Processors.ts @@ -55,6 +55,13 @@ export class ProcessorContainer { * @doc_id ingest-circle-processor */ circle?: CircleProcessor + /** + * Computes the Community ID for network flow data as defined in the + * Community ID Specification. You can use a community ID to correlate network + * events related to a single flow. + * @doc_id community-id-processor + */ + community_id?: CommunityIDProcessor /** * Converts a field in the currently ingested document to a different type, such as converting a string to an integer. * If the field value is an array, all members will be converted. @@ -106,6 +113,12 @@ export class ProcessorContainer { * @doc_id fail-processor */ fail?: FailProcessor + /** + * Computes a hash of the document’s content. You can use this hash for + * content fingerprinting. + * @doc_id fingerprint-processor + */ + fingerprint?: FingerprintProcessor /** * Runs an ingest processor on each element of an array or object. * @doc_id foreach-processor @@ -169,6 +182,12 @@ export class ProcessorContainer { * @doc_id lowercase-processor */ lowercase?: LowercaseProcessor + /** + * Calculates the network direction given a source IP address, destination IP + * address, and a list of internal networks. + * @doc_id network-direction-processor + */ + network_direction?: NetworkDirectionProcessor /** * Executes another pipeline. * @doc_id pipeline-processor @@ -181,6 +200,14 @@ export class ProcessorContainer { * @doc_id redact-processor */ redact?: RedactProcessor + /** + * Extracts the registered domain (also known as the effective top-level + * domain or eTLD), sub-domain, and top-level domain from a fully qualified + * domain name (FQDN). Uses the registered domains defined in the Mozilla + * Public Suffix List. + * @doc_id registered-domain-processor + */ + registered_domain?: RegisteredDomainProcessor /** * Removes existing fields. * If one field doesn’t exist, an exception will be thrown. @@ -528,13 +555,77 @@ export class CircleProcessor extends ProcessorBase { target_field?: Field } +export class CommunityIDProcessor extends ProcessorBase { + /** + * Field containing the source IP address. + * @server_default source.ip + */ + source_ip?: Field + /** + * Field containing the source port. + * @server_default source.port + */ + source_port?: Field + /** + * Field containing the destination IP address. + * @server_default destination.ip + */ + destination_ip?: Field + /** + * Field containing the destination port. + * @server_default destination.port + */ + destination_port?: Field + /** + * Field containing the IANA number. + * @server_default network.iana_number + */ + iana_number?: Field + /** + * Field containing the ICMP type. + * @server_default icmp.type + */ + icmp_type?: Field + /** + * Field containing the ICMP code. + * @server_default icmp.code + */ + icmp_code?: Field + /** + * Field containing the transport protocol name or number. Used only when the + * iana_number field is not present. The following protocol names are currently + * supported: eigrp, gre, icmp, icmpv6, igmp, ipv6-icmp, ospf, pim, sctp, tcp, udp + * @server_default network.transport + */ + transport?: Field + /** + * Output field for the community ID. + * @server_default network.community_id + */ + target_field?: Field + /** + * Seed for the community ID hash. Must be between 0 and 65535 (inclusive). The + * seed can prevent hash collisions between network domains, such as a staging + * and production network that use the same addressing scheme. + * @server_default 0 + */ + seed?: integer + /** + * If true and any required fields are missing, the processor quietly exits + * without modifying the document. + * @server_default true + */ + ignore_missing?: boolean +} + export enum ConvertType { integer, long, - float, double, - string, + float, boolean, + ip, + string, auto } @@ -662,6 +753,12 @@ export class DateProcessor extends ProcessorBase { * @server_default UTC */ timezone?: string + /** + * The format to use when writing the date to target_field. Must be a valid + * java time pattern. + * @server_default yyyy-MM-dd'T'HH:mm:ss.SSSXXX + */ + output_format?: string } export class DissectProcessor extends ProcessorBase { @@ -756,6 +853,44 @@ export class FailProcessor extends ProcessorBase { message: string } +export enum FingerprintDigest { + md5 = 'MD5', + sha1 = 'SHA-1', + sha256 = 'SHA-256', + sha512 = 'SHA-512', + murmurHash3 = 'MurmurHash3' +} + +export class FingerprintProcessor extends ProcessorBase { + /** + * Array of fields to include in the fingerprint. For objects, the processor + * hashes both the field key and value. For other fields, the processor hashes + * only the field value. + */ + fields: Fields + /** + * Output field for the fingerprint. + * @server_default fingerprint + */ + target_field?: Field + /** + * Salt value for the hash function. + */ + salt?: string + /** + * The hash method used to compute the fingerprint. Must be one of MD5, SHA-1, + * SHA-256, SHA-512, or MurmurHash3. + * @server_default SHA-1 + */ + method?: FingerprintDigest + /** + * If true, the processor ignores any missing fields. If all fields are + * missing, the processor silently exits without modifying the document. + * @server_default false + */ + ignore_missing?: boolean +} + export class ForeachProcessor extends ProcessorBase { /** * Field containing array or object values. @@ -773,6 +908,12 @@ export class ForeachProcessor extends ProcessorBase { } export class GrokProcessor extends ProcessorBase { + /** + * Must be disabled or v1. If v1, the processor uses patterns with Elastic + * Common Schema (ECS) field names. + * @server_default disabled + */ + ecs_compatibility?: string /** * The field to use for grok expression parsing. */ @@ -1046,6 +1187,42 @@ export class LowercaseProcessor extends ProcessorBase { target_field?: Field } +export class NetworkDirectionProcessor extends ProcessorBase { + /** + * Field containing the source IP address. + * @server_default source.ip + */ + source_ip?: Field + /** + * Field containing the destination IP address. + * @server_default destination.ip + */ + destination_ip?: Field + /** + * Output field for the network direction. + * @server_default network.direction + */ + target_field?: Field + /** + * List of internal networks. Supports IPv4 and IPv6 addresses and ranges in + * CIDR notation. Also supports the named ranges listed below. These may be + * constructed with template snippets. Must specify only one of + * internal_networks or internal_networks_field. + */ + internal_networks?: string[] + /** + * A field on the given document to read the internal_networks configuration + * from. + */ + internal_networks_field?: Field + /** + * If true and any required fields are missing, the processor quietly exits + * without modifying the document. + * @server_default true + */ + ignore_missing?: boolean +} + export class PipelineProcessor extends ProcessorBase { /** * The name of the pipeline to execute. @@ -1102,6 +1279,24 @@ export class RedactProcessor extends ProcessorBase { trace_redact?: boolean } +export class RegisteredDomainProcessor extends ProcessorBase { + /** + * Field containing the source FQDN. + */ + field: Field + /** + * Object field containing extracted domain components. If an empty string, + * the processor adds components to the document’s root. + */ + target_field?: Field + /** + * If true and any required fields are missing, the processor quietly exits + * without modifying the document. + * @server_default true + */ + ignore_missing?: boolean +} + export class RemoveProcessor extends ProcessorBase { /** * Fields to be removed. Supports template snippets.