diff --git a/client/dist/src/gen/broker/protocol/broker.ts b/client/dist/src/gen/broker/protocol/broker.ts index 964b6ea..555d57f 100644 --- a/client/dist/src/gen/broker/protocol/broker.ts +++ b/client/dist/src/gen/broker/protocol/broker.ts @@ -14,37 +14,37 @@ of JournalSpecs. Exactly one of |upsert| or |delete| must be set. */ export interface ApplyRequestChange { - /** - * Expected ModRevision of the current JournalSpec. If the Journal is being - * created, expect_mod_revision is zero. - * @format int64 - */ - expectModRevision?: string; - /** - * JournalSpec to be updated (if expect_mod_revision > 0) or created - * (if expect_mod_revision == 0). - */ - upsert?: ProtocolJournalSpec; - /** Journal to be deleted. expect_mod_revision must not be zero. */ - delete?: string; + /** + * Expected ModRevision of the current JournalSpec. If the Journal is being + * created, expect_mod_revision is zero. + * @format int64 + */ + expectModRevision?: string; + /** + * JournalSpec to be updated (if expect_mod_revision > 0) or created + * (if expect_mod_revision == 0). + */ + upsert?: ProtocolJournalSpec; + /** Journal to be deleted. expect_mod_revision must not be zero. */ + delete?: string; } /** * Fragments of the Response. */ export interface FragmentsResponseFragment { - /** - * Fragment is a content-addressed description of a contiguous Journal span, - * defined by the [begin, end) offset range covered by the Fragment and the - * SHA1 sum of the corresponding Journal content. - */ - spec?: ProtocolFragment; - /** - * SignedURL is a temporary URL at which a direct GET of the Fragment may - * be issued, signed by the broker's credentials. Set only if the request - * specified a SignatureTTL. - */ - signedUrl?: string; + /** + * Fragment is a content-addressed description of a contiguous Journal span, + * defined by the [begin, end) offset range covered by the Fragment and the + * SHA1 sum of the corresponding Journal content. + */ + spec?: ProtocolFragment; + /** + * SignedURL is a temporary URL at which a direct GET of the Fragment may + * be issued, signed by the broker's credentials. Set only if the request + * specified a SignatureTTL. + */ + signedUrl?: string; } /** @@ -58,106 +58,106 @@ different Etcd clusters). Etcd is kept in sync with etcdserverpb.ResponseHeader. */ export interface HeaderEtcd { - /** - * cluster_id is the ID of the cluster. - * @format uint64 - */ - clusterId?: string; - /** - * member_id is the ID of the member. - * @format uint64 - */ - memberId?: string; - /** - * revision is the Etcd key-value store revision when the request was - * applied. - * @format int64 - */ - revision?: string; - /** - * raft_term is the raft term when the request was applied. - * @format uint64 - */ - raftTerm?: string; + /** + * cluster_id is the ID of the cluster. + * @format uint64 + */ + clusterId?: string; + /** + * member_id is the ID of the member. + * @format uint64 + */ + memberId?: string; + /** + * revision is the Etcd key-value store revision when the request was + * applied. + * @format int64 + */ + revision?: string; + /** + * raft_term is the raft term when the request was applied. + * @format uint64 + */ + raftTerm?: string; } /** * Journals of the response. */ export interface ListResponseJournal { - /** JournalSpec describes a Journal and its configuration. */ - spec?: ProtocolJournalSpec; - /** - * Current ModRevision of the JournalSpec. - * @format int64 - */ - modRevision?: string; - /** Route of the journal, including endpoints. */ - route?: ProtocolRoute; + /** JournalSpec describes a Journal and its configuration. */ + spec?: ProtocolJournalSpec; + /** + * Current ModRevision of the JournalSpec. + * @format int64 + */ + modRevision?: string; + /** Route of the journal, including endpoints. */ + route?: ProtocolRoute; } /** * ID composes a zone and a suffix to uniquely identify a ProcessSpec. */ export interface ProcessSpecID { - /** - * "Zone" in which the process is running. Zones may be AWS, Azure, or - * Google Cloud Platform zone identifiers, or rack locations within a colo, - * or given some other custom meaning. Gazette will replicate across - * multiple zones, and seeks to minimize traffic which must cross zones (for - * example, by proxying reads to a broker in the current zone). - */ - zone?: string; - /** - * Unique suffix of the process within |zone|. It is permissible for a - * suffix value to repeat across zones, but never within zones. In practice, - * it's recommended to use a FQDN, Kubernetes Pod name, or comparable unique - * and self-describing value as the ID suffix. - */ - suffix?: string; + /** + * "Zone" in which the process is running. Zones may be AWS, Azure, or + * Google Cloud Platform zone identifiers, or rack locations within a colo, + * or given some other custom meaning. Gazette will replicate across + * multiple zones, and seeks to minimize traffic which must cross zones (for + * example, by proxying reads to a broker in the current zone). + */ + zone?: string; + /** + * Unique suffix of the process within |zone|. It is permissible for a + * suffix value to repeat across zones, but never within zones. In practice, + * it's recommended to use a FQDN, Kubernetes Pod name, or comparable unique + * and self-describing value as the ID suffix. + */ + suffix?: string; } export interface ProtobufAny { - typeUrl?: string; - /** @format byte */ - value?: string; + typeUrl?: string; + /** @format byte */ + value?: string; } /** * AppendResponse is the unary response message of the broker Append RPC. */ export interface ProtocolAppendResponse { - /** Status of the Append RPC. */ - status?: ProtocolStatus; - /** Header of the response. */ - header?: ProtocolHeader; - /** - * If status is OK, then |commit| is the Fragment which places the - * committed Append content within the Journal. - */ - commit?: ProtocolFragment; - /** Current registers of the journal. */ - registers?: ProtocolLabelSet; - /** - * Total number of RPC content chunks processed in this append. - * @format int64 - */ - totalChunks?: string; - /** - * Number of content chunks which were delayed by journal flow control. - * @format int64 - */ - delayedChunks?: string; + /** Status of the Append RPC. */ + status?: ProtocolStatus; + /** Header of the response. */ + header?: ProtocolHeader; + /** + * If status is OK, then |commit| is the Fragment which places the + * committed Append content within the Journal. + */ + commit?: ProtocolFragment; + /** Current registers of the journal. */ + registers?: ProtocolLabelSet; + /** + * Total number of RPC content chunks processed in this append. + * @format int64 + */ + totalChunks?: string; + /** + * Number of content chunks which were delayed by journal flow control. + * @format int64 + */ + delayedChunks?: string; } /** * ApplyResponse is the unary response message of the broker Apply RPC. */ export interface ProtocolApplyResponse { - /** Status of the Apply RPC. */ - status?: ProtocolStatus; - /** Header of the response. */ - header?: ProtocolHeader; + /** Status of the Apply RPC. */ + status?: ProtocolStatus; + /** Header of the response. */ + header?: ProtocolHeader; } /** @@ -185,12 +185,12 @@ header handling can be subtle and sometimes confusing. It uses the default suffix ".gzod". */ export type ProtocolCompressionCodec = - | 'INVALID' - | 'NONE' - | 'GZIP' - | 'ZSTANDARD' - | 'SNAPPY' - | 'GZIP_OFFLOAD_DECOMPRESSION'; + | "INVALID" + | "NONE" + | "GZIP" + | "ZSTANDARD" + | "SNAPPY" + | "GZIP_OFFLOAD_DECOMPRESSION"; /** * Fragment is a content-addressed description of a contiguous Journal span, @@ -198,37 +198,37 @@ defined by the [begin, end) offset range covered by the Fragment and the SHA1 sum of the corresponding Journal content. */ export interface ProtocolFragment { - /** Journal of the Fragment. */ - journal?: string; - /** - * Begin (inclusive) and end (exclusive) offset of the Fragment within the - * Journal. - * @format int64 - */ - begin?: string; - /** @format int64 */ - end?: string; - /** SHA1 sum of the Fragment's content. */ - sum?: ProtocolSHA1Sum; - /** Codec with which the Fragment's content is compressed. */ - compressionCodec?: ProtocolCompressionCodec; - /** - * Fragment store which backs the Fragment. Empty if the Fragment has yet to - * be persisted and is still local to a Broker. - */ - backingStore?: string; - /** - * Modification timestamp of the Fragment within the backing store, - * represented as seconds since the epoch. - * @format int64 - */ - modTime?: string; - /** - * Path postfix under which the fragment is persisted to the store. - * The complete Fragment store path is built from any path components of the - * backing store, followed by the journal name, followed by the path postfix. - */ - pathPostfix?: string; + /** Journal of the Fragment. */ + journal?: string; + /** + * Begin (inclusive) and end (exclusive) offset of the Fragment within the + * Journal. + * @format int64 + */ + begin?: string; + /** @format int64 */ + end?: string; + /** SHA1 sum of the Fragment's content. */ + sum?: ProtocolSHA1Sum; + /** Codec with which the Fragment's content is compressed. */ + compressionCodec?: ProtocolCompressionCodec; + /** + * Fragment store which backs the Fragment. Empty if the Fragment has yet to + * be persisted and is still local to a Broker. + */ + backingStore?: string; + /** + * Modification timestamp of the Fragment within the backing store, + * represented as seconds since the epoch. + * @format int64 + */ + modTime?: string; + /** + * Path postfix under which the fragment is persisted to the store. + * The complete Fragment store path is built from any path components of the + * backing store, followed by the journal name, followed by the path postfix. + */ + pathPostfix?: string; } /** @@ -236,18 +236,18 @@ export interface ProtocolFragment { RPC. */ export interface ProtocolFragmentsResponse { - /** Status of the Apply RPC. */ - status?: ProtocolStatus; - /** Header of the response. */ - header?: ProtocolHeader; - fragments?: FragmentsResponseFragment[]; - /** - * The NextPageToke value to be returned on subsequent Fragments requests. If - * the value is zero then there are no more fragments to be returned for this - * page. - * @format int64 - */ - nextPageToken?: string; + /** Status of the Apply RPC. */ + status?: ProtocolStatus; + /** Header of the response. */ + header?: ProtocolHeader; + fragments?: FragmentsResponseFragment[]; + /** + * The NextPageToke value to be returned on subsequent Fragments requests. If + * the value is zero then there are no more fragments to be returned for this + * page. + * @format int64 + */ + nextPageToken?: string; } /** @@ -255,67 +255,67 @@ export interface ProtocolFragmentsResponse { an RPC, and its effective Etcd state. */ export interface ProtocolHeader { - /** - * ID of the process responsible for request processing. May be empty iff - * Header is being used within a proxied request, and that request may be - * dispatched to any member of the Route. - */ - processId?: ProcessSpecID; - /** - * Route of processes specifically responsible for this RPC, or an empty Route - * if any process is capable of serving the RPC. - */ - route?: ProtocolRoute; - /** - * Etcd represents the effective Etcd MVCC state under which a Gazette broker - * is operating in its processing of requests and responses. Its inclusion - * allows brokers to reason about relative "happened before" Revision ordering - * of apparent routing conflicts in proxied or replicated requests, as well - * as enabling sanity checks over equality of Etcd ClusterId (and precluding, - * for example, split-brain scenarios where different brokers are backed by - * different Etcd clusters). Etcd is kept in sync with - * etcdserverpb.ResponseHeader. - */ - etcd?: HeaderEtcd; + /** + * ID of the process responsible for request processing. May be empty iff + * Header is being used within a proxied request, and that request may be + * dispatched to any member of the Route. + */ + processId?: ProcessSpecID; + /** + * Route of processes specifically responsible for this RPC, or an empty Route + * if any process is capable of serving the RPC. + */ + route?: ProtocolRoute; + /** + * Etcd represents the effective Etcd MVCC state under which a Gazette broker + * is operating in its processing of requests and responses. Its inclusion + * allows brokers to reason about relative "happened before" Revision ordering + * of apparent routing conflicts in proxied or replicated requests, as well + * as enabling sanity checks over equality of Etcd ClusterId (and precluding, + * for example, split-brain scenarios where different brokers are backed by + * different Etcd clusters). Etcd is kept in sync with + * etcdserverpb.ResponseHeader. + */ + etcd?: HeaderEtcd; } /** * JournalSpec describes a Journal and its configuration. */ export interface ProtocolJournalSpec { - /** Name of the Journal. */ - name?: string; - /** - * Desired replication of this Journal. This defines the Journal's tolerance - * to broker failures before data loss can occur (eg, a replication factor - * of three means two failures are tolerated). - * @format int32 - */ - replication?: number; - /** - * User-defined Labels of this JournalSpec. Two label names are reserved - * and may not be used within a JournalSpec's Labels: "name" and "prefix". - */ - labels?: ProtocolLabelSet; - /** - * Fragment is JournalSpec configuration which pertains to the creation, - * persistence, and indexing of the Journal's Fragments. - */ - fragment?: ProtocolJournalSpecFragment; - /** - * Flags of the Journal, as a combination of Flag enum values. The Flag enum - * is not used directly, as protobuf enums do not allow for or'ed bitfields. - * @format int64 - */ - flags?: number; - /** - * Maximum rate, in bytes-per-second, at which appends of this journal will - * be processed. If zero (the default), no rate limiting is applied. A global - * rate limit still may be in effect, in which case the effective rate is the - * smaller of the journal vs global rate. - * @format int64 - */ - maxAppendRate?: string; + /** Name of the Journal. */ + name?: string; + /** + * Desired replication of this Journal. This defines the Journal's tolerance + * to broker failures before data loss can occur (eg, a replication factor + * of three means two failures are tolerated). + * @format int32 + */ + replication?: number; + /** + * User-defined Labels of this JournalSpec. Two label names are reserved + * and may not be used within a JournalSpec's Labels: "name" and "prefix". + */ + labels?: ProtocolLabelSet; + /** + * Fragment is JournalSpec configuration which pertains to the creation, + * persistence, and indexing of the Journal's Fragments. + */ + fragment?: ProtocolJournalSpecFragment; + /** + * Flags of the Journal, as a combination of Flag enum values. The Flag enum + * is not used directly, as protobuf enums do not allow for or'ed bitfields. + * @format int64 + */ + flags?: number; + /** + * Maximum rate, in bytes-per-second, at which appends of this journal will + * be processed. If zero (the default), no rate limiting is applied. A global + * rate limit still may be in effect, in which case the effective rate is the + * smaller of the journal vs global rate. + * @format int64 + */ + maxAppendRate?: string; } /** @@ -323,80 +323,80 @@ export interface ProtocolJournalSpec { persistence, and indexing of the Journal's Fragments. */ export interface ProtocolJournalSpecFragment { - /** - * Target content length of each Fragment. In normal operation after - * Fragments reach at least this length, they will be closed and new ones - * begun. Note lengths may be smaller at times (eg, due to changes in - * Journal routing topology). Content length differs from Fragment file - * size, in that the former reflects uncompressed bytes. - * @format int64 - */ - length?: string; - /** Codec used to compress Journal Fragments. */ - compressionCodec?: ProtocolCompressionCodec; - /** - * Storage backend base path for this Journal's Fragments. Must be in URL - * form, with the choice of backend defined by the scheme. The full path of - * a Journal's Fragment is derived by joining the store path with the - * Fragment's ContentPath. Eg, given a fragment_store of - * "s3://My-AWS-bucket/a/prefix" and a JournalSpec of name "my/journal", - * a complete Fragment path might be: - * "s3://My-AWS-bucket/a/prefix/my/journal/000123-000456-789abcdef.gzip - * Multiple stores may be specified, in which case the Journal's Fragments - * are the union of all Fragments present across all stores, and new - * Fragments always persist to the first specified store. This can be - * helpful in performing incremental migrations, where new Journal content - * is written to the new store, while content in the old store remains - * available (and, depending on fragment_retention or recovery log pruning, - * may eventually be removed). - * - * If no stores are specified, the Journal is still use-able but will - * not persist Fragments to any a backing fragment store. This allows for - * real-time streaming use cases where reads of historical data are not - * needed. - */ - stores?: string[]; - /** - * Interval of time between refreshes of remote Fragment listings from - * configured fragment_stores. - */ - refreshInterval?: string; - /** - * Retention duration for historical Fragments of this Journal within the - * Fragment stores. If less than or equal to zero, Fragments are retained - * indefinitely. - */ - retention?: string; - /** - * Flush interval defines a uniform UTC time segment which, when passed, - * will prompt brokers to close and persist a fragment presently being - * written. - * - * Flush interval may be helpful in integrating the journal with a regularly - * scheduled batch work-flow which processes new files from the fragment - * store and has no particular awareness of Gazette. For example, setting - * flush_interval to 3600s will cause brokers to persist their present - * fragment on the hour, every hour, even if it has not yet reached its - * target length. A batch work-flow running at 5 minutes past the hour is - * then reasonably assured of seeing all events from the past hour. - * See also "gazctl journals fragments --help" for more discussion. - */ - flushInterval?: string; - /** - * Path postfix template is a Go template which evaluates to a partial - * path under which fragments are persisted to the store. A complete - * fragment path is constructed by appending path components from the - * fragment store, then the journal name, and then the postfix template. - * Path post-fixes can help in maintaining Hive compatible partitioning - * over fragment creation time. The fields ".Spool" and ".JournalSpec" - * are available for introspection in the template. For example, - * to partition on the UTC date and hour of creation, use: - * date={{ .Spool.FirstAppendTime.Format "2006-01-02" }}/hour={{ - * .Spool.FirstAppendTime.Format "15" }} - * - * Which will produce a path postfix like "date=2019-11-19/hour=22". - */ - pathPostfixTemplate?: string; + /** + * Target content length of each Fragment. In normal operation after + * Fragments reach at least this length, they will be closed and new ones + * begun. Note lengths may be smaller at times (eg, due to changes in + * Journal routing topology). Content length differs from Fragment file + * size, in that the former reflects uncompressed bytes. + * @format int64 + */ + length?: string; + /** Codec used to compress Journal Fragments. */ + compressionCodec?: ProtocolCompressionCodec; + /** + * Storage backend base path for this Journal's Fragments. Must be in URL + * form, with the choice of backend defined by the scheme. The full path of + * a Journal's Fragment is derived by joining the store path with the + * Fragment's ContentPath. Eg, given a fragment_store of + * "s3://My-AWS-bucket/a/prefix" and a JournalSpec of name "my/journal", + * a complete Fragment path might be: + * "s3://My-AWS-bucket/a/prefix/my/journal/000123-000456-789abcdef.gzip + * Multiple stores may be specified, in which case the Journal's Fragments + * are the union of all Fragments present across all stores, and new + * Fragments always persist to the first specified store. This can be + * helpful in performing incremental migrations, where new Journal content + * is written to the new store, while content in the old store remains + * available (and, depending on fragment_retention or recovery log pruning, + * may eventually be removed). + * + * If no stores are specified, the Journal is still use-able but will + * not persist Fragments to any a backing fragment store. This allows for + * real-time streaming use cases where reads of historical data are not + * needed. + */ + stores?: string[]; + /** + * Interval of time between refreshes of remote Fragment listings from + * configured fragment_stores. + */ + refreshInterval?: string; + /** + * Retention duration for historical Fragments of this Journal within the + * Fragment stores. If less than or equal to zero, Fragments are retained + * indefinitely. + */ + retention?: string; + /** + * Flush interval defines a uniform UTC time segment which, when passed, + * will prompt brokers to close and persist a fragment presently being + * written. + * + * Flush interval may be helpful in integrating the journal with a regularly + * scheduled batch work-flow which processes new files from the fragment + * store and has no particular awareness of Gazette. For example, setting + * flush_interval to 3600s will cause brokers to persist their present + * fragment on the hour, every hour, even if it has not yet reached its + * target length. A batch work-flow running at 5 minutes past the hour is + * then reasonably assured of seeing all events from the past hour. + * See also "gazctl journals fragments --help" for more discussion. + */ + flushInterval?: string; + /** + * Path postfix template is a Go template which evaluates to a partial + * path under which fragments are persisted to the store. A complete + * fragment path is constructed by appending path components from the + * fragment store, then the journal name, and then the postfix template. + * Path post-fixes can help in maintaining Hive compatible partitioning + * over fragment creation time. The fields ".Spool" and ".JournalSpec" + * are available for introspection in the template. For example, + * to partition on the UTC date and hour of creation, use: + * date={{ .Spool.FirstAppendTime.Format "2006-01-02" }}/hour={{ + * .Spool.FirstAppendTime.Format "15" }} + * + * Which will produce a path postfix like "date=2019-11-19/hour=22". + */ + pathPostfixTemplate?: string; } /** @@ -406,111 +406,111 @@ attributes which do not directly imply semantics to the core system, but are meaningful to users or for higher-level Gazette tools. */ export interface ProtocolLabel { - name?: string; - value?: string; + name?: string; + value?: string; } /** * LabelSelector defines a filter over LabelSets. */ export interface ProtocolLabelSelector { - /** - * Include is Labels which must be matched for a LabelSet to be selected. If - * empty, all Labels are included. An include Label with empty ("") value is - * matched by a Label of the same name having any value. - */ - include?: ProtocolLabelSet; - /** - * Exclude is Labels which cannot be matched for a LabelSet to be selected. If - * empty, no Labels are excluded. An exclude Label with empty ("") value - * excludes a Label of the same name having any value. - */ - exclude?: ProtocolLabelSet; + /** + * Include is Labels which must be matched for a LabelSet to be selected. If + * empty, all Labels are included. An include Label with empty ("") value is + * matched by a Label of the same name having any value. + */ + include?: ProtocolLabelSet; + /** + * Exclude is Labels which cannot be matched for a LabelSet to be selected. If + * empty, no Labels are excluded. An exclude Label with empty ("") value + * excludes a Label of the same name having any value. + */ + exclude?: ProtocolLabelSet; } /** * LabelSet is a collection of labels and their values. */ export interface ProtocolLabelSet { - /** Labels of the set. Instances must be unique and sorted over (Name, Value). */ - labels?: ProtocolLabel[]; + /** Labels of the set. Instances must be unique and sorted over (Name, Value). */ + labels?: ProtocolLabel[]; } /** * ListRequest is the unary request message of the broker List RPC. */ export interface ProtocolListRequest { - /** - * Selector optionally refines the set of journals which will be enumerated. - * If zero-valued, all journals are returned. Otherwise, only JournalSpecs - * matching the LabelSelector will be returned. Two meta-labels "name" and - * "prefix" are additionally supported by the selector, where: - * * name=examples/a-name will match a JournalSpec with Name - * "examples/a-name" - * * prefix=examples/ will match any JournalSpec having prefix "examples/". - * The prefix Label value must end in '/'. - */ - selector?: ProtocolLabelSelector; + /** + * Selector optionally refines the set of journals which will be enumerated. + * If zero-valued, all journals are returned. Otherwise, only JournalSpecs + * matching the LabelSelector will be returned. Two meta-labels "name" and + * "prefix" are additionally supported by the selector, where: + * * name=examples/a-name will match a JournalSpec with Name + * "examples/a-name" + * * prefix=examples/ will match any JournalSpec having prefix "examples/". + * The prefix Label value must end in '/'. + */ + selector?: ProtocolLabelSelector; } /** * ListResponse is the unary response message of the broker List RPC. */ export interface ProtocolListResponse { - /** Status of the List RPC. */ - status?: ProtocolStatus; - /** Header of the response. */ - header?: ProtocolHeader; - journals?: ListResponseJournal[]; + /** Status of the List RPC. */ + status?: ProtocolStatus; + /** Header of the response. */ + header?: ProtocolHeader; + journals?: ListResponseJournal[]; } /** * ReadRequest is the unary request message of the broker Read RPC. */ export interface ProtocolReadRequest { - /** Header is attached by a proxying broker peer. */ - header?: ProtocolHeader; - /** Journal to be read. */ - journal?: string; - /** - * Desired offset to begin reading from. Value -1 has special handling, where - * the read is performed from the current write head. All other positive - * values specify a desired exact byte offset to read from. If the offset is - * not available (eg, because it represents a portion of Journal which has - * been permanently deleted), the broker will return the next available - * offset. Callers should therefore always inspect the ReadResponse offset. - * @format int64 - */ - offset?: string; - /** - * Whether the operation should block until content becomes available. - * OFFSET_NOT_YET_AVAILABLE is returned if a non-blocking read has no ready - * content. - */ - block?: boolean; - /** - * If do_not_proxy is true, the broker will not proxy the read to another - * broker, or open and proxy a remote Fragment on the client's behalf. - */ - doNotProxy?: boolean; - /** - * If metadata_only is true, the broker will respond with Journal and - * Fragment metadata but not content. - */ - metadataOnly?: boolean; - /** - * Offset to read through. If zero, then the read end offset is unconstrained. - * @format int64 - */ - endOffset?: string; - /** - * BeginModTime is an optional inclusive lower bound on the modification - * timestamps of fragments read from the backing store, represented as - * seconds since the epoch. The request Offset will be advanced as-needed - * to skip persisted Fragments having a modication time before the bound. - * @format int64 - */ - beginModTime?: string; + /** Header is attached by a proxying broker peer. */ + header?: ProtocolHeader; + /** Journal to be read. */ + journal?: string; + /** + * Desired offset to begin reading from. Value -1 has special handling, where + * the read is performed from the current write head. All other positive + * values specify a desired exact byte offset to read from. If the offset is + * not available (eg, because it represents a portion of Journal which has + * been permanently deleted), the broker will return the next available + * offset. Callers should therefore always inspect the ReadResponse offset. + * @format int64 + */ + offset?: string; + /** + * Whether the operation should block until content becomes available. + * OFFSET_NOT_YET_AVAILABLE is returned if a non-blocking read has no ready + * content. + */ + block?: boolean; + /** + * If do_not_proxy is true, the broker will not proxy the read to another + * broker, or open and proxy a remote Fragment on the client's behalf. + */ + doNotProxy?: boolean; + /** + * If metadata_only is true, the broker will respond with Journal and + * Fragment metadata but not content. + */ + metadataOnly?: boolean; + /** + * Offset to read through. If zero, then the read end offset is unconstrained. + * @format int64 + */ + endOffset?: string; + /** + * BeginModTime is an optional inclusive lower bound on the modification + * timestamps of fragments read from the backing store, represented as + * seconds since the epoch. The request Offset will be advanced as-needed + * to skip persisted Fragments having a modication time before the bound. + * @format int64 + */ + beginModTime?: string; } /** @@ -528,38 +528,38 @@ hasn't yet committed). However once a metadata message is read, the reader is assured that its associated chunk messages are immediately forthcoming. */ export interface ProtocolReadResponse { - /** Status of the Read RPC. */ - status?: ProtocolStatus; - /** - * Header of the response. Accompanies the first ReadResponse of the response - * stream. - */ - header?: ProtocolHeader; - /** - * The effective offset of the read. See ReadRequest offset. - * @format int64 - */ - offset?: string; - /** - * The offset to next be written, by the next append transaction served by - * broker. In other words, the last offset through which content is - * available to be read from the Journal. This is a metadata field and will - * not be returned with a content response. - * @format int64 - */ - writeHead?: string; - /** - * Fragment to which the offset was mapped. This is a metadata field and will - * not be returned with a content response. - */ - fragment?: ProtocolFragment; - /** If Fragment is remote, a URL from which it may be directly read. */ - fragmentUrl?: string; - /** - * Content chunks of the read. - * @format byte - */ - content?: string; + /** Status of the Read RPC. */ + status?: ProtocolStatus; + /** + * Header of the response. Accompanies the first ReadResponse of the response + * stream. + */ + header?: ProtocolHeader; + /** + * The effective offset of the read. See ReadRequest offset. + * @format int64 + */ + offset?: string; + /** + * The offset to next be written, by the next append transaction served by + * broker. In other words, the last offset through which content is + * available to be read from the Journal. This is a metadata field and will + * not be returned with a content response. + * @format int64 + */ + writeHead?: string; + /** + * Fragment to which the offset was mapped. This is a metadata field and will + * not be returned with a content response. + */ + fragment?: ProtocolFragment; + /** If Fragment is remote, a URL from which it may be directly read. */ + fragmentUrl?: string; + /** + * Content chunks of the read. + * @format byte + */ + content?: string; } /** @@ -568,55 +568,55 @@ Replicate RPC. Each message is a 1:1 response to a previously read "proposal" ReplicateRequest with |acknowledge| set. */ export interface ProtocolReplicateResponse { - /** Status of the Replicate RPC. */ - status?: ProtocolStatus; - /** - * Header of the response. Accompanies the first ReplicateResponse of the - * response stream. - */ - header?: ProtocolHeader; - /** - * If status is PROPOSAL_MISMATCH, then |fragment| is the replica's current - * journal Fragment, and either it or |registers| will differ from the - * primary's proposal. - */ - fragment?: ProtocolFragment; - /** - * If status is PROPOSAL_MISMATCH, then |registers| are the replica's current - * journal registers. - */ - registers?: ProtocolLabelSet; + /** Status of the Replicate RPC. */ + status?: ProtocolStatus; + /** + * Header of the response. Accompanies the first ReplicateResponse of the + * response stream. + */ + header?: ProtocolHeader; + /** + * If status is PROPOSAL_MISMATCH, then |fragment| is the replica's current + * journal Fragment, and either it or |registers| will differ from the + * primary's proposal. + */ + fragment?: ProtocolFragment; + /** + * If status is PROPOSAL_MISMATCH, then |registers| are the replica's current + * journal registers. + */ + registers?: ProtocolLabelSet; } /** * Route captures the current topology of an item and the processes serving it. */ export interface ProtocolRoute { - /** Members of the Route, ordered on ascending ProcessSpec.ID (zone, suffix). */ - members?: ProcessSpecID[]; - /** - * Index of the ProcessSpec serving as primary within |members|, - * or -1 of no member is currently primary. - * @format int32 - */ - primary?: number; - /** - * Endpoints of each Route member. If not empty, |endpoints| has the same - * length and order as |members|, and captures the endpoint of each one. - */ - endpoints?: string[]; + /** Members of the Route, ordered on ascending ProcessSpec.ID (zone, suffix). */ + members?: ProcessSpecID[]; + /** + * Index of the ProcessSpec serving as primary within |members|, + * or -1 of no member is currently primary. + * @format int32 + */ + primary?: number; + /** + * Endpoints of each Route member. If not empty, |endpoints| has the same + * length and order as |members|, and captures the endpoint of each one. + */ + endpoints?: string[]; } /** * SHA1Sum is a 160-bit SHA1 digest. */ export interface ProtocolSHA1Sum { - /** @format uint64 */ - part1?: string; - /** @format uint64 */ - part2?: string; - /** @format int64 */ - part3?: number; + /** @format uint64 */ + part1?: string; + /** @format uint64 */ + part2?: string; + /** @format int64 */ + part3?: number; } /** @@ -650,35 +650,35 @@ too many broker or Etcd failures. request, but it was not matched by current register values of the journal. */ export type ProtocolStatus = - | 'OK' - | 'JOURNAL_NOT_FOUND' - | 'NO_JOURNAL_PRIMARY_BROKER' - | 'NOT_JOURNAL_PRIMARY_BROKER' - | 'NOT_JOURNAL_BROKER' - | 'INSUFFICIENT_JOURNAL_BROKERS' - | 'OFFSET_NOT_YET_AVAILABLE' - | 'WRONG_ROUTE' - | 'PROPOSAL_MISMATCH' - | 'ETCD_TRANSACTION_FAILED' - | 'NOT_ALLOWED' - | 'WRONG_APPEND_OFFSET' - | 'INDEX_HAS_GREATER_OFFSET' - | 'REGISTER_MISMATCH'; + | "OK" + | "JOURNAL_NOT_FOUND" + | "NO_JOURNAL_PRIMARY_BROKER" + | "NOT_JOURNAL_PRIMARY_BROKER" + | "NOT_JOURNAL_BROKER" + | "INSUFFICIENT_JOURNAL_BROKERS" + | "OFFSET_NOT_YET_AVAILABLE" + | "WRONG_ROUTE" + | "PROPOSAL_MISMATCH" + | "ETCD_TRANSACTION_FAILED" + | "NOT_ALLOWED" + | "WRONG_APPEND_OFFSET" + | "INDEX_HAS_GREATER_OFFSET" + | "REGISTER_MISMATCH"; export interface RuntimeError { - error?: string; - /** @format int32 */ - code?: number; - message?: string; - details?: ProtobufAny[]; + error?: string; + /** @format int32 */ + code?: number; + message?: string; + details?: ProtobufAny[]; } export interface RuntimeStreamError { - /** @format int32 */ - grpcCode?: number; - /** @format int32 */ - httpCode?: number; - message?: string; - httpStatus?: string; - details?: ProtobufAny[]; + /** @format int32 */ + grpcCode?: number; + /** @format int32 */ + httpCode?: number; + message?: string; + httpStatus?: string; + details?: ProtobufAny[]; } diff --git a/client/dist/src/gen/consumer/protocol/consumer.ts b/client/dist/src/gen/consumer/protocol/consumer.ts index e8c797e..5286064 100644 --- a/client/dist/src/gen/consumer/protocol/consumer.ts +++ b/client/dist/src/gen/consumer/protocol/consumer.ts @@ -10,8 +10,8 @@ */ export interface GetHintsResponseResponseHints { - /** If the hints value does not exist Hints will be nil. */ - hints?: RecoverylogFSMHints; + /** If the hints value does not exist Hints will be nil. */ + hints?: RecoverylogFSMHints; } /** @@ -25,76 +25,76 @@ different Etcd clusters). Etcd is kept in sync with etcdserverpb.ResponseHeader. */ export interface HeaderEtcd { - /** - * cluster_id is the ID of the cluster. - * @format uint64 - */ - clusterId?: string; - /** - * member_id is the ID of the member. - * @format uint64 - */ - memberId?: string; - /** - * revision is the Etcd key-value store revision when the request was - * applied. - * @format int64 - */ - revision?: string; - /** - * raft_term is the raft term when the request was applied. - * @format uint64 - */ - raftTerm?: string; + /** + * cluster_id is the ID of the cluster. + * @format uint64 + */ + clusterId?: string; + /** + * member_id is the ID of the member. + * @format uint64 + */ + memberId?: string; + /** + * revision is the Etcd key-value store revision when the request was + * applied. + * @format int64 + */ + revision?: string; + /** + * raft_term is the raft term when the request was applied. + * @format uint64 + */ + raftTerm?: string; } /** * Shards of the response. */ export interface ListResponseShard { - /** - * ShardSpec describes a shard and its configuration, and is the long-lived unit - * of work and scaling for a consumer application. Each shard is allocated to a - * one "primary" at-a-time selected from the current processes of a consumer - * application, and is re-assigned on process fault or exit. - * - * ShardSpecs describe all configuration of the shard and its processing, - * including journals to consume, configuration for processing transactions, its - * recovery log, hot standbys, etc. ShardSpecs may be further extended with - * domain-specific labels & values to further define application behavior. - * ShardSpec is-a allocator.ItemValue. - */ - spec?: ConsumerShardSpec; - /** - * Current ModRevision of the ShardSpec. - * @format int64 - */ - modRevision?: string; - /** Route of the shard, including endpoints. */ - route?: ProtocolRoute; - /** Status of each replica. Cardinality and ordering matches |route|. */ - status?: ConsumerReplicaStatus[]; + /** + * ShardSpec describes a shard and its configuration, and is the long-lived unit + * of work and scaling for a consumer application. Each shard is allocated to a + * one "primary" at-a-time selected from the current processes of a consumer + * application, and is re-assigned on process fault or exit. + * + * ShardSpecs describe all configuration of the shard and its processing, + * including journals to consume, configuration for processing transactions, its + * recovery log, hot standbys, etc. ShardSpecs may be further extended with + * domain-specific labels & values to further define application behavior. + * ShardSpec is-a allocator.ItemValue. + */ + spec?: ConsumerShardSpec; + /** + * Current ModRevision of the ShardSpec. + * @format int64 + */ + modRevision?: string; + /** Route of the shard, including endpoints. */ + route?: ProtocolRoute; + /** Status of each replica. Cardinality and ordering matches |route|. */ + status?: ConsumerReplicaStatus[]; } /** * ID composes a zone and a suffix to uniquely identify a ProcessSpec. */ export interface ProcessSpecID { - /** - * "Zone" in which the process is running. Zones may be AWS, Azure, or - * Google Cloud Platform zone identifiers, or rack locations within a colo, - * or given some other custom meaning. Gazette will replicate across - * multiple zones, and seeks to minimize traffic which must cross zones (for - * example, by proxying reads to a broker in the current zone). - */ - zone?: string; - /** - * Unique suffix of the process within |zone|. It is permissible for a - * suffix value to repeat across zones, but never within zones. In practice, - * it's recommended to use a FQDN, Kubernetes Pod name, or comparable unique - * and self-describing value as the ID suffix. - */ - suffix?: string; + /** + * "Zone" in which the process is running. Zones may be AWS, Azure, or + * Google Cloud Platform zone identifiers, or rack locations within a colo, + * or given some other custom meaning. Gazette will replicate across + * multiple zones, and seeks to minimize traffic which must cross zones (for + * example, by proxying reads to a broker in the current zone). + */ + zone?: string; + /** + * Unique suffix of the process within |zone|. It is permissible for a + * suffix value to repeat across zones, but never within zones. In practice, + * it's recommended to use a FQDN, Kubernetes Pod name, or comparable unique + * and self-describing value as the ID suffix. + */ + suffix?: string; } /** @@ -107,87 +107,87 @@ Shards not having recovery logs immediately transition to STANDBY. - PRIMARY: The replica is actively serving as primary. - FAILED: The replica has encountered an unrecoverable error. */ -export type ReplicaStatusCode = 'IDLE' | 'BACKFILL' | 'STANDBY' | 'PRIMARY' | 'FAILED'; +export type ReplicaStatusCode = "IDLE" | "BACKFILL" | "STANDBY" | "PRIMARY" | "FAILED"; /** * Change defines an insertion, update, or deletion to be applied to the set of ShardSpecs. Exactly one of |upsert| or |delete| must be set. */ export interface ConsumerApplyRequestChange { - /** - * Expected ModRevision of the current ShardSpec. If the shard is being - * created, expect_mod_revision is zero. - * @format int64 - */ - expectModRevision?: string; - /** - * ShardSpec to be updated (if expect_mod_revision > 0) or created - * (if expect_mod_revision == 0). - */ - upsert?: ConsumerShardSpec; - /** Shard to be deleted. expect_mod_revision must not be zero. */ - delete?: string; + /** + * Expected ModRevision of the current ShardSpec. If the shard is being + * created, expect_mod_revision is zero. + * @format int64 + */ + expectModRevision?: string; + /** + * ShardSpec to be updated (if expect_mod_revision > 0) or created + * (if expect_mod_revision == 0). + */ + upsert?: ConsumerShardSpec; + /** Shard to be deleted. expect_mod_revision must not be zero. */ + delete?: string; } export interface ConsumerApplyResponse { - /** Status of the Apply RPC. */ - status?: ConsumerStatus; - /** Header of the response. */ - header?: ProtocolHeader; - /** - * Optional extension of the ApplyResponse. - * @format byte - */ - extension?: string; + /** Status of the Apply RPC. */ + status?: ConsumerStatus; + /** Header of the response. */ + header?: ProtocolHeader; + /** + * Optional extension of the ApplyResponse. + * @format byte + */ + extension?: string; } export interface ConsumerGetHintsResponse { - /** Status of the Hints RPC. */ - status?: ConsumerStatus; - /** Header of the response. */ - header?: ProtocolHeader; - /** Primary hints for the shard. */ - primaryHints?: GetHintsResponseResponseHints; - /** - * List of backup hints for a shard. The most recent recovery log hints will - * be first, any subsequent hints are for historical backup. If there is no - * value for a hint key the value corresponding hints will be nil. - */ - backupHints?: GetHintsResponseResponseHints[]; - /** - * Optional extension of the GetHintsResponse. - * @format byte - */ - extension?: string; + /** Status of the Hints RPC. */ + status?: ConsumerStatus; + /** Header of the response. */ + header?: ProtocolHeader; + /** Primary hints for the shard. */ + primaryHints?: GetHintsResponseResponseHints; + /** + * List of backup hints for a shard. The most recent recovery log hints will + * be first, any subsequent hints are for historical backup. If there is no + * value for a hint key the value corresponding hints will be nil. + */ + backupHints?: GetHintsResponseResponseHints[]; + /** + * Optional extension of the GetHintsResponse. + * @format byte + */ + extension?: string; } export interface ConsumerListRequest { - /** - * Selector optionally refines the set of shards which will be enumerated. - * If zero-valued, all shards are returned. Otherwise, only ShardSpecs - * matching the LabelSelector will be returned. One meta-label "id" is - * additionally supported by the selector, where "id=example-shard-ID" - * will match a ShardSpec with ID "example-shard-ID". - */ - selector?: ProtocolLabelSelector; - /** - * Optional extension of the ListRequest. - * @format byte - */ - extension?: string; + /** + * Selector optionally refines the set of shards which will be enumerated. + * If zero-valued, all shards are returned. Otherwise, only ShardSpecs + * matching the LabelSelector will be returned. One meta-label "id" is + * additionally supported by the selector, where "id=example-shard-ID" + * will match a ShardSpec with ID "example-shard-ID". + */ + selector?: ProtocolLabelSelector; + /** + * Optional extension of the ListRequest. + * @format byte + */ + extension?: string; } export interface ConsumerListResponse { - /** Status of the List RPC. */ - status?: ConsumerStatus; - /** Header of the response. */ - header?: ProtocolHeader; - shards?: ListResponseShard[]; - /** - * Optional extension of the ListResponse. - * @format byte - */ - extension?: string; + /** Status of the List RPC. */ + status?: ConsumerStatus; + /** Header of the response. */ + header?: ProtocolHeader; + shards?: ListResponseShard[]; + /** + * Optional extension of the ListResponse. + * @format byte + */ + extension?: string; } /** @@ -199,19 +199,19 @@ of the replicas transitioned to FAILED, than the status is FAILED. This reduction behavior is used to summarize status across all replicas. */ export interface ConsumerReplicaStatus { - /** - * - BACKFILL: The replica is actively playing the historical recovery log. - * - STANDBY: The replica has finished playing the historical recovery log and is - * live-tailing it to locally mirror recorded operations as they are - * produced. It can take over as primary at any time. - * - * Shards not having recovery logs immediately transition to STANDBY. - * - PRIMARY: The replica is actively serving as primary. - * - FAILED: The replica has encountered an unrecoverable error. - */ - code?: ReplicaStatusCode; - /** Errors encountered during replica processing. Set iff |code| is FAILED. */ - errors?: string[]; + /** + * - BACKFILL: The replica is actively playing the historical recovery log. + * - STANDBY: The replica has finished playing the historical recovery log and is + * live-tailing it to locally mirror recorded operations as they are + * produced. It can take over as primary at any time. + * + * Shards not having recovery logs immediately transition to STANDBY. + * - PRIMARY: The replica is actively serving as primary. + * - FAILED: The replica has encountered an unrecoverable error. + */ + code?: ReplicaStatusCode; + /** Errors encountered during replica processing. Set iff |code| is FAILED. */ + errors?: string[]; } /** @@ -227,131 +227,131 @@ domain-specific labels & values to further define application behavior. ShardSpec is-a allocator.ItemValue. */ export interface ConsumerShardSpec { - /** ID of the shard. */ - id?: string; - /** Sources of the shard, uniquely ordered on Source journal. */ - sources?: ConsumerShardSpecSource[]; - /** - * Prefix of the Journal into which the shard's recovery log will be recorded. - * The complete Journal name is built as "{recovery_log_prefix}/{shard_id}". - * If empty, the shard does not use a recovery log. - */ - recoveryLogPrefix?: string; - /** - * Prefix of Etcd keys into which recovery log FSMHints are written to and - * read from. FSMHints allow readers of the recovery log to efficiently - * determine the minimum fragments of log which must be read to fully recover - * local store state. The complete hint key written by the shard primary is: - * "{hint_prefix}/{shard_id}.primary" - * - * The primary will regularly produce updated hints into this key, and - * players of the log will similarly utilize hints from this key. - * If |recovery_log_prefix| is set, |hint_prefix| must be also. - */ - hintPrefix?: string; - /** - * Backups of verified recovery log FSMHints, retained as a disaster-recovery - * mechanism. On completing playback, a player will write recovered hints to: - * "{hints_prefix}/{shard_id}.backup.0". - * - * It also move hints previously stored under - * "{hints_prefix/{shard_id}.backup.0" to - * "{hints_prefix/{shard_id}.backup.1", and so on, keeping at most - * |hint_backups| distinct sets of FSMHints. - * In the case of disaster or data-loss, these copied hints can be an - * important fallback for recovering a consistent albeit older version of the - * shard's store, with each relying on only progressively older portions of - * the recovery log. - * When pruning the recovery log, log fragments which are older than (and no - * longer required by) the *oldest* backup are discarded, ensuring that - * all hints remain valid for playback. - * @format int32 - */ - hintBackups?: number; - /** - * Max duration of shard transactions. This duration upper-bounds the amount - * of time during which a transaction may process messages before it must - * flush and commit. It may run for less time if an input message stall occurs - * (eg, no decoded journal message is ready without blocking). A typical value - * would be `1s`: applications which perform extensive aggregation over - * message streams exhibiting locality of "hot" keys may benefit from larger - * values. - */ - maxTxnDuration?: string; - /** - * Min duration of shard transactions. This duration lower-bounds the amount - * of time during which a transaction must process messages before it may - * flush and commit. It may run for more time if additional messages are - * available (eg, decoded journal messages are ready without blocking). Note - * also that transactions are pipelined: a current transaction may process - * messages while a prior transaction's recovery log writes flush to Gazette, - * but it cannot prepare to commit until the prior transaction writes - * complete. In other words even if |min_txn_quantum| is zero, some degree of - * message batching is expected due to the network delay inherent in Gazette - * writes. A typical value of would be `0s`: applications which perform - * extensive aggregation may benefit from larger values. - */ - minTxnDuration?: string; - /** Disable processing of the shard. */ - disable?: boolean; - /** - * Hot standbys is the desired number of consumer processes which should be - * replicating the primary consumer's recovery log. Standbys are allocated in - * a separate availability zone of the current primary, and tail the live log - * to continuously mirror the primary's on-disk DB file structure. Should the - * primary experience failure, one of the hot standbys will be assigned to - * take over as the new shard primary, which is accomplished by simply opening - * its local copy of the recovered store files. - * - * Note that under regular operation, shard hand-off is zero downtime even if - * standbys are zero, as the current primary will not cede ownership until the - * replacement process declares itself ready. However, without standbys a - * process failure will leave the shard without an active primary while its - * replacement starts and completes playback of its recovery log. - * @format int64 - */ - hotStandbys?: number; - /** - * User-defined Labels of this ShardSpec. The label "id" is reserved and may - * not be used with a ShardSpec's labels. - */ - labels?: ProtocolLabelSet; - /** - * Disable waiting for acknowledgements of pending message(s). - * - * If a consumer transaction reads uncommitted messages, it will by default - * remain open (subject to the max duration) awaiting an acknowledgement of - * those messages, in the hope that that acknowledgement will be quickly - * forthcoming and, by remaining open, we can process all messages in this - * transaction. Effectively we're trading a small amount of increased local - * latency for a global reduction in end-to-end latency. - * This works well for acyclic message flows, but can introduce unnecessary - * stalls if there are message cycles between shards. In the simplest case, - * a transaction could block awaiting an ACK of a message that it itself - * produced -- an ACK which can't arrive until the transaction closes. - */ - disableWaitForAck?: boolean; - /** - * Size of the ring buffer used to sequence read-uncommitted messages - * into consumed, read-committed ones. The ring buffer is a performance - * optimization only: applications will replay portions of journals as - * needed when messages aren't available in the buffer. - * It can remain small if source journal transactions are small, - * but larger transactions will achieve better performance with a - * larger ring. - * If zero, a reasonable default (currently 8192) is used. - * @format int64 - */ - ringBufferSize?: number; - /** - * Size of the channel used to bridge message read and decode with - * sequencing and consumption. Larger values may reduce data stalls, - * particularly for larger transactions and/or bursty custom - * MessageProducer implementations. - * If zero, a reasonable default (currently 8192) is used. - * @format int64 - */ - readChannelSize?: number; + /** ID of the shard. */ + id?: string; + /** Sources of the shard, uniquely ordered on Source journal. */ + sources?: ConsumerShardSpecSource[]; + /** + * Prefix of the Journal into which the shard's recovery log will be recorded. + * The complete Journal name is built as "{recovery_log_prefix}/{shard_id}". + * If empty, the shard does not use a recovery log. + */ + recoveryLogPrefix?: string; + /** + * Prefix of Etcd keys into which recovery log FSMHints are written to and + * read from. FSMHints allow readers of the recovery log to efficiently + * determine the minimum fragments of log which must be read to fully recover + * local store state. The complete hint key written by the shard primary is: + * "{hint_prefix}/{shard_id}.primary" + * + * The primary will regularly produce updated hints into this key, and + * players of the log will similarly utilize hints from this key. + * If |recovery_log_prefix| is set, |hint_prefix| must be also. + */ + hintPrefix?: string; + /** + * Backups of verified recovery log FSMHints, retained as a disaster-recovery + * mechanism. On completing playback, a player will write recovered hints to: + * "{hints_prefix}/{shard_id}.backup.0". + * + * It also move hints previously stored under + * "{hints_prefix/{shard_id}.backup.0" to + * "{hints_prefix/{shard_id}.backup.1", and so on, keeping at most + * |hint_backups| distinct sets of FSMHints. + * In the case of disaster or data-loss, these copied hints can be an + * important fallback for recovering a consistent albeit older version of the + * shard's store, with each relying on only progressively older portions of + * the recovery log. + * When pruning the recovery log, log fragments which are older than (and no + * longer required by) the *oldest* backup are discarded, ensuring that + * all hints remain valid for playback. + * @format int32 + */ + hintBackups?: number; + /** + * Max duration of shard transactions. This duration upper-bounds the amount + * of time during which a transaction may process messages before it must + * flush and commit. It may run for less time if an input message stall occurs + * (eg, no decoded journal message is ready without blocking). A typical value + * would be `1s`: applications which perform extensive aggregation over + * message streams exhibiting locality of "hot" keys may benefit from larger + * values. + */ + maxTxnDuration?: string; + /** + * Min duration of shard transactions. This duration lower-bounds the amount + * of time during which a transaction must process messages before it may + * flush and commit. It may run for more time if additional messages are + * available (eg, decoded journal messages are ready without blocking). Note + * also that transactions are pipelined: a current transaction may process + * messages while a prior transaction's recovery log writes flush to Gazette, + * but it cannot prepare to commit until the prior transaction writes + * complete. In other words even if |min_txn_quantum| is zero, some degree of + * message batching is expected due to the network delay inherent in Gazette + * writes. A typical value of would be `0s`: applications which perform + * extensive aggregation may benefit from larger values. + */ + minTxnDuration?: string; + /** Disable processing of the shard. */ + disable?: boolean; + /** + * Hot standbys is the desired number of consumer processes which should be + * replicating the primary consumer's recovery log. Standbys are allocated in + * a separate availability zone of the current primary, and tail the live log + * to continuously mirror the primary's on-disk DB file structure. Should the + * primary experience failure, one of the hot standbys will be assigned to + * take over as the new shard primary, which is accomplished by simply opening + * its local copy of the recovered store files. + * + * Note that under regular operation, shard hand-off is zero downtime even if + * standbys are zero, as the current primary will not cede ownership until the + * replacement process declares itself ready. However, without standbys a + * process failure will leave the shard without an active primary while its + * replacement starts and completes playback of its recovery log. + * @format int64 + */ + hotStandbys?: number; + /** + * User-defined Labels of this ShardSpec. The label "id" is reserved and may + * not be used with a ShardSpec's labels. + */ + labels?: ProtocolLabelSet; + /** + * Disable waiting for acknowledgements of pending message(s). + * + * If a consumer transaction reads uncommitted messages, it will by default + * remain open (subject to the max duration) awaiting an acknowledgement of + * those messages, in the hope that that acknowledgement will be quickly + * forthcoming and, by remaining open, we can process all messages in this + * transaction. Effectively we're trading a small amount of increased local + * latency for a global reduction in end-to-end latency. + * This works well for acyclic message flows, but can introduce unnecessary + * stalls if there are message cycles between shards. In the simplest case, + * a transaction could block awaiting an ACK of a message that it itself + * produced -- an ACK which can't arrive until the transaction closes. + */ + disableWaitForAck?: boolean; + /** + * Size of the ring buffer used to sequence read-uncommitted messages + * into consumed, read-committed ones. The ring buffer is a performance + * optimization only: applications will replay portions of journals as + * needed when messages aren't available in the buffer. + * It can remain small if source journal transactions are small, + * but larger transactions will achieve better performance with a + * larger ring. + * If zero, a reasonable default (currently 8192) is used. + * @format int64 + */ + ringBufferSize?: number; + /** + * Size of the channel used to bridge message read and decode with + * sequencing and consumption. Larger values may reduce data stalls, + * particularly for larger transactions and/or bursty custom + * MessageProducer implementations. + * If zero, a reasonable default (currently 8192) is used. + * @format int64 + */ + readChannelSize?: number; } /** @@ -372,65 +372,65 @@ events, read a comparably low-volume "purchase" event stream, and on each purchase publish the bundle of its corresponding prior product views. */ export interface ConsumerShardSpecSource { - /** Journal which this shard is consuming. */ - journal?: string; - /** - * Minimum journal byte offset the shard should begin reading from. - * Typically this should be zero, as read offsets are check-pointed and - * restored from the shard's Store as it processes. |min_offset| can be - * useful for shard initialization, directing it to skip over historical - * portions of the journal not needed for the application's use case. - * @format int64 - */ - minOffset?: string; + /** Journal which this shard is consuming. */ + journal?: string; + /** + * Minimum journal byte offset the shard should begin reading from. + * Typically this should be zero, as read offsets are check-pointed and + * restored from the shard's Store as it processes. |min_offset| can be + * useful for shard initialization, directing it to skip over historical + * portions of the journal not needed for the application's use case. + * @format int64 + */ + minOffset?: string; } export interface ConsumerStatRequest { - /** Header may be attached by a proxying consumer peer. */ - header?: ProtocolHeader; - /** Shard to Stat. */ - shard?: string; - /** - * Journals and offsets which must be reflected in a completed consumer - * transaction before Stat returns, blocking if required. Offsets of journals - * not read by this shard are ignored. - */ - readThrough?: Record; - /** - * Optional extension of the StatRequest. - * @format byte - */ - extension?: string; + /** Header may be attached by a proxying consumer peer. */ + header?: ProtocolHeader; + /** Shard to Stat. */ + shard?: string; + /** + * Journals and offsets which must be reflected in a completed consumer + * transaction before Stat returns, blocking if required. Offsets of journals + * not read by this shard are ignored. + */ + readThrough?: Record; + /** + * Optional extension of the StatRequest. + * @format byte + */ + extension?: string; } export interface ConsumerStatResponse { - /** Status of the Stat RPC. */ - status?: ConsumerStatus; - /** Header of the response. */ - header?: ProtocolHeader; - /** - * Journals and offsets read through by the most recent completed consumer - * transaction. - */ - readThrough?: Record; - /** - * Journals and offsets this shard has published through, including - * acknowledgements, as-of the most recent completed consumer transaction. - * - * Formally, if an acknowledged message A results in this shard publishing - * messages B, and A falls within |read_through|, then all messages B & their - * acknowledgements fall within |publish_at|. - * The composition of |read_through| and |publish_at| allow CQRS applications - * to provide read-your-writes consistency, even if written events pass - * through multiple intermediate consumers and arbitrary transformations - * before arriving at the materialized view which is ultimately queried. - */ - publishAt?: Record; - /** - * Optional extension of the StatResponse. - * @format byte - */ - extension?: string; + /** Status of the Stat RPC. */ + status?: ConsumerStatus; + /** Header of the response. */ + header?: ProtocolHeader; + /** + * Journals and offsets read through by the most recent completed consumer + * transaction. + */ + readThrough?: Record; + /** + * Journals and offsets this shard has published through, including + * acknowledgements, as-of the most recent completed consumer transaction. + * + * Formally, if an acknowledged message A results in this shard publishing + * messages B, and A falls within |read_through|, then all messages B & their + * acknowledgements fall within |publish_at|. + * The composition of |read_through| and |publish_at| allow CQRS applications + * to provide read-your-writes consistency, even if written events pass + * through multiple intermediate consumers and arbitrary transformations + * before arriving at the materialized view which is ultimately queried. + */ + publishAt?: Record; + /** + * Optional extension of the StatResponse. + * @format byte + */ + extension?: string; } /** @@ -452,24 +452,24 @@ For example, a Stat RPC will return SHARD_STOPPED if the StatRequest cannot be satisfied. */ export type ConsumerStatus = - | 'OK' - | 'SHARD_NOT_FOUND' - | 'NO_SHARD_PRIMARY' - | 'NOT_SHARD_PRIMARY' - | 'ETCD_TRANSACTION_FAILED' - | 'SHARD_STOPPED'; + | "OK" + | "SHARD_NOT_FOUND" + | "NO_SHARD_PRIMARY" + | "NOT_SHARD_PRIMARY" + | "ETCD_TRANSACTION_FAILED" + | "SHARD_STOPPED"; export interface ConsumerUnassignResponse { - /** Status of the Unassign RPC. */ - status?: ConsumerStatus; - /** Shards which had assignments removed. */ - shards?: string[]; + /** Status of the Unassign RPC. */ + status?: ConsumerStatus; + /** Shards which had assignments removed. */ + shards?: string[]; } export interface ProtobufAny { - typeUrl?: string; - /** @format byte */ - value?: string; + typeUrl?: string; + /** @format byte */ + value?: string; } /** @@ -477,28 +477,28 @@ export interface ProtobufAny { an RPC, and its effective Etcd state. */ export interface ProtocolHeader { - /** - * ID of the process responsible for request processing. May be empty iff - * Header is being used within a proxied request, and that request may be - * dispatched to any member of the Route. - */ - processId?: ProcessSpecID; - /** - * Route of processes specifically responsible for this RPC, or an empty Route - * if any process is capable of serving the RPC. - */ - route?: ProtocolRoute; - /** - * Etcd represents the effective Etcd MVCC state under which a Gazette broker - * is operating in its processing of requests and responses. Its inclusion - * allows brokers to reason about relative "happened before" Revision ordering - * of apparent routing conflicts in proxied or replicated requests, as well - * as enabling sanity checks over equality of Etcd ClusterId (and precluding, - * for example, split-brain scenarios where different brokers are backed by - * different Etcd clusters). Etcd is kept in sync with - * etcdserverpb.ResponseHeader. - */ - etcd?: HeaderEtcd; + /** + * ID of the process responsible for request processing. May be empty iff + * Header is being used within a proxied request, and that request may be + * dispatched to any member of the Route. + */ + processId?: ProcessSpecID; + /** + * Route of processes specifically responsible for this RPC, or an empty Route + * if any process is capable of serving the RPC. + */ + route?: ProtocolRoute; + /** + * Etcd represents the effective Etcd MVCC state under which a Gazette broker + * is operating in its processing of requests and responses. Its inclusion + * allows brokers to reason about relative "happened before" Revision ordering + * of apparent routing conflicts in proxied or replicated requests, as well + * as enabling sanity checks over equality of Etcd ClusterId (and precluding, + * for example, split-brain scenarios where different brokers are backed by + * different Etcd clusters). Etcd is kept in sync with + * etcdserverpb.ResponseHeader. + */ + etcd?: HeaderEtcd; } /** @@ -508,53 +508,53 @@ attributes which do not directly imply semantics to the core system, but are meaningful to users or for higher-level Gazette tools. */ export interface ProtocolLabel { - name?: string; - value?: string; + name?: string; + value?: string; } /** * LabelSelector defines a filter over LabelSets. */ export interface ProtocolLabelSelector { - /** - * Include is Labels which must be matched for a LabelSet to be selected. If - * empty, all Labels are included. An include Label with empty ("") value is - * matched by a Label of the same name having any value. - */ - include?: ProtocolLabelSet; - /** - * Exclude is Labels which cannot be matched for a LabelSet to be selected. If - * empty, no Labels are excluded. An exclude Label with empty ("") value - * excludes a Label of the same name having any value. - */ - exclude?: ProtocolLabelSet; + /** + * Include is Labels which must be matched for a LabelSet to be selected. If + * empty, all Labels are included. An include Label with empty ("") value is + * matched by a Label of the same name having any value. + */ + include?: ProtocolLabelSet; + /** + * Exclude is Labels which cannot be matched for a LabelSet to be selected. If + * empty, no Labels are excluded. An exclude Label with empty ("") value + * excludes a Label of the same name having any value. + */ + exclude?: ProtocolLabelSet; } /** * LabelSet is a collection of labels and their values. */ export interface ProtocolLabelSet { - /** Labels of the set. Instances must be unique and sorted over (Name, Value). */ - labels?: ProtocolLabel[]; + /** Labels of the set. Instances must be unique and sorted over (Name, Value). */ + labels?: ProtocolLabel[]; } /** * Route captures the current topology of an item and the processes serving it. */ export interface ProtocolRoute { - /** Members of the Route, ordered on ascending ProcessSpec.ID (zone, suffix). */ - members?: ProcessSpecID[]; - /** - * Index of the ProcessSpec serving as primary within |members|, - * or -1 of no member is currently primary. - * @format int32 - */ - primary?: number; - /** - * Endpoints of each Route member. If not empty, |endpoints| has the same - * length and order as |members|, and captures the endpoint of each one. - */ - endpoints?: string[]; + /** Members of the Route, ordered on ascending ProcessSpec.ID (zone, suffix). */ + members?: ProcessSpecID[]; + /** + * Index of the ProcessSpec serving as primary within |members|, + * or -1 of no member is currently primary. + * @format int32 + */ + primary?: number; + /** + * Endpoints of each Route member. If not empty, |endpoints| has the same + * length and order as |members|, and captures the endpoint of each one. + */ + endpoints?: string[]; } /** @@ -568,41 +568,41 @@ that of the Recorder producing the FSMHints. Next tag: 4. */ export interface RecoverylogFSMHints { - /** - * Log is the implied recovery log of any contained |live_nodes| Segments - * which omit a |log| value. This implied behavior is both for backward- - * compatibility (Segments didn't always have a |log| field) and also for - * compacting the representation in the common case of Segments mostly or - * entirely addressing a single log. - */ - log?: string; - /** Live Fnodes and their Segments as-of the generation of these FSMHints. */ - liveNodes?: RecoverylogFnodeSegments[]; - /** Property files and contents as-of the generation of these FSMHints. */ - properties?: RecoverylogProperty[]; + /** + * Log is the implied recovery log of any contained |live_nodes| Segments + * which omit a |log| value. This implied behavior is both for backward- + * compatibility (Segments didn't always have a |log| field) and also for + * compacting the representation in the common case of Segments mostly or + * entirely addressing a single log. + */ + log?: string; + /** Live Fnodes and their Segments as-of the generation of these FSMHints. */ + liveNodes?: RecoverylogFnodeSegments[]; + /** Property files and contents as-of the generation of these FSMHints. */ + properties?: RecoverylogProperty[]; } /** * FnodeSegments captures log Segments containing all RecordedOps of the Fnode. */ export interface RecoverylogFnodeSegments { - /** - * Fnode being hinted. - * @format int64 - */ - fnode?: string; - /** - * Segments of the Fnode in the log. Currently, FSM tracks only a single - * Segment per Fnode per Author & Log. A specific implication of this is that Fnodes - * modified over long periods of time will result in Segments spanning large - * chunks of the log. For best performance, Fnodes should be opened & written - * once, and then never be modified again (this is RocksDB's behavior). - * If supporting this case is desired, FSM will have to be a bit smarter about - * not extending Segments which gap over significant portions of the log - * (eg, there's a trade-off to make over size of the hinted manifest, vs - * savings incurred on playback by being able to skip portions of the log). - */ - segments?: RecoverylogSegment[]; + /** + * Fnode being hinted. + * @format int64 + */ + fnode?: string; + /** + * Segments of the Fnode in the log. Currently, FSM tracks only a single + * Segment per Fnode per Author & Log. A specific implication of this is that Fnodes + * modified over long periods of time will result in Segments spanning large + * chunks of the log. For best performance, Fnodes should be opened & written + * once, and then never be modified again (this is RocksDB's behavior). + * If supporting this case is desired, FSM will have to be a bit smarter about + * not extending Segments which gap over significant portions of the log + * (eg, there's a trade-off to make over size of the hinted manifest, vs + * savings incurred on playback by being able to skip portions of the log). + */ + segments?: RecoverylogSegment[]; } /** @@ -610,10 +610,10 @@ export interface RecoverylogFnodeSegments { outside of regular Fnode tracking. See FSM.Properties. */ export interface RecoverylogProperty { - /** Filesystem path of this property, relative to the common base directory. */ - path?: string; - /** Complete file content of this property. */ - content?: string; + /** Filesystem path of this property, relative to the common base directory. */ + path?: string; + /** Complete file content of this property. */ + content?: string; } /** @@ -629,50 +629,50 @@ recorded operations at their exact offsets. Next tag: 8. */ export interface RecoverylogSegment { - /** - * Author which wrote RecordedOps of this Segment. - * @format int64 - */ - author?: number; - /** - * First (lowest) sequence number of RecordedOps within this Segment. - * @format int64 - */ - firstSeqNo?: string; - /** - * First byte offset of the Segment, where |first_seq_no| is recorded. - * If this Segment was produced by a Recorder, this is guaranteed only to be a - * lower-bound (eg, a Player reading at this offset may encounter irrelevant - * operations prior to the RecordedOp indicated by the tuple - * (|author|, |first_seq_no|, |first_checksum|). If a Player produced the Segment, - * first_offset is exact. - * @format int64 - */ - firstOffset?: string; - /** - * Checksum of the RecordedOp having |first_seq_no|. - * @format int64 - */ - firstChecksum?: number; - /** - * Last (highest, inclusive) sequence number of RecordedOps within this Segment. - * @format int64 - */ - lastSeqNo?: string; - /** - * Last offset (exclusive) of the Segment. Zero means the offset is not known - * (eg, because the Segment was produced by a Recorder). - * @format int64 - */ - lastOffset?: string; - /** Log is the Journal holding this Segment's data, and to which offsets are relative. */ - log?: string; + /** + * Author which wrote RecordedOps of this Segment. + * @format int64 + */ + author?: number; + /** + * First (lowest) sequence number of RecordedOps within this Segment. + * @format int64 + */ + firstSeqNo?: string; + /** + * First byte offset of the Segment, where |first_seq_no| is recorded. + * If this Segment was produced by a Recorder, this is guaranteed only to be a + * lower-bound (eg, a Player reading at this offset may encounter irrelevant + * operations prior to the RecordedOp indicated by the tuple + * (|author|, |first_seq_no|, |first_checksum|). If a Player produced the Segment, + * first_offset is exact. + * @format int64 + */ + firstOffset?: string; + /** + * Checksum of the RecordedOp having |first_seq_no|. + * @format int64 + */ + firstChecksum?: number; + /** + * Last (highest, inclusive) sequence number of RecordedOps within this Segment. + * @format int64 + */ + lastSeqNo?: string; + /** + * Last offset (exclusive) of the Segment. Zero means the offset is not known + * (eg, because the Segment was produced by a Recorder). + * @format int64 + */ + lastOffset?: string; + /** Log is the Journal holding this Segment's data, and to which offsets are relative. */ + log?: string; } export interface RuntimeError { - error?: string; - /** @format int32 */ - code?: number; - message?: string; - details?: ProtobufAny[]; + error?: string; + /** @format int32 */ + code?: number; + message?: string; + details?: ProtobufAny[]; } diff --git a/client/dist/types/gen/broker/protocol/broker.d.ts b/client/dist/types/gen/broker/protocol/broker.d.ts index 66d4208..1568452 100644 --- a/client/dist/types/gen/broker/protocol/broker.d.ts +++ b/client/dist/types/gen/broker/protocol/broker.d.ts @@ -165,7 +165,7 @@ improvements, it is an advanced configuration and the "Content-Encoding" header handling can be subtle and sometimes confusing. It uses the default suffix ".gzod". */ -export declare type ProtocolCompressionCodec = 'INVALID' | 'NONE' | 'GZIP' | 'ZSTANDARD' | 'SNAPPY' | 'GZIP_OFFLOAD_DECOMPRESSION'; +export declare type ProtocolCompressionCodec = "INVALID" | "NONE" | "GZIP" | "ZSTANDARD" | "SNAPPY" | "GZIP_OFFLOAD_DECOMPRESSION"; /** * Fragment is a content-addressed description of a contiguous Journal span, defined by the [begin, end) offset range covered by the Fragment and the @@ -608,7 +608,7 @@ too many broker or Etcd failures. - REGISTER_MISMATCH: The Append is refused because a registers selector was provided with the request, but it was not matched by current register values of the journal. */ -export declare type ProtocolStatus = 'OK' | 'JOURNAL_NOT_FOUND' | 'NO_JOURNAL_PRIMARY_BROKER' | 'NOT_JOURNAL_PRIMARY_BROKER' | 'NOT_JOURNAL_BROKER' | 'INSUFFICIENT_JOURNAL_BROKERS' | 'OFFSET_NOT_YET_AVAILABLE' | 'WRONG_ROUTE' | 'PROPOSAL_MISMATCH' | 'ETCD_TRANSACTION_FAILED' | 'NOT_ALLOWED' | 'WRONG_APPEND_OFFSET' | 'INDEX_HAS_GREATER_OFFSET' | 'REGISTER_MISMATCH'; +export declare type ProtocolStatus = "OK" | "JOURNAL_NOT_FOUND" | "NO_JOURNAL_PRIMARY_BROKER" | "NOT_JOURNAL_PRIMARY_BROKER" | "NOT_JOURNAL_BROKER" | "INSUFFICIENT_JOURNAL_BROKERS" | "OFFSET_NOT_YET_AVAILABLE" | "WRONG_ROUTE" | "PROPOSAL_MISMATCH" | "ETCD_TRANSACTION_FAILED" | "NOT_ALLOWED" | "WRONG_APPEND_OFFSET" | "INDEX_HAS_GREATER_OFFSET" | "REGISTER_MISMATCH"; export interface RuntimeError { error?: string; /** @format int32 */ diff --git a/client/dist/types/gen/consumer/protocol/consumer.d.ts b/client/dist/types/gen/consumer/protocol/consumer.d.ts index 59f5665..2de4163 100644 --- a/client/dist/types/gen/consumer/protocol/consumer.d.ts +++ b/client/dist/types/gen/consumer/protocol/consumer.d.ts @@ -92,7 +92,7 @@ Shards not having recovery logs immediately transition to STANDBY. - PRIMARY: The replica is actively serving as primary. - FAILED: The replica has encountered an unrecoverable error. */ -export declare type ReplicaStatusCode = 'IDLE' | 'BACKFILL' | 'STANDBY' | 'PRIMARY' | 'FAILED'; +export declare type ReplicaStatusCode = "IDLE" | "BACKFILL" | "STANDBY" | "PRIMARY" | "FAILED"; /** * Change defines an insertion, update, or deletion to be applied to the set of ShardSpecs. Exactly one of |upsert| or |delete| must be set. @@ -425,7 +425,7 @@ requested operation. For example, a Stat RPC will return SHARD_STOPPED if the StatRequest cannot be satisfied. */ -export declare type ConsumerStatus = 'OK' | 'SHARD_NOT_FOUND' | 'NO_SHARD_PRIMARY' | 'NOT_SHARD_PRIMARY' | 'ETCD_TRANSACTION_FAILED' | 'SHARD_STOPPED'; +export declare type ConsumerStatus = "OK" | "SHARD_NOT_FOUND" | "NO_SHARD_PRIMARY" | "NOT_SHARD_PRIMARY" | "ETCD_TRANSACTION_FAILED" | "SHARD_STOPPED"; export interface ConsumerUnassignResponse { /** Status of the Unassign RPC. */ status?: ConsumerStatus; diff --git a/client/src/gen/broker/protocol/broker.ts b/client/src/gen/broker/protocol/broker.ts index 964b6ea..555d57f 100644 --- a/client/src/gen/broker/protocol/broker.ts +++ b/client/src/gen/broker/protocol/broker.ts @@ -14,37 +14,37 @@ of JournalSpecs. Exactly one of |upsert| or |delete| must be set. */ export interface ApplyRequestChange { - /** - * Expected ModRevision of the current JournalSpec. If the Journal is being - * created, expect_mod_revision is zero. - * @format int64 - */ - expectModRevision?: string; - /** - * JournalSpec to be updated (if expect_mod_revision > 0) or created - * (if expect_mod_revision == 0). - */ - upsert?: ProtocolJournalSpec; - /** Journal to be deleted. expect_mod_revision must not be zero. */ - delete?: string; + /** + * Expected ModRevision of the current JournalSpec. If the Journal is being + * created, expect_mod_revision is zero. + * @format int64 + */ + expectModRevision?: string; + /** + * JournalSpec to be updated (if expect_mod_revision > 0) or created + * (if expect_mod_revision == 0). + */ + upsert?: ProtocolJournalSpec; + /** Journal to be deleted. expect_mod_revision must not be zero. */ + delete?: string; } /** * Fragments of the Response. */ export interface FragmentsResponseFragment { - /** - * Fragment is a content-addressed description of a contiguous Journal span, - * defined by the [begin, end) offset range covered by the Fragment and the - * SHA1 sum of the corresponding Journal content. - */ - spec?: ProtocolFragment; - /** - * SignedURL is a temporary URL at which a direct GET of the Fragment may - * be issued, signed by the broker's credentials. Set only if the request - * specified a SignatureTTL. - */ - signedUrl?: string; + /** + * Fragment is a content-addressed description of a contiguous Journal span, + * defined by the [begin, end) offset range covered by the Fragment and the + * SHA1 sum of the corresponding Journal content. + */ + spec?: ProtocolFragment; + /** + * SignedURL is a temporary URL at which a direct GET of the Fragment may + * be issued, signed by the broker's credentials. Set only if the request + * specified a SignatureTTL. + */ + signedUrl?: string; } /** @@ -58,106 +58,106 @@ different Etcd clusters). Etcd is kept in sync with etcdserverpb.ResponseHeader. */ export interface HeaderEtcd { - /** - * cluster_id is the ID of the cluster. - * @format uint64 - */ - clusterId?: string; - /** - * member_id is the ID of the member. - * @format uint64 - */ - memberId?: string; - /** - * revision is the Etcd key-value store revision when the request was - * applied. - * @format int64 - */ - revision?: string; - /** - * raft_term is the raft term when the request was applied. - * @format uint64 - */ - raftTerm?: string; + /** + * cluster_id is the ID of the cluster. + * @format uint64 + */ + clusterId?: string; + /** + * member_id is the ID of the member. + * @format uint64 + */ + memberId?: string; + /** + * revision is the Etcd key-value store revision when the request was + * applied. + * @format int64 + */ + revision?: string; + /** + * raft_term is the raft term when the request was applied. + * @format uint64 + */ + raftTerm?: string; } /** * Journals of the response. */ export interface ListResponseJournal { - /** JournalSpec describes a Journal and its configuration. */ - spec?: ProtocolJournalSpec; - /** - * Current ModRevision of the JournalSpec. - * @format int64 - */ - modRevision?: string; - /** Route of the journal, including endpoints. */ - route?: ProtocolRoute; + /** JournalSpec describes a Journal and its configuration. */ + spec?: ProtocolJournalSpec; + /** + * Current ModRevision of the JournalSpec. + * @format int64 + */ + modRevision?: string; + /** Route of the journal, including endpoints. */ + route?: ProtocolRoute; } /** * ID composes a zone and a suffix to uniquely identify a ProcessSpec. */ export interface ProcessSpecID { - /** - * "Zone" in which the process is running. Zones may be AWS, Azure, or - * Google Cloud Platform zone identifiers, or rack locations within a colo, - * or given some other custom meaning. Gazette will replicate across - * multiple zones, and seeks to minimize traffic which must cross zones (for - * example, by proxying reads to a broker in the current zone). - */ - zone?: string; - /** - * Unique suffix of the process within |zone|. It is permissible for a - * suffix value to repeat across zones, but never within zones. In practice, - * it's recommended to use a FQDN, Kubernetes Pod name, or comparable unique - * and self-describing value as the ID suffix. - */ - suffix?: string; + /** + * "Zone" in which the process is running. Zones may be AWS, Azure, or + * Google Cloud Platform zone identifiers, or rack locations within a colo, + * or given some other custom meaning. Gazette will replicate across + * multiple zones, and seeks to minimize traffic which must cross zones (for + * example, by proxying reads to a broker in the current zone). + */ + zone?: string; + /** + * Unique suffix of the process within |zone|. It is permissible for a + * suffix value to repeat across zones, but never within zones. In practice, + * it's recommended to use a FQDN, Kubernetes Pod name, or comparable unique + * and self-describing value as the ID suffix. + */ + suffix?: string; } export interface ProtobufAny { - typeUrl?: string; - /** @format byte */ - value?: string; + typeUrl?: string; + /** @format byte */ + value?: string; } /** * AppendResponse is the unary response message of the broker Append RPC. */ export interface ProtocolAppendResponse { - /** Status of the Append RPC. */ - status?: ProtocolStatus; - /** Header of the response. */ - header?: ProtocolHeader; - /** - * If status is OK, then |commit| is the Fragment which places the - * committed Append content within the Journal. - */ - commit?: ProtocolFragment; - /** Current registers of the journal. */ - registers?: ProtocolLabelSet; - /** - * Total number of RPC content chunks processed in this append. - * @format int64 - */ - totalChunks?: string; - /** - * Number of content chunks which were delayed by journal flow control. - * @format int64 - */ - delayedChunks?: string; + /** Status of the Append RPC. */ + status?: ProtocolStatus; + /** Header of the response. */ + header?: ProtocolHeader; + /** + * If status is OK, then |commit| is the Fragment which places the + * committed Append content within the Journal. + */ + commit?: ProtocolFragment; + /** Current registers of the journal. */ + registers?: ProtocolLabelSet; + /** + * Total number of RPC content chunks processed in this append. + * @format int64 + */ + totalChunks?: string; + /** + * Number of content chunks which were delayed by journal flow control. + * @format int64 + */ + delayedChunks?: string; } /** * ApplyResponse is the unary response message of the broker Apply RPC. */ export interface ProtocolApplyResponse { - /** Status of the Apply RPC. */ - status?: ProtocolStatus; - /** Header of the response. */ - header?: ProtocolHeader; + /** Status of the Apply RPC. */ + status?: ProtocolStatus; + /** Header of the response. */ + header?: ProtocolHeader; } /** @@ -185,12 +185,12 @@ header handling can be subtle and sometimes confusing. It uses the default suffix ".gzod". */ export type ProtocolCompressionCodec = - | 'INVALID' - | 'NONE' - | 'GZIP' - | 'ZSTANDARD' - | 'SNAPPY' - | 'GZIP_OFFLOAD_DECOMPRESSION'; + | "INVALID" + | "NONE" + | "GZIP" + | "ZSTANDARD" + | "SNAPPY" + | "GZIP_OFFLOAD_DECOMPRESSION"; /** * Fragment is a content-addressed description of a contiguous Journal span, @@ -198,37 +198,37 @@ defined by the [begin, end) offset range covered by the Fragment and the SHA1 sum of the corresponding Journal content. */ export interface ProtocolFragment { - /** Journal of the Fragment. */ - journal?: string; - /** - * Begin (inclusive) and end (exclusive) offset of the Fragment within the - * Journal. - * @format int64 - */ - begin?: string; - /** @format int64 */ - end?: string; - /** SHA1 sum of the Fragment's content. */ - sum?: ProtocolSHA1Sum; - /** Codec with which the Fragment's content is compressed. */ - compressionCodec?: ProtocolCompressionCodec; - /** - * Fragment store which backs the Fragment. Empty if the Fragment has yet to - * be persisted and is still local to a Broker. - */ - backingStore?: string; - /** - * Modification timestamp of the Fragment within the backing store, - * represented as seconds since the epoch. - * @format int64 - */ - modTime?: string; - /** - * Path postfix under which the fragment is persisted to the store. - * The complete Fragment store path is built from any path components of the - * backing store, followed by the journal name, followed by the path postfix. - */ - pathPostfix?: string; + /** Journal of the Fragment. */ + journal?: string; + /** + * Begin (inclusive) and end (exclusive) offset of the Fragment within the + * Journal. + * @format int64 + */ + begin?: string; + /** @format int64 */ + end?: string; + /** SHA1 sum of the Fragment's content. */ + sum?: ProtocolSHA1Sum; + /** Codec with which the Fragment's content is compressed. */ + compressionCodec?: ProtocolCompressionCodec; + /** + * Fragment store which backs the Fragment. Empty if the Fragment has yet to + * be persisted and is still local to a Broker. + */ + backingStore?: string; + /** + * Modification timestamp of the Fragment within the backing store, + * represented as seconds since the epoch. + * @format int64 + */ + modTime?: string; + /** + * Path postfix under which the fragment is persisted to the store. + * The complete Fragment store path is built from any path components of the + * backing store, followed by the journal name, followed by the path postfix. + */ + pathPostfix?: string; } /** @@ -236,18 +236,18 @@ export interface ProtocolFragment { RPC. */ export interface ProtocolFragmentsResponse { - /** Status of the Apply RPC. */ - status?: ProtocolStatus; - /** Header of the response. */ - header?: ProtocolHeader; - fragments?: FragmentsResponseFragment[]; - /** - * The NextPageToke value to be returned on subsequent Fragments requests. If - * the value is zero then there are no more fragments to be returned for this - * page. - * @format int64 - */ - nextPageToken?: string; + /** Status of the Apply RPC. */ + status?: ProtocolStatus; + /** Header of the response. */ + header?: ProtocolHeader; + fragments?: FragmentsResponseFragment[]; + /** + * The NextPageToke value to be returned on subsequent Fragments requests. If + * the value is zero then there are no more fragments to be returned for this + * page. + * @format int64 + */ + nextPageToken?: string; } /** @@ -255,67 +255,67 @@ export interface ProtocolFragmentsResponse { an RPC, and its effective Etcd state. */ export interface ProtocolHeader { - /** - * ID of the process responsible for request processing. May be empty iff - * Header is being used within a proxied request, and that request may be - * dispatched to any member of the Route. - */ - processId?: ProcessSpecID; - /** - * Route of processes specifically responsible for this RPC, or an empty Route - * if any process is capable of serving the RPC. - */ - route?: ProtocolRoute; - /** - * Etcd represents the effective Etcd MVCC state under which a Gazette broker - * is operating in its processing of requests and responses. Its inclusion - * allows brokers to reason about relative "happened before" Revision ordering - * of apparent routing conflicts in proxied or replicated requests, as well - * as enabling sanity checks over equality of Etcd ClusterId (and precluding, - * for example, split-brain scenarios where different brokers are backed by - * different Etcd clusters). Etcd is kept in sync with - * etcdserverpb.ResponseHeader. - */ - etcd?: HeaderEtcd; + /** + * ID of the process responsible for request processing. May be empty iff + * Header is being used within a proxied request, and that request may be + * dispatched to any member of the Route. + */ + processId?: ProcessSpecID; + /** + * Route of processes specifically responsible for this RPC, or an empty Route + * if any process is capable of serving the RPC. + */ + route?: ProtocolRoute; + /** + * Etcd represents the effective Etcd MVCC state under which a Gazette broker + * is operating in its processing of requests and responses. Its inclusion + * allows brokers to reason about relative "happened before" Revision ordering + * of apparent routing conflicts in proxied or replicated requests, as well + * as enabling sanity checks over equality of Etcd ClusterId (and precluding, + * for example, split-brain scenarios where different brokers are backed by + * different Etcd clusters). Etcd is kept in sync with + * etcdserverpb.ResponseHeader. + */ + etcd?: HeaderEtcd; } /** * JournalSpec describes a Journal and its configuration. */ export interface ProtocolJournalSpec { - /** Name of the Journal. */ - name?: string; - /** - * Desired replication of this Journal. This defines the Journal's tolerance - * to broker failures before data loss can occur (eg, a replication factor - * of three means two failures are tolerated). - * @format int32 - */ - replication?: number; - /** - * User-defined Labels of this JournalSpec. Two label names are reserved - * and may not be used within a JournalSpec's Labels: "name" and "prefix". - */ - labels?: ProtocolLabelSet; - /** - * Fragment is JournalSpec configuration which pertains to the creation, - * persistence, and indexing of the Journal's Fragments. - */ - fragment?: ProtocolJournalSpecFragment; - /** - * Flags of the Journal, as a combination of Flag enum values. The Flag enum - * is not used directly, as protobuf enums do not allow for or'ed bitfields. - * @format int64 - */ - flags?: number; - /** - * Maximum rate, in bytes-per-second, at which appends of this journal will - * be processed. If zero (the default), no rate limiting is applied. A global - * rate limit still may be in effect, in which case the effective rate is the - * smaller of the journal vs global rate. - * @format int64 - */ - maxAppendRate?: string; + /** Name of the Journal. */ + name?: string; + /** + * Desired replication of this Journal. This defines the Journal's tolerance + * to broker failures before data loss can occur (eg, a replication factor + * of three means two failures are tolerated). + * @format int32 + */ + replication?: number; + /** + * User-defined Labels of this JournalSpec. Two label names are reserved + * and may not be used within a JournalSpec's Labels: "name" and "prefix". + */ + labels?: ProtocolLabelSet; + /** + * Fragment is JournalSpec configuration which pertains to the creation, + * persistence, and indexing of the Journal's Fragments. + */ + fragment?: ProtocolJournalSpecFragment; + /** + * Flags of the Journal, as a combination of Flag enum values. The Flag enum + * is not used directly, as protobuf enums do not allow for or'ed bitfields. + * @format int64 + */ + flags?: number; + /** + * Maximum rate, in bytes-per-second, at which appends of this journal will + * be processed. If zero (the default), no rate limiting is applied. A global + * rate limit still may be in effect, in which case the effective rate is the + * smaller of the journal vs global rate. + * @format int64 + */ + maxAppendRate?: string; } /** @@ -323,80 +323,80 @@ export interface ProtocolJournalSpec { persistence, and indexing of the Journal's Fragments. */ export interface ProtocolJournalSpecFragment { - /** - * Target content length of each Fragment. In normal operation after - * Fragments reach at least this length, they will be closed and new ones - * begun. Note lengths may be smaller at times (eg, due to changes in - * Journal routing topology). Content length differs from Fragment file - * size, in that the former reflects uncompressed bytes. - * @format int64 - */ - length?: string; - /** Codec used to compress Journal Fragments. */ - compressionCodec?: ProtocolCompressionCodec; - /** - * Storage backend base path for this Journal's Fragments. Must be in URL - * form, with the choice of backend defined by the scheme. The full path of - * a Journal's Fragment is derived by joining the store path with the - * Fragment's ContentPath. Eg, given a fragment_store of - * "s3://My-AWS-bucket/a/prefix" and a JournalSpec of name "my/journal", - * a complete Fragment path might be: - * "s3://My-AWS-bucket/a/prefix/my/journal/000123-000456-789abcdef.gzip - * Multiple stores may be specified, in which case the Journal's Fragments - * are the union of all Fragments present across all stores, and new - * Fragments always persist to the first specified store. This can be - * helpful in performing incremental migrations, where new Journal content - * is written to the new store, while content in the old store remains - * available (and, depending on fragment_retention or recovery log pruning, - * may eventually be removed). - * - * If no stores are specified, the Journal is still use-able but will - * not persist Fragments to any a backing fragment store. This allows for - * real-time streaming use cases where reads of historical data are not - * needed. - */ - stores?: string[]; - /** - * Interval of time between refreshes of remote Fragment listings from - * configured fragment_stores. - */ - refreshInterval?: string; - /** - * Retention duration for historical Fragments of this Journal within the - * Fragment stores. If less than or equal to zero, Fragments are retained - * indefinitely. - */ - retention?: string; - /** - * Flush interval defines a uniform UTC time segment which, when passed, - * will prompt brokers to close and persist a fragment presently being - * written. - * - * Flush interval may be helpful in integrating the journal with a regularly - * scheduled batch work-flow which processes new files from the fragment - * store and has no particular awareness of Gazette. For example, setting - * flush_interval to 3600s will cause brokers to persist their present - * fragment on the hour, every hour, even if it has not yet reached its - * target length. A batch work-flow running at 5 minutes past the hour is - * then reasonably assured of seeing all events from the past hour. - * See also "gazctl journals fragments --help" for more discussion. - */ - flushInterval?: string; - /** - * Path postfix template is a Go template which evaluates to a partial - * path under which fragments are persisted to the store. A complete - * fragment path is constructed by appending path components from the - * fragment store, then the journal name, and then the postfix template. - * Path post-fixes can help in maintaining Hive compatible partitioning - * over fragment creation time. The fields ".Spool" and ".JournalSpec" - * are available for introspection in the template. For example, - * to partition on the UTC date and hour of creation, use: - * date={{ .Spool.FirstAppendTime.Format "2006-01-02" }}/hour={{ - * .Spool.FirstAppendTime.Format "15" }} - * - * Which will produce a path postfix like "date=2019-11-19/hour=22". - */ - pathPostfixTemplate?: string; + /** + * Target content length of each Fragment. In normal operation after + * Fragments reach at least this length, they will be closed and new ones + * begun. Note lengths may be smaller at times (eg, due to changes in + * Journal routing topology). Content length differs from Fragment file + * size, in that the former reflects uncompressed bytes. + * @format int64 + */ + length?: string; + /** Codec used to compress Journal Fragments. */ + compressionCodec?: ProtocolCompressionCodec; + /** + * Storage backend base path for this Journal's Fragments. Must be in URL + * form, with the choice of backend defined by the scheme. The full path of + * a Journal's Fragment is derived by joining the store path with the + * Fragment's ContentPath. Eg, given a fragment_store of + * "s3://My-AWS-bucket/a/prefix" and a JournalSpec of name "my/journal", + * a complete Fragment path might be: + * "s3://My-AWS-bucket/a/prefix/my/journal/000123-000456-789abcdef.gzip + * Multiple stores may be specified, in which case the Journal's Fragments + * are the union of all Fragments present across all stores, and new + * Fragments always persist to the first specified store. This can be + * helpful in performing incremental migrations, where new Journal content + * is written to the new store, while content in the old store remains + * available (and, depending on fragment_retention or recovery log pruning, + * may eventually be removed). + * + * If no stores are specified, the Journal is still use-able but will + * not persist Fragments to any a backing fragment store. This allows for + * real-time streaming use cases where reads of historical data are not + * needed. + */ + stores?: string[]; + /** + * Interval of time between refreshes of remote Fragment listings from + * configured fragment_stores. + */ + refreshInterval?: string; + /** + * Retention duration for historical Fragments of this Journal within the + * Fragment stores. If less than or equal to zero, Fragments are retained + * indefinitely. + */ + retention?: string; + /** + * Flush interval defines a uniform UTC time segment which, when passed, + * will prompt brokers to close and persist a fragment presently being + * written. + * + * Flush interval may be helpful in integrating the journal with a regularly + * scheduled batch work-flow which processes new files from the fragment + * store and has no particular awareness of Gazette. For example, setting + * flush_interval to 3600s will cause brokers to persist their present + * fragment on the hour, every hour, even if it has not yet reached its + * target length. A batch work-flow running at 5 minutes past the hour is + * then reasonably assured of seeing all events from the past hour. + * See also "gazctl journals fragments --help" for more discussion. + */ + flushInterval?: string; + /** + * Path postfix template is a Go template which evaluates to a partial + * path under which fragments are persisted to the store. A complete + * fragment path is constructed by appending path components from the + * fragment store, then the journal name, and then the postfix template. + * Path post-fixes can help in maintaining Hive compatible partitioning + * over fragment creation time. The fields ".Spool" and ".JournalSpec" + * are available for introspection in the template. For example, + * to partition on the UTC date and hour of creation, use: + * date={{ .Spool.FirstAppendTime.Format "2006-01-02" }}/hour={{ + * .Spool.FirstAppendTime.Format "15" }} + * + * Which will produce a path postfix like "date=2019-11-19/hour=22". + */ + pathPostfixTemplate?: string; } /** @@ -406,111 +406,111 @@ attributes which do not directly imply semantics to the core system, but are meaningful to users or for higher-level Gazette tools. */ export interface ProtocolLabel { - name?: string; - value?: string; + name?: string; + value?: string; } /** * LabelSelector defines a filter over LabelSets. */ export interface ProtocolLabelSelector { - /** - * Include is Labels which must be matched for a LabelSet to be selected. If - * empty, all Labels are included. An include Label with empty ("") value is - * matched by a Label of the same name having any value. - */ - include?: ProtocolLabelSet; - /** - * Exclude is Labels which cannot be matched for a LabelSet to be selected. If - * empty, no Labels are excluded. An exclude Label with empty ("") value - * excludes a Label of the same name having any value. - */ - exclude?: ProtocolLabelSet; + /** + * Include is Labels which must be matched for a LabelSet to be selected. If + * empty, all Labels are included. An include Label with empty ("") value is + * matched by a Label of the same name having any value. + */ + include?: ProtocolLabelSet; + /** + * Exclude is Labels which cannot be matched for a LabelSet to be selected. If + * empty, no Labels are excluded. An exclude Label with empty ("") value + * excludes a Label of the same name having any value. + */ + exclude?: ProtocolLabelSet; } /** * LabelSet is a collection of labels and their values. */ export interface ProtocolLabelSet { - /** Labels of the set. Instances must be unique and sorted over (Name, Value). */ - labels?: ProtocolLabel[]; + /** Labels of the set. Instances must be unique and sorted over (Name, Value). */ + labels?: ProtocolLabel[]; } /** * ListRequest is the unary request message of the broker List RPC. */ export interface ProtocolListRequest { - /** - * Selector optionally refines the set of journals which will be enumerated. - * If zero-valued, all journals are returned. Otherwise, only JournalSpecs - * matching the LabelSelector will be returned. Two meta-labels "name" and - * "prefix" are additionally supported by the selector, where: - * * name=examples/a-name will match a JournalSpec with Name - * "examples/a-name" - * * prefix=examples/ will match any JournalSpec having prefix "examples/". - * The prefix Label value must end in '/'. - */ - selector?: ProtocolLabelSelector; + /** + * Selector optionally refines the set of journals which will be enumerated. + * If zero-valued, all journals are returned. Otherwise, only JournalSpecs + * matching the LabelSelector will be returned. Two meta-labels "name" and + * "prefix" are additionally supported by the selector, where: + * * name=examples/a-name will match a JournalSpec with Name + * "examples/a-name" + * * prefix=examples/ will match any JournalSpec having prefix "examples/". + * The prefix Label value must end in '/'. + */ + selector?: ProtocolLabelSelector; } /** * ListResponse is the unary response message of the broker List RPC. */ export interface ProtocolListResponse { - /** Status of the List RPC. */ - status?: ProtocolStatus; - /** Header of the response. */ - header?: ProtocolHeader; - journals?: ListResponseJournal[]; + /** Status of the List RPC. */ + status?: ProtocolStatus; + /** Header of the response. */ + header?: ProtocolHeader; + journals?: ListResponseJournal[]; } /** * ReadRequest is the unary request message of the broker Read RPC. */ export interface ProtocolReadRequest { - /** Header is attached by a proxying broker peer. */ - header?: ProtocolHeader; - /** Journal to be read. */ - journal?: string; - /** - * Desired offset to begin reading from. Value -1 has special handling, where - * the read is performed from the current write head. All other positive - * values specify a desired exact byte offset to read from. If the offset is - * not available (eg, because it represents a portion of Journal which has - * been permanently deleted), the broker will return the next available - * offset. Callers should therefore always inspect the ReadResponse offset. - * @format int64 - */ - offset?: string; - /** - * Whether the operation should block until content becomes available. - * OFFSET_NOT_YET_AVAILABLE is returned if a non-blocking read has no ready - * content. - */ - block?: boolean; - /** - * If do_not_proxy is true, the broker will not proxy the read to another - * broker, or open and proxy a remote Fragment on the client's behalf. - */ - doNotProxy?: boolean; - /** - * If metadata_only is true, the broker will respond with Journal and - * Fragment metadata but not content. - */ - metadataOnly?: boolean; - /** - * Offset to read through. If zero, then the read end offset is unconstrained. - * @format int64 - */ - endOffset?: string; - /** - * BeginModTime is an optional inclusive lower bound on the modification - * timestamps of fragments read from the backing store, represented as - * seconds since the epoch. The request Offset will be advanced as-needed - * to skip persisted Fragments having a modication time before the bound. - * @format int64 - */ - beginModTime?: string; + /** Header is attached by a proxying broker peer. */ + header?: ProtocolHeader; + /** Journal to be read. */ + journal?: string; + /** + * Desired offset to begin reading from. Value -1 has special handling, where + * the read is performed from the current write head. All other positive + * values specify a desired exact byte offset to read from. If the offset is + * not available (eg, because it represents a portion of Journal which has + * been permanently deleted), the broker will return the next available + * offset. Callers should therefore always inspect the ReadResponse offset. + * @format int64 + */ + offset?: string; + /** + * Whether the operation should block until content becomes available. + * OFFSET_NOT_YET_AVAILABLE is returned if a non-blocking read has no ready + * content. + */ + block?: boolean; + /** + * If do_not_proxy is true, the broker will not proxy the read to another + * broker, or open and proxy a remote Fragment on the client's behalf. + */ + doNotProxy?: boolean; + /** + * If metadata_only is true, the broker will respond with Journal and + * Fragment metadata but not content. + */ + metadataOnly?: boolean; + /** + * Offset to read through. If zero, then the read end offset is unconstrained. + * @format int64 + */ + endOffset?: string; + /** + * BeginModTime is an optional inclusive lower bound on the modification + * timestamps of fragments read from the backing store, represented as + * seconds since the epoch. The request Offset will be advanced as-needed + * to skip persisted Fragments having a modication time before the bound. + * @format int64 + */ + beginModTime?: string; } /** @@ -528,38 +528,38 @@ hasn't yet committed). However once a metadata message is read, the reader is assured that its associated chunk messages are immediately forthcoming. */ export interface ProtocolReadResponse { - /** Status of the Read RPC. */ - status?: ProtocolStatus; - /** - * Header of the response. Accompanies the first ReadResponse of the response - * stream. - */ - header?: ProtocolHeader; - /** - * The effective offset of the read. See ReadRequest offset. - * @format int64 - */ - offset?: string; - /** - * The offset to next be written, by the next append transaction served by - * broker. In other words, the last offset through which content is - * available to be read from the Journal. This is a metadata field and will - * not be returned with a content response. - * @format int64 - */ - writeHead?: string; - /** - * Fragment to which the offset was mapped. This is a metadata field and will - * not be returned with a content response. - */ - fragment?: ProtocolFragment; - /** If Fragment is remote, a URL from which it may be directly read. */ - fragmentUrl?: string; - /** - * Content chunks of the read. - * @format byte - */ - content?: string; + /** Status of the Read RPC. */ + status?: ProtocolStatus; + /** + * Header of the response. Accompanies the first ReadResponse of the response + * stream. + */ + header?: ProtocolHeader; + /** + * The effective offset of the read. See ReadRequest offset. + * @format int64 + */ + offset?: string; + /** + * The offset to next be written, by the next append transaction served by + * broker. In other words, the last offset through which content is + * available to be read from the Journal. This is a metadata field and will + * not be returned with a content response. + * @format int64 + */ + writeHead?: string; + /** + * Fragment to which the offset was mapped. This is a metadata field and will + * not be returned with a content response. + */ + fragment?: ProtocolFragment; + /** If Fragment is remote, a URL from which it may be directly read. */ + fragmentUrl?: string; + /** + * Content chunks of the read. + * @format byte + */ + content?: string; } /** @@ -568,55 +568,55 @@ Replicate RPC. Each message is a 1:1 response to a previously read "proposal" ReplicateRequest with |acknowledge| set. */ export interface ProtocolReplicateResponse { - /** Status of the Replicate RPC. */ - status?: ProtocolStatus; - /** - * Header of the response. Accompanies the first ReplicateResponse of the - * response stream. - */ - header?: ProtocolHeader; - /** - * If status is PROPOSAL_MISMATCH, then |fragment| is the replica's current - * journal Fragment, and either it or |registers| will differ from the - * primary's proposal. - */ - fragment?: ProtocolFragment; - /** - * If status is PROPOSAL_MISMATCH, then |registers| are the replica's current - * journal registers. - */ - registers?: ProtocolLabelSet; + /** Status of the Replicate RPC. */ + status?: ProtocolStatus; + /** + * Header of the response. Accompanies the first ReplicateResponse of the + * response stream. + */ + header?: ProtocolHeader; + /** + * If status is PROPOSAL_MISMATCH, then |fragment| is the replica's current + * journal Fragment, and either it or |registers| will differ from the + * primary's proposal. + */ + fragment?: ProtocolFragment; + /** + * If status is PROPOSAL_MISMATCH, then |registers| are the replica's current + * journal registers. + */ + registers?: ProtocolLabelSet; } /** * Route captures the current topology of an item and the processes serving it. */ export interface ProtocolRoute { - /** Members of the Route, ordered on ascending ProcessSpec.ID (zone, suffix). */ - members?: ProcessSpecID[]; - /** - * Index of the ProcessSpec serving as primary within |members|, - * or -1 of no member is currently primary. - * @format int32 - */ - primary?: number; - /** - * Endpoints of each Route member. If not empty, |endpoints| has the same - * length and order as |members|, and captures the endpoint of each one. - */ - endpoints?: string[]; + /** Members of the Route, ordered on ascending ProcessSpec.ID (zone, suffix). */ + members?: ProcessSpecID[]; + /** + * Index of the ProcessSpec serving as primary within |members|, + * or -1 of no member is currently primary. + * @format int32 + */ + primary?: number; + /** + * Endpoints of each Route member. If not empty, |endpoints| has the same + * length and order as |members|, and captures the endpoint of each one. + */ + endpoints?: string[]; } /** * SHA1Sum is a 160-bit SHA1 digest. */ export interface ProtocolSHA1Sum { - /** @format uint64 */ - part1?: string; - /** @format uint64 */ - part2?: string; - /** @format int64 */ - part3?: number; + /** @format uint64 */ + part1?: string; + /** @format uint64 */ + part2?: string; + /** @format int64 */ + part3?: number; } /** @@ -650,35 +650,35 @@ too many broker or Etcd failures. request, but it was not matched by current register values of the journal. */ export type ProtocolStatus = - | 'OK' - | 'JOURNAL_NOT_FOUND' - | 'NO_JOURNAL_PRIMARY_BROKER' - | 'NOT_JOURNAL_PRIMARY_BROKER' - | 'NOT_JOURNAL_BROKER' - | 'INSUFFICIENT_JOURNAL_BROKERS' - | 'OFFSET_NOT_YET_AVAILABLE' - | 'WRONG_ROUTE' - | 'PROPOSAL_MISMATCH' - | 'ETCD_TRANSACTION_FAILED' - | 'NOT_ALLOWED' - | 'WRONG_APPEND_OFFSET' - | 'INDEX_HAS_GREATER_OFFSET' - | 'REGISTER_MISMATCH'; + | "OK" + | "JOURNAL_NOT_FOUND" + | "NO_JOURNAL_PRIMARY_BROKER" + | "NOT_JOURNAL_PRIMARY_BROKER" + | "NOT_JOURNAL_BROKER" + | "INSUFFICIENT_JOURNAL_BROKERS" + | "OFFSET_NOT_YET_AVAILABLE" + | "WRONG_ROUTE" + | "PROPOSAL_MISMATCH" + | "ETCD_TRANSACTION_FAILED" + | "NOT_ALLOWED" + | "WRONG_APPEND_OFFSET" + | "INDEX_HAS_GREATER_OFFSET" + | "REGISTER_MISMATCH"; export interface RuntimeError { - error?: string; - /** @format int32 */ - code?: number; - message?: string; - details?: ProtobufAny[]; + error?: string; + /** @format int32 */ + code?: number; + message?: string; + details?: ProtobufAny[]; } export interface RuntimeStreamError { - /** @format int32 */ - grpcCode?: number; - /** @format int32 */ - httpCode?: number; - message?: string; - httpStatus?: string; - details?: ProtobufAny[]; + /** @format int32 */ + grpcCode?: number; + /** @format int32 */ + httpCode?: number; + message?: string; + httpStatus?: string; + details?: ProtobufAny[]; } diff --git a/client/src/gen/consumer/protocol/consumer.ts b/client/src/gen/consumer/protocol/consumer.ts index e8c797e..5286064 100644 --- a/client/src/gen/consumer/protocol/consumer.ts +++ b/client/src/gen/consumer/protocol/consumer.ts @@ -10,8 +10,8 @@ */ export interface GetHintsResponseResponseHints { - /** If the hints value does not exist Hints will be nil. */ - hints?: RecoverylogFSMHints; + /** If the hints value does not exist Hints will be nil. */ + hints?: RecoverylogFSMHints; } /** @@ -25,76 +25,76 @@ different Etcd clusters). Etcd is kept in sync with etcdserverpb.ResponseHeader. */ export interface HeaderEtcd { - /** - * cluster_id is the ID of the cluster. - * @format uint64 - */ - clusterId?: string; - /** - * member_id is the ID of the member. - * @format uint64 - */ - memberId?: string; - /** - * revision is the Etcd key-value store revision when the request was - * applied. - * @format int64 - */ - revision?: string; - /** - * raft_term is the raft term when the request was applied. - * @format uint64 - */ - raftTerm?: string; + /** + * cluster_id is the ID of the cluster. + * @format uint64 + */ + clusterId?: string; + /** + * member_id is the ID of the member. + * @format uint64 + */ + memberId?: string; + /** + * revision is the Etcd key-value store revision when the request was + * applied. + * @format int64 + */ + revision?: string; + /** + * raft_term is the raft term when the request was applied. + * @format uint64 + */ + raftTerm?: string; } /** * Shards of the response. */ export interface ListResponseShard { - /** - * ShardSpec describes a shard and its configuration, and is the long-lived unit - * of work and scaling for a consumer application. Each shard is allocated to a - * one "primary" at-a-time selected from the current processes of a consumer - * application, and is re-assigned on process fault or exit. - * - * ShardSpecs describe all configuration of the shard and its processing, - * including journals to consume, configuration for processing transactions, its - * recovery log, hot standbys, etc. ShardSpecs may be further extended with - * domain-specific labels & values to further define application behavior. - * ShardSpec is-a allocator.ItemValue. - */ - spec?: ConsumerShardSpec; - /** - * Current ModRevision of the ShardSpec. - * @format int64 - */ - modRevision?: string; - /** Route of the shard, including endpoints. */ - route?: ProtocolRoute; - /** Status of each replica. Cardinality and ordering matches |route|. */ - status?: ConsumerReplicaStatus[]; + /** + * ShardSpec describes a shard and its configuration, and is the long-lived unit + * of work and scaling for a consumer application. Each shard is allocated to a + * one "primary" at-a-time selected from the current processes of a consumer + * application, and is re-assigned on process fault or exit. + * + * ShardSpecs describe all configuration of the shard and its processing, + * including journals to consume, configuration for processing transactions, its + * recovery log, hot standbys, etc. ShardSpecs may be further extended with + * domain-specific labels & values to further define application behavior. + * ShardSpec is-a allocator.ItemValue. + */ + spec?: ConsumerShardSpec; + /** + * Current ModRevision of the ShardSpec. + * @format int64 + */ + modRevision?: string; + /** Route of the shard, including endpoints. */ + route?: ProtocolRoute; + /** Status of each replica. Cardinality and ordering matches |route|. */ + status?: ConsumerReplicaStatus[]; } /** * ID composes a zone and a suffix to uniquely identify a ProcessSpec. */ export interface ProcessSpecID { - /** - * "Zone" in which the process is running. Zones may be AWS, Azure, or - * Google Cloud Platform zone identifiers, or rack locations within a colo, - * or given some other custom meaning. Gazette will replicate across - * multiple zones, and seeks to minimize traffic which must cross zones (for - * example, by proxying reads to a broker in the current zone). - */ - zone?: string; - /** - * Unique suffix of the process within |zone|. It is permissible for a - * suffix value to repeat across zones, but never within zones. In practice, - * it's recommended to use a FQDN, Kubernetes Pod name, or comparable unique - * and self-describing value as the ID suffix. - */ - suffix?: string; + /** + * "Zone" in which the process is running. Zones may be AWS, Azure, or + * Google Cloud Platform zone identifiers, or rack locations within a colo, + * or given some other custom meaning. Gazette will replicate across + * multiple zones, and seeks to minimize traffic which must cross zones (for + * example, by proxying reads to a broker in the current zone). + */ + zone?: string; + /** + * Unique suffix of the process within |zone|. It is permissible for a + * suffix value to repeat across zones, but never within zones. In practice, + * it's recommended to use a FQDN, Kubernetes Pod name, or comparable unique + * and self-describing value as the ID suffix. + */ + suffix?: string; } /** @@ -107,87 +107,87 @@ Shards not having recovery logs immediately transition to STANDBY. - PRIMARY: The replica is actively serving as primary. - FAILED: The replica has encountered an unrecoverable error. */ -export type ReplicaStatusCode = 'IDLE' | 'BACKFILL' | 'STANDBY' | 'PRIMARY' | 'FAILED'; +export type ReplicaStatusCode = "IDLE" | "BACKFILL" | "STANDBY" | "PRIMARY" | "FAILED"; /** * Change defines an insertion, update, or deletion to be applied to the set of ShardSpecs. Exactly one of |upsert| or |delete| must be set. */ export interface ConsumerApplyRequestChange { - /** - * Expected ModRevision of the current ShardSpec. If the shard is being - * created, expect_mod_revision is zero. - * @format int64 - */ - expectModRevision?: string; - /** - * ShardSpec to be updated (if expect_mod_revision > 0) or created - * (if expect_mod_revision == 0). - */ - upsert?: ConsumerShardSpec; - /** Shard to be deleted. expect_mod_revision must not be zero. */ - delete?: string; + /** + * Expected ModRevision of the current ShardSpec. If the shard is being + * created, expect_mod_revision is zero. + * @format int64 + */ + expectModRevision?: string; + /** + * ShardSpec to be updated (if expect_mod_revision > 0) or created + * (if expect_mod_revision == 0). + */ + upsert?: ConsumerShardSpec; + /** Shard to be deleted. expect_mod_revision must not be zero. */ + delete?: string; } export interface ConsumerApplyResponse { - /** Status of the Apply RPC. */ - status?: ConsumerStatus; - /** Header of the response. */ - header?: ProtocolHeader; - /** - * Optional extension of the ApplyResponse. - * @format byte - */ - extension?: string; + /** Status of the Apply RPC. */ + status?: ConsumerStatus; + /** Header of the response. */ + header?: ProtocolHeader; + /** + * Optional extension of the ApplyResponse. + * @format byte + */ + extension?: string; } export interface ConsumerGetHintsResponse { - /** Status of the Hints RPC. */ - status?: ConsumerStatus; - /** Header of the response. */ - header?: ProtocolHeader; - /** Primary hints for the shard. */ - primaryHints?: GetHintsResponseResponseHints; - /** - * List of backup hints for a shard. The most recent recovery log hints will - * be first, any subsequent hints are for historical backup. If there is no - * value for a hint key the value corresponding hints will be nil. - */ - backupHints?: GetHintsResponseResponseHints[]; - /** - * Optional extension of the GetHintsResponse. - * @format byte - */ - extension?: string; + /** Status of the Hints RPC. */ + status?: ConsumerStatus; + /** Header of the response. */ + header?: ProtocolHeader; + /** Primary hints for the shard. */ + primaryHints?: GetHintsResponseResponseHints; + /** + * List of backup hints for a shard. The most recent recovery log hints will + * be first, any subsequent hints are for historical backup. If there is no + * value for a hint key the value corresponding hints will be nil. + */ + backupHints?: GetHintsResponseResponseHints[]; + /** + * Optional extension of the GetHintsResponse. + * @format byte + */ + extension?: string; } export interface ConsumerListRequest { - /** - * Selector optionally refines the set of shards which will be enumerated. - * If zero-valued, all shards are returned. Otherwise, only ShardSpecs - * matching the LabelSelector will be returned. One meta-label "id" is - * additionally supported by the selector, where "id=example-shard-ID" - * will match a ShardSpec with ID "example-shard-ID". - */ - selector?: ProtocolLabelSelector; - /** - * Optional extension of the ListRequest. - * @format byte - */ - extension?: string; + /** + * Selector optionally refines the set of shards which will be enumerated. + * If zero-valued, all shards are returned. Otherwise, only ShardSpecs + * matching the LabelSelector will be returned. One meta-label "id" is + * additionally supported by the selector, where "id=example-shard-ID" + * will match a ShardSpec with ID "example-shard-ID". + */ + selector?: ProtocolLabelSelector; + /** + * Optional extension of the ListRequest. + * @format byte + */ + extension?: string; } export interface ConsumerListResponse { - /** Status of the List RPC. */ - status?: ConsumerStatus; - /** Header of the response. */ - header?: ProtocolHeader; - shards?: ListResponseShard[]; - /** - * Optional extension of the ListResponse. - * @format byte - */ - extension?: string; + /** Status of the List RPC. */ + status?: ConsumerStatus; + /** Header of the response. */ + header?: ProtocolHeader; + shards?: ListResponseShard[]; + /** + * Optional extension of the ListResponse. + * @format byte + */ + extension?: string; } /** @@ -199,19 +199,19 @@ of the replicas transitioned to FAILED, than the status is FAILED. This reduction behavior is used to summarize status across all replicas. */ export interface ConsumerReplicaStatus { - /** - * - BACKFILL: The replica is actively playing the historical recovery log. - * - STANDBY: The replica has finished playing the historical recovery log and is - * live-tailing it to locally mirror recorded operations as they are - * produced. It can take over as primary at any time. - * - * Shards not having recovery logs immediately transition to STANDBY. - * - PRIMARY: The replica is actively serving as primary. - * - FAILED: The replica has encountered an unrecoverable error. - */ - code?: ReplicaStatusCode; - /** Errors encountered during replica processing. Set iff |code| is FAILED. */ - errors?: string[]; + /** + * - BACKFILL: The replica is actively playing the historical recovery log. + * - STANDBY: The replica has finished playing the historical recovery log and is + * live-tailing it to locally mirror recorded operations as they are + * produced. It can take over as primary at any time. + * + * Shards not having recovery logs immediately transition to STANDBY. + * - PRIMARY: The replica is actively serving as primary. + * - FAILED: The replica has encountered an unrecoverable error. + */ + code?: ReplicaStatusCode; + /** Errors encountered during replica processing. Set iff |code| is FAILED. */ + errors?: string[]; } /** @@ -227,131 +227,131 @@ domain-specific labels & values to further define application behavior. ShardSpec is-a allocator.ItemValue. */ export interface ConsumerShardSpec { - /** ID of the shard. */ - id?: string; - /** Sources of the shard, uniquely ordered on Source journal. */ - sources?: ConsumerShardSpecSource[]; - /** - * Prefix of the Journal into which the shard's recovery log will be recorded. - * The complete Journal name is built as "{recovery_log_prefix}/{shard_id}". - * If empty, the shard does not use a recovery log. - */ - recoveryLogPrefix?: string; - /** - * Prefix of Etcd keys into which recovery log FSMHints are written to and - * read from. FSMHints allow readers of the recovery log to efficiently - * determine the minimum fragments of log which must be read to fully recover - * local store state. The complete hint key written by the shard primary is: - * "{hint_prefix}/{shard_id}.primary" - * - * The primary will regularly produce updated hints into this key, and - * players of the log will similarly utilize hints from this key. - * If |recovery_log_prefix| is set, |hint_prefix| must be also. - */ - hintPrefix?: string; - /** - * Backups of verified recovery log FSMHints, retained as a disaster-recovery - * mechanism. On completing playback, a player will write recovered hints to: - * "{hints_prefix}/{shard_id}.backup.0". - * - * It also move hints previously stored under - * "{hints_prefix/{shard_id}.backup.0" to - * "{hints_prefix/{shard_id}.backup.1", and so on, keeping at most - * |hint_backups| distinct sets of FSMHints. - * In the case of disaster or data-loss, these copied hints can be an - * important fallback for recovering a consistent albeit older version of the - * shard's store, with each relying on only progressively older portions of - * the recovery log. - * When pruning the recovery log, log fragments which are older than (and no - * longer required by) the *oldest* backup are discarded, ensuring that - * all hints remain valid for playback. - * @format int32 - */ - hintBackups?: number; - /** - * Max duration of shard transactions. This duration upper-bounds the amount - * of time during which a transaction may process messages before it must - * flush and commit. It may run for less time if an input message stall occurs - * (eg, no decoded journal message is ready without blocking). A typical value - * would be `1s`: applications which perform extensive aggregation over - * message streams exhibiting locality of "hot" keys may benefit from larger - * values. - */ - maxTxnDuration?: string; - /** - * Min duration of shard transactions. This duration lower-bounds the amount - * of time during which a transaction must process messages before it may - * flush and commit. It may run for more time if additional messages are - * available (eg, decoded journal messages are ready without blocking). Note - * also that transactions are pipelined: a current transaction may process - * messages while a prior transaction's recovery log writes flush to Gazette, - * but it cannot prepare to commit until the prior transaction writes - * complete. In other words even if |min_txn_quantum| is zero, some degree of - * message batching is expected due to the network delay inherent in Gazette - * writes. A typical value of would be `0s`: applications which perform - * extensive aggregation may benefit from larger values. - */ - minTxnDuration?: string; - /** Disable processing of the shard. */ - disable?: boolean; - /** - * Hot standbys is the desired number of consumer processes which should be - * replicating the primary consumer's recovery log. Standbys are allocated in - * a separate availability zone of the current primary, and tail the live log - * to continuously mirror the primary's on-disk DB file structure. Should the - * primary experience failure, one of the hot standbys will be assigned to - * take over as the new shard primary, which is accomplished by simply opening - * its local copy of the recovered store files. - * - * Note that under regular operation, shard hand-off is zero downtime even if - * standbys are zero, as the current primary will not cede ownership until the - * replacement process declares itself ready. However, without standbys a - * process failure will leave the shard without an active primary while its - * replacement starts and completes playback of its recovery log. - * @format int64 - */ - hotStandbys?: number; - /** - * User-defined Labels of this ShardSpec. The label "id" is reserved and may - * not be used with a ShardSpec's labels. - */ - labels?: ProtocolLabelSet; - /** - * Disable waiting for acknowledgements of pending message(s). - * - * If a consumer transaction reads uncommitted messages, it will by default - * remain open (subject to the max duration) awaiting an acknowledgement of - * those messages, in the hope that that acknowledgement will be quickly - * forthcoming and, by remaining open, we can process all messages in this - * transaction. Effectively we're trading a small amount of increased local - * latency for a global reduction in end-to-end latency. - * This works well for acyclic message flows, but can introduce unnecessary - * stalls if there are message cycles between shards. In the simplest case, - * a transaction could block awaiting an ACK of a message that it itself - * produced -- an ACK which can't arrive until the transaction closes. - */ - disableWaitForAck?: boolean; - /** - * Size of the ring buffer used to sequence read-uncommitted messages - * into consumed, read-committed ones. The ring buffer is a performance - * optimization only: applications will replay portions of journals as - * needed when messages aren't available in the buffer. - * It can remain small if source journal transactions are small, - * but larger transactions will achieve better performance with a - * larger ring. - * If zero, a reasonable default (currently 8192) is used. - * @format int64 - */ - ringBufferSize?: number; - /** - * Size of the channel used to bridge message read and decode with - * sequencing and consumption. Larger values may reduce data stalls, - * particularly for larger transactions and/or bursty custom - * MessageProducer implementations. - * If zero, a reasonable default (currently 8192) is used. - * @format int64 - */ - readChannelSize?: number; + /** ID of the shard. */ + id?: string; + /** Sources of the shard, uniquely ordered on Source journal. */ + sources?: ConsumerShardSpecSource[]; + /** + * Prefix of the Journal into which the shard's recovery log will be recorded. + * The complete Journal name is built as "{recovery_log_prefix}/{shard_id}". + * If empty, the shard does not use a recovery log. + */ + recoveryLogPrefix?: string; + /** + * Prefix of Etcd keys into which recovery log FSMHints are written to and + * read from. FSMHints allow readers of the recovery log to efficiently + * determine the minimum fragments of log which must be read to fully recover + * local store state. The complete hint key written by the shard primary is: + * "{hint_prefix}/{shard_id}.primary" + * + * The primary will regularly produce updated hints into this key, and + * players of the log will similarly utilize hints from this key. + * If |recovery_log_prefix| is set, |hint_prefix| must be also. + */ + hintPrefix?: string; + /** + * Backups of verified recovery log FSMHints, retained as a disaster-recovery + * mechanism. On completing playback, a player will write recovered hints to: + * "{hints_prefix}/{shard_id}.backup.0". + * + * It also move hints previously stored under + * "{hints_prefix/{shard_id}.backup.0" to + * "{hints_prefix/{shard_id}.backup.1", and so on, keeping at most + * |hint_backups| distinct sets of FSMHints. + * In the case of disaster or data-loss, these copied hints can be an + * important fallback for recovering a consistent albeit older version of the + * shard's store, with each relying on only progressively older portions of + * the recovery log. + * When pruning the recovery log, log fragments which are older than (and no + * longer required by) the *oldest* backup are discarded, ensuring that + * all hints remain valid for playback. + * @format int32 + */ + hintBackups?: number; + /** + * Max duration of shard transactions. This duration upper-bounds the amount + * of time during which a transaction may process messages before it must + * flush and commit. It may run for less time if an input message stall occurs + * (eg, no decoded journal message is ready without blocking). A typical value + * would be `1s`: applications which perform extensive aggregation over + * message streams exhibiting locality of "hot" keys may benefit from larger + * values. + */ + maxTxnDuration?: string; + /** + * Min duration of shard transactions. This duration lower-bounds the amount + * of time during which a transaction must process messages before it may + * flush and commit. It may run for more time if additional messages are + * available (eg, decoded journal messages are ready without blocking). Note + * also that transactions are pipelined: a current transaction may process + * messages while a prior transaction's recovery log writes flush to Gazette, + * but it cannot prepare to commit until the prior transaction writes + * complete. In other words even if |min_txn_quantum| is zero, some degree of + * message batching is expected due to the network delay inherent in Gazette + * writes. A typical value of would be `0s`: applications which perform + * extensive aggregation may benefit from larger values. + */ + minTxnDuration?: string; + /** Disable processing of the shard. */ + disable?: boolean; + /** + * Hot standbys is the desired number of consumer processes which should be + * replicating the primary consumer's recovery log. Standbys are allocated in + * a separate availability zone of the current primary, and tail the live log + * to continuously mirror the primary's on-disk DB file structure. Should the + * primary experience failure, one of the hot standbys will be assigned to + * take over as the new shard primary, which is accomplished by simply opening + * its local copy of the recovered store files. + * + * Note that under regular operation, shard hand-off is zero downtime even if + * standbys are zero, as the current primary will not cede ownership until the + * replacement process declares itself ready. However, without standbys a + * process failure will leave the shard without an active primary while its + * replacement starts and completes playback of its recovery log. + * @format int64 + */ + hotStandbys?: number; + /** + * User-defined Labels of this ShardSpec. The label "id" is reserved and may + * not be used with a ShardSpec's labels. + */ + labels?: ProtocolLabelSet; + /** + * Disable waiting for acknowledgements of pending message(s). + * + * If a consumer transaction reads uncommitted messages, it will by default + * remain open (subject to the max duration) awaiting an acknowledgement of + * those messages, in the hope that that acknowledgement will be quickly + * forthcoming and, by remaining open, we can process all messages in this + * transaction. Effectively we're trading a small amount of increased local + * latency for a global reduction in end-to-end latency. + * This works well for acyclic message flows, but can introduce unnecessary + * stalls if there are message cycles between shards. In the simplest case, + * a transaction could block awaiting an ACK of a message that it itself + * produced -- an ACK which can't arrive until the transaction closes. + */ + disableWaitForAck?: boolean; + /** + * Size of the ring buffer used to sequence read-uncommitted messages + * into consumed, read-committed ones. The ring buffer is a performance + * optimization only: applications will replay portions of journals as + * needed when messages aren't available in the buffer. + * It can remain small if source journal transactions are small, + * but larger transactions will achieve better performance with a + * larger ring. + * If zero, a reasonable default (currently 8192) is used. + * @format int64 + */ + ringBufferSize?: number; + /** + * Size of the channel used to bridge message read and decode with + * sequencing and consumption. Larger values may reduce data stalls, + * particularly for larger transactions and/or bursty custom + * MessageProducer implementations. + * If zero, a reasonable default (currently 8192) is used. + * @format int64 + */ + readChannelSize?: number; } /** @@ -372,65 +372,65 @@ events, read a comparably low-volume "purchase" event stream, and on each purchase publish the bundle of its corresponding prior product views. */ export interface ConsumerShardSpecSource { - /** Journal which this shard is consuming. */ - journal?: string; - /** - * Minimum journal byte offset the shard should begin reading from. - * Typically this should be zero, as read offsets are check-pointed and - * restored from the shard's Store as it processes. |min_offset| can be - * useful for shard initialization, directing it to skip over historical - * portions of the journal not needed for the application's use case. - * @format int64 - */ - minOffset?: string; + /** Journal which this shard is consuming. */ + journal?: string; + /** + * Minimum journal byte offset the shard should begin reading from. + * Typically this should be zero, as read offsets are check-pointed and + * restored from the shard's Store as it processes. |min_offset| can be + * useful for shard initialization, directing it to skip over historical + * portions of the journal not needed for the application's use case. + * @format int64 + */ + minOffset?: string; } export interface ConsumerStatRequest { - /** Header may be attached by a proxying consumer peer. */ - header?: ProtocolHeader; - /** Shard to Stat. */ - shard?: string; - /** - * Journals and offsets which must be reflected in a completed consumer - * transaction before Stat returns, blocking if required. Offsets of journals - * not read by this shard are ignored. - */ - readThrough?: Record; - /** - * Optional extension of the StatRequest. - * @format byte - */ - extension?: string; + /** Header may be attached by a proxying consumer peer. */ + header?: ProtocolHeader; + /** Shard to Stat. */ + shard?: string; + /** + * Journals and offsets which must be reflected in a completed consumer + * transaction before Stat returns, blocking if required. Offsets of journals + * not read by this shard are ignored. + */ + readThrough?: Record; + /** + * Optional extension of the StatRequest. + * @format byte + */ + extension?: string; } export interface ConsumerStatResponse { - /** Status of the Stat RPC. */ - status?: ConsumerStatus; - /** Header of the response. */ - header?: ProtocolHeader; - /** - * Journals and offsets read through by the most recent completed consumer - * transaction. - */ - readThrough?: Record; - /** - * Journals and offsets this shard has published through, including - * acknowledgements, as-of the most recent completed consumer transaction. - * - * Formally, if an acknowledged message A results in this shard publishing - * messages B, and A falls within |read_through|, then all messages B & their - * acknowledgements fall within |publish_at|. - * The composition of |read_through| and |publish_at| allow CQRS applications - * to provide read-your-writes consistency, even if written events pass - * through multiple intermediate consumers and arbitrary transformations - * before arriving at the materialized view which is ultimately queried. - */ - publishAt?: Record; - /** - * Optional extension of the StatResponse. - * @format byte - */ - extension?: string; + /** Status of the Stat RPC. */ + status?: ConsumerStatus; + /** Header of the response. */ + header?: ProtocolHeader; + /** + * Journals and offsets read through by the most recent completed consumer + * transaction. + */ + readThrough?: Record; + /** + * Journals and offsets this shard has published through, including + * acknowledgements, as-of the most recent completed consumer transaction. + * + * Formally, if an acknowledged message A results in this shard publishing + * messages B, and A falls within |read_through|, then all messages B & their + * acknowledgements fall within |publish_at|. + * The composition of |read_through| and |publish_at| allow CQRS applications + * to provide read-your-writes consistency, even if written events pass + * through multiple intermediate consumers and arbitrary transformations + * before arriving at the materialized view which is ultimately queried. + */ + publishAt?: Record; + /** + * Optional extension of the StatResponse. + * @format byte + */ + extension?: string; } /** @@ -452,24 +452,24 @@ For example, a Stat RPC will return SHARD_STOPPED if the StatRequest cannot be satisfied. */ export type ConsumerStatus = - | 'OK' - | 'SHARD_NOT_FOUND' - | 'NO_SHARD_PRIMARY' - | 'NOT_SHARD_PRIMARY' - | 'ETCD_TRANSACTION_FAILED' - | 'SHARD_STOPPED'; + | "OK" + | "SHARD_NOT_FOUND" + | "NO_SHARD_PRIMARY" + | "NOT_SHARD_PRIMARY" + | "ETCD_TRANSACTION_FAILED" + | "SHARD_STOPPED"; export interface ConsumerUnassignResponse { - /** Status of the Unassign RPC. */ - status?: ConsumerStatus; - /** Shards which had assignments removed. */ - shards?: string[]; + /** Status of the Unassign RPC. */ + status?: ConsumerStatus; + /** Shards which had assignments removed. */ + shards?: string[]; } export interface ProtobufAny { - typeUrl?: string; - /** @format byte */ - value?: string; + typeUrl?: string; + /** @format byte */ + value?: string; } /** @@ -477,28 +477,28 @@ export interface ProtobufAny { an RPC, and its effective Etcd state. */ export interface ProtocolHeader { - /** - * ID of the process responsible for request processing. May be empty iff - * Header is being used within a proxied request, and that request may be - * dispatched to any member of the Route. - */ - processId?: ProcessSpecID; - /** - * Route of processes specifically responsible for this RPC, or an empty Route - * if any process is capable of serving the RPC. - */ - route?: ProtocolRoute; - /** - * Etcd represents the effective Etcd MVCC state under which a Gazette broker - * is operating in its processing of requests and responses. Its inclusion - * allows brokers to reason about relative "happened before" Revision ordering - * of apparent routing conflicts in proxied or replicated requests, as well - * as enabling sanity checks over equality of Etcd ClusterId (and precluding, - * for example, split-brain scenarios where different brokers are backed by - * different Etcd clusters). Etcd is kept in sync with - * etcdserverpb.ResponseHeader. - */ - etcd?: HeaderEtcd; + /** + * ID of the process responsible for request processing. May be empty iff + * Header is being used within a proxied request, and that request may be + * dispatched to any member of the Route. + */ + processId?: ProcessSpecID; + /** + * Route of processes specifically responsible for this RPC, or an empty Route + * if any process is capable of serving the RPC. + */ + route?: ProtocolRoute; + /** + * Etcd represents the effective Etcd MVCC state under which a Gazette broker + * is operating in its processing of requests and responses. Its inclusion + * allows brokers to reason about relative "happened before" Revision ordering + * of apparent routing conflicts in proxied or replicated requests, as well + * as enabling sanity checks over equality of Etcd ClusterId (and precluding, + * for example, split-brain scenarios where different brokers are backed by + * different Etcd clusters). Etcd is kept in sync with + * etcdserverpb.ResponseHeader. + */ + etcd?: HeaderEtcd; } /** @@ -508,53 +508,53 @@ attributes which do not directly imply semantics to the core system, but are meaningful to users or for higher-level Gazette tools. */ export interface ProtocolLabel { - name?: string; - value?: string; + name?: string; + value?: string; } /** * LabelSelector defines a filter over LabelSets. */ export interface ProtocolLabelSelector { - /** - * Include is Labels which must be matched for a LabelSet to be selected. If - * empty, all Labels are included. An include Label with empty ("") value is - * matched by a Label of the same name having any value. - */ - include?: ProtocolLabelSet; - /** - * Exclude is Labels which cannot be matched for a LabelSet to be selected. If - * empty, no Labels are excluded. An exclude Label with empty ("") value - * excludes a Label of the same name having any value. - */ - exclude?: ProtocolLabelSet; + /** + * Include is Labels which must be matched for a LabelSet to be selected. If + * empty, all Labels are included. An include Label with empty ("") value is + * matched by a Label of the same name having any value. + */ + include?: ProtocolLabelSet; + /** + * Exclude is Labels which cannot be matched for a LabelSet to be selected. If + * empty, no Labels are excluded. An exclude Label with empty ("") value + * excludes a Label of the same name having any value. + */ + exclude?: ProtocolLabelSet; } /** * LabelSet is a collection of labels and their values. */ export interface ProtocolLabelSet { - /** Labels of the set. Instances must be unique and sorted over (Name, Value). */ - labels?: ProtocolLabel[]; + /** Labels of the set. Instances must be unique and sorted over (Name, Value). */ + labels?: ProtocolLabel[]; } /** * Route captures the current topology of an item and the processes serving it. */ export interface ProtocolRoute { - /** Members of the Route, ordered on ascending ProcessSpec.ID (zone, suffix). */ - members?: ProcessSpecID[]; - /** - * Index of the ProcessSpec serving as primary within |members|, - * or -1 of no member is currently primary. - * @format int32 - */ - primary?: number; - /** - * Endpoints of each Route member. If not empty, |endpoints| has the same - * length and order as |members|, and captures the endpoint of each one. - */ - endpoints?: string[]; + /** Members of the Route, ordered on ascending ProcessSpec.ID (zone, suffix). */ + members?: ProcessSpecID[]; + /** + * Index of the ProcessSpec serving as primary within |members|, + * or -1 of no member is currently primary. + * @format int32 + */ + primary?: number; + /** + * Endpoints of each Route member. If not empty, |endpoints| has the same + * length and order as |members|, and captures the endpoint of each one. + */ + endpoints?: string[]; } /** @@ -568,41 +568,41 @@ that of the Recorder producing the FSMHints. Next tag: 4. */ export interface RecoverylogFSMHints { - /** - * Log is the implied recovery log of any contained |live_nodes| Segments - * which omit a |log| value. This implied behavior is both for backward- - * compatibility (Segments didn't always have a |log| field) and also for - * compacting the representation in the common case of Segments mostly or - * entirely addressing a single log. - */ - log?: string; - /** Live Fnodes and their Segments as-of the generation of these FSMHints. */ - liveNodes?: RecoverylogFnodeSegments[]; - /** Property files and contents as-of the generation of these FSMHints. */ - properties?: RecoverylogProperty[]; + /** + * Log is the implied recovery log of any contained |live_nodes| Segments + * which omit a |log| value. This implied behavior is both for backward- + * compatibility (Segments didn't always have a |log| field) and also for + * compacting the representation in the common case of Segments mostly or + * entirely addressing a single log. + */ + log?: string; + /** Live Fnodes and their Segments as-of the generation of these FSMHints. */ + liveNodes?: RecoverylogFnodeSegments[]; + /** Property files and contents as-of the generation of these FSMHints. */ + properties?: RecoverylogProperty[]; } /** * FnodeSegments captures log Segments containing all RecordedOps of the Fnode. */ export interface RecoverylogFnodeSegments { - /** - * Fnode being hinted. - * @format int64 - */ - fnode?: string; - /** - * Segments of the Fnode in the log. Currently, FSM tracks only a single - * Segment per Fnode per Author & Log. A specific implication of this is that Fnodes - * modified over long periods of time will result in Segments spanning large - * chunks of the log. For best performance, Fnodes should be opened & written - * once, and then never be modified again (this is RocksDB's behavior). - * If supporting this case is desired, FSM will have to be a bit smarter about - * not extending Segments which gap over significant portions of the log - * (eg, there's a trade-off to make over size of the hinted manifest, vs - * savings incurred on playback by being able to skip portions of the log). - */ - segments?: RecoverylogSegment[]; + /** + * Fnode being hinted. + * @format int64 + */ + fnode?: string; + /** + * Segments of the Fnode in the log. Currently, FSM tracks only a single + * Segment per Fnode per Author & Log. A specific implication of this is that Fnodes + * modified over long periods of time will result in Segments spanning large + * chunks of the log. For best performance, Fnodes should be opened & written + * once, and then never be modified again (this is RocksDB's behavior). + * If supporting this case is desired, FSM will have to be a bit smarter about + * not extending Segments which gap over significant portions of the log + * (eg, there's a trade-off to make over size of the hinted manifest, vs + * savings incurred on playback by being able to skip portions of the log). + */ + segments?: RecoverylogSegment[]; } /** @@ -610,10 +610,10 @@ export interface RecoverylogFnodeSegments { outside of regular Fnode tracking. See FSM.Properties. */ export interface RecoverylogProperty { - /** Filesystem path of this property, relative to the common base directory. */ - path?: string; - /** Complete file content of this property. */ - content?: string; + /** Filesystem path of this property, relative to the common base directory. */ + path?: string; + /** Complete file content of this property. */ + content?: string; } /** @@ -629,50 +629,50 @@ recorded operations at their exact offsets. Next tag: 8. */ export interface RecoverylogSegment { - /** - * Author which wrote RecordedOps of this Segment. - * @format int64 - */ - author?: number; - /** - * First (lowest) sequence number of RecordedOps within this Segment. - * @format int64 - */ - firstSeqNo?: string; - /** - * First byte offset of the Segment, where |first_seq_no| is recorded. - * If this Segment was produced by a Recorder, this is guaranteed only to be a - * lower-bound (eg, a Player reading at this offset may encounter irrelevant - * operations prior to the RecordedOp indicated by the tuple - * (|author|, |first_seq_no|, |first_checksum|). If a Player produced the Segment, - * first_offset is exact. - * @format int64 - */ - firstOffset?: string; - /** - * Checksum of the RecordedOp having |first_seq_no|. - * @format int64 - */ - firstChecksum?: number; - /** - * Last (highest, inclusive) sequence number of RecordedOps within this Segment. - * @format int64 - */ - lastSeqNo?: string; - /** - * Last offset (exclusive) of the Segment. Zero means the offset is not known - * (eg, because the Segment was produced by a Recorder). - * @format int64 - */ - lastOffset?: string; - /** Log is the Journal holding this Segment's data, and to which offsets are relative. */ - log?: string; + /** + * Author which wrote RecordedOps of this Segment. + * @format int64 + */ + author?: number; + /** + * First (lowest) sequence number of RecordedOps within this Segment. + * @format int64 + */ + firstSeqNo?: string; + /** + * First byte offset of the Segment, where |first_seq_no| is recorded. + * If this Segment was produced by a Recorder, this is guaranteed only to be a + * lower-bound (eg, a Player reading at this offset may encounter irrelevant + * operations prior to the RecordedOp indicated by the tuple + * (|author|, |first_seq_no|, |first_checksum|). If a Player produced the Segment, + * first_offset is exact. + * @format int64 + */ + firstOffset?: string; + /** + * Checksum of the RecordedOp having |first_seq_no|. + * @format int64 + */ + firstChecksum?: number; + /** + * Last (highest, inclusive) sequence number of RecordedOps within this Segment. + * @format int64 + */ + lastSeqNo?: string; + /** + * Last offset (exclusive) of the Segment. Zero means the offset is not known + * (eg, because the Segment was produced by a Recorder). + * @format int64 + */ + lastOffset?: string; + /** Log is the Journal holding this Segment's data, and to which offsets are relative. */ + log?: string; } export interface RuntimeError { - error?: string; - /** @format int32 */ - code?: number; - message?: string; - details?: ProtobufAny[]; + error?: string; + /** @format int32 */ + code?: number; + message?: string; + details?: ProtobufAny[]; }