Skip to content

Commit

Permalink
Feat: Add DefaultFilterSelector and DefaultOutputSelector to fluentd (#…
Browse files Browse the repository at this point in the history 
…804)

* apis: Add DefaultFilterSelector, DefaultOutputSelector to fluentdSpec

Signed-off-by: Jakob Hahn <[email protected]>

* Render manifests with updated fluentdSpec

Signed-off-by: Jakob Hahn <[email protected]>

* apis: Update/Add comments, Fix PatchAndFilter errors

Signed-off-by: Jakob Hahn <[email protected]>

* apis: Generate deepcopy code

Signed-off-by: Jakob Hahn <[email protected]>

* controllers: Update/Add comments, Use DefaultFilter and DefaultOutput for fluentd

Signed-off-by: Jakob Hahn <[email protected]>

* docs: Updated

Signed-off-by: Jakob Hahn <[email protected]>

---------

Signed-off-by: Jakob Hahn <[email protected]>
  • Loading branch information
@Jakob3xD
Jakob3xD committed Jun 27, 2023
1 parent ab72254 commit 251932e
Show file tree
Hide file tree
Showing 10 changed files with 496 additions and 56 deletions.
4 changes: 4 additions & 0 deletions apis/fluentd/v1alpha1/fluentd_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ const (
type FluentdSpec struct {
// Fluentd global inputs.
GlobalInputs []input.Input `json:"globalInputs,omitempty"`
// Select cluster filter plugins used to filter for the default cluster output
DefaultFilterSelector *metav1.LabelSelector `json:"defaultFilterSelector,omitempty"`
// Select cluster output plugins used to send all logs that did not match any route to the matching outputs
DefaultOutputSelector *metav1.LabelSelector `json:"defaultOutputSelector,omitempty"`
// By default will build the related service according to the globalinputs definition.
DisableService bool `json:"disableService,omitempty"`
// Numbers of the Fluentd instance
Expand Down
33 changes: 24 additions & 9 deletions apis/fluentd/v1alpha1/helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,15 +98,20 @@ func (pgr *PluginResources) BuildCfgRouter(cfg Renderer) (*fluentdRouter.Route,
return nil, err
}

// Insert the route to the MainRouter
pgr.MainRouterPlugins.InsertChilds(routePluginStore)

return cfgRoute, nil
}

// PatchAndFilterClusterLevelResources will combine and patch all the cluster CRs that the fluentdconfig selected,
// convert the related filter/output pluginstores to the global pluginresources.
func (pgr *PluginResources) PatchAndFilterClusterLevelResources(sl plugins.SecretLoader, cfgId string,
clusterfilters []ClusterFilter, clusteroutputs []ClusterOutput) (*CfgResources, []string) {
func (pgr *PluginResources) PatchAndFilterClusterLevelResources(
sl plugins.SecretLoader,
cfgId string,
clusterfilters []ClusterFilter,
clusteroutputs []ClusterOutput,
) (*CfgResources, []string) {
// To store all filters/outputs plugins that this cfg selected
cfgResources := NewCfgResources()

Expand Down Expand Up @@ -135,8 +140,12 @@ func (pgr *PluginResources) PatchAndFilterClusterLevelResources(sl plugins.Secre

// PatchAndFilterNamespacedLevelResources will combine and patch all the cluster CRs that the fluentdconfig selected,
// convert the related filter/output pluginstores to the global pluginresources.
func (pgr *PluginResources) PatchAndFilterNamespacedLevelResources(sl plugins.SecretLoader, cfgId string,
filters []Filter, outputs []Output) (*CfgResources, []string) {
func (pgr *PluginResources) PatchAndFilterNamespacedLevelResources(
sl plugins.SecretLoader,
cfgId string,
filters []Filter,
outputs []Output,
) (*CfgResources, []string) {
// To store all filters/outputs plugins that this cfg selected
cfgResources := NewCfgResources()

Expand All @@ -163,8 +172,11 @@ func (pgr *PluginResources) PatchAndFilterNamespacedLevelResources(sl plugins.Se
return cfgResources, errs
}

func (r *CfgResources) filterForFilters(cfgId, namespace, name, crdtype string,
sl plugins.SecretLoader, filters []filter.Filter) error {
func (r *CfgResources) filterForFilters(
cfgId, namespace, name, crdtype string,
sl plugins.SecretLoader,
filters []filter.Filter,
) error {
for n, filter := range filters {
filterId := fmt.Sprintf("%s::%s::%s::%s-%d", cfgId, namespace, crdtype, name, n)
filter.FilterCommon.Id = &filterId
Expand All @@ -189,8 +201,11 @@ func (r *CfgResources) filterForFilters(cfgId, namespace, name, crdtype string,
return nil
}

func (r *CfgResources) filterForOutputs(cfgId, namespace, name, crdtype string,
sl plugins.SecretLoader, outputs []output.Output) error {
func (r *CfgResources) filterForOutputs(
cfgId, namespace, name, crdtype string,
sl plugins.SecretLoader,
outputs []output.Output,
) error {
for n, output := range outputs {
outputId := fmt.Sprintf("%s::%s::%s::%s-%d", cfgId, namespace, crdtype, name, n)
output.OutputCommon.Id = &outputId
Expand Down Expand Up @@ -218,7 +233,7 @@ func (r *CfgResources) filterForOutputs(cfgId, namespace, name, crdtype string,
// convert the cfg plugins to a label plugin, appends to the global label plugins
func (pgr *PluginResources) WithCfgResources(cfgRouteLabel string, r *CfgResources) error {
if len(r.FilterPlugins) == 0 && len(r.OutputPlugins) == 0 {
return errors.New("no filter plugins or output plugins matched")
return errors.New("no filter plugins and no output plugins matched")
}

cfgLabelPlugin := params.NewPluginStore("label")
Expand Down
19 changes: 10 additions & 9 deletions apis/fluentd/v1alpha1/plugins/params/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,12 @@ func (ps *PluginStore) InsertPairs(key, value string) {
ps.Store[key] = value
}

// The @type parameter specifies the type of the plugin.
// The @type parameter specifies the type of the plugin
func (ps *PluginStore) InsertType(value string) {
ps.InsertPairs("@type", value)
}

// SetIgnorePath will ignore the buffer path.
// SetIgnorePath will ignore the buffer path
func (ps *PluginStore) SetIgnorePath() {
ps.IgnorePath = true
}
Expand All @@ -58,7 +58,7 @@ func (ps *PluginStore) InsertChilds(childs ...*PluginStore) {
}
}

// The total hash string for this plugin store.
// The total hash string for this plugin store
func (ps *PluginStore) Hash() string {
c := NewPluginStore(ps.Name)

Expand All @@ -78,7 +78,7 @@ func (ps *PluginStore) GetTag() string {
return ps.Store["tag"]
}

// Returns the @label value string of this plugin store.
// Returns the @label value string of this plugin store
func (ps *PluginStore) RouteLabel() string {
if ps.Name != "route" {
return ""
Expand All @@ -100,10 +100,10 @@ func (ps *PluginStore) String() string {
}
var buf bytes.Buffer

// Handles the head section.
// Handles the head directive
ps.processHead(&buf)

// The body needs to be indented by two whitespace characters.
// The body needs to be indented by two whitespace characters
parentPrefixWhitespaces := ps.PrefixWhitespaces
ps.setWhitespaces(parentPrefixWhitespaces + IntervalWhitespaces)
ps.processBody(&buf)
Expand All @@ -115,7 +115,7 @@ func (ps *PluginStore) String() string {
}
}

// The tail must be indented in the same format as head.
// The tail must be indented in the same format as head directive
ps.setWhitespaces(parentPrefixWhitespaces)
ps.processTail(&buf)

Expand All @@ -126,7 +126,7 @@ func (ps *PluginStore) setWhitespaces(curentWhitespaces string) {
ps.PrefixWhitespaces = curentWhitespaces
}

// processes head, i.e: <match xx>
// write the head directive to buffer, i.e.: <match xx>
func (ps *PluginStore) processHead(buf *bytes.Buffer) {
var head string
switch PluginName(ps.Name) {
Expand Down Expand Up @@ -173,11 +173,12 @@ func (ps *PluginStore) processBody(buf *bytes.Buffer) {
buf.WriteString(body)
}

// processes the tail
// write the tail directive to the buffer, i.e.: </match>
func (ps *PluginStore) processTail(buf *bytes.Buffer) {
buf.WriteString(fmt.Sprintf("%s</%s>\n", ps.PrefixWhitespaces, ps.Name))
}

// decide to return the head directive with our without a filter - <match> or <match xx>
func (ps *PluginStore) headFmtSprintf(value string) string {
if value != "" {
return fmt.Sprintf("%s<%s %s>\n", ps.PrefixWhitespaces, ps.Name, value)
Expand Down
10 changes: 10 additions & 0 deletions apis/fluentd/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

92 changes: 92 additions & 0 deletions charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_fluentds.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1262,6 +1262,98 @@ spec:
type: object
type: object
type: object
defaultFilterSelector:
description: Select cluster filter plugins used to filter for the
default cluster output
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements.
The requirements are ANDed.
items:
description: A label selector requirement is a selector that
contains values, a key, and an operator that relates the key
and values.
properties:
key:
description: key is the label key that the selector applies
to.
type: string
operator:
description: operator represents a key's relationship to
a set of values. Valid operators are In, NotIn, Exists
and DoesNotExist.
type: string
values:
description: values is an array of string values. If the
operator is In or NotIn, the values array must be non-empty.
If the operator is Exists or DoesNotExist, the values
array must be empty. This array is replaced during a strategic
merge patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
matchLabels:
additionalProperties:
type: string
description: matchLabels is a map of {key,value} pairs. A single
{key,value} in the matchLabels map is equivalent to an element
of matchExpressions, whose key field is "key", the operator
is "In", and the values array contains only "value". The requirements
are ANDed.
type: object
type: object
x-kubernetes-map-type: atomic
defaultOutputSelector:
description: Select cluster output plugins used to send all logs that
did not match a route to the matching outputs
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements.
The requirements are ANDed.
items:
description: A label selector requirement is a selector that
contains values, a key, and an operator that relates the key
and values.
properties:
key:
description: key is the label key that the selector applies
to.
type: string
operator:
description: operator represents a key's relationship to
a set of values. Valid operators are In, NotIn, Exists
and DoesNotExist.
type: string
values:
description: values is an array of string values. If the
operator is In or NotIn, the values array must be non-empty.
If the operator is Exists or DoesNotExist, the values
array must be empty. This array is replaced during a strategic
merge patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
matchLabels:
additionalProperties:
type: string
description: matchLabels is a map of {key,value} pairs. A single
{key,value} in the matchLabels map is equivalent to an element
of matchExpressions, whose key field is "key", the operator
is "In", and the values array contains only "value". The requirements
are ANDed.
type: object
type: object
x-kubernetes-map-type: atomic
disableService:
description: By default will build the related service according to
the globalinputs definition.
Expand Down
92 changes: 92 additions & 0 deletions config/crd/bases/fluentd.fluent.io_fluentds.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1262,6 +1262,98 @@ spec:
type: object
type: object
type: object
defaultFilterSelector:
description: Select cluster filter plugins used to filter for the
default cluster output
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements.
The requirements are ANDed.
items:
description: A label selector requirement is a selector that
contains values, a key, and an operator that relates the key
and values.
properties:
key:
description: key is the label key that the selector applies
to.
type: string
operator:
description: operator represents a key's relationship to
a set of values. Valid operators are In, NotIn, Exists
and DoesNotExist.
type: string
values:
description: values is an array of string values. If the
operator is In or NotIn, the values array must be non-empty.
If the operator is Exists or DoesNotExist, the values
array must be empty. This array is replaced during a strategic
merge patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
matchLabels:
additionalProperties:
type: string
description: matchLabels is a map of {key,value} pairs. A single
{key,value} in the matchLabels map is equivalent to an element
of matchExpressions, whose key field is "key", the operator
is "In", and the values array contains only "value". The requirements
are ANDed.
type: object
type: object
x-kubernetes-map-type: atomic
defaultOutputSelector:
description: Select cluster output plugins used to send all logs that
did not match a route to the matching outputs
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements.
The requirements are ANDed.
items:
description: A label selector requirement is a selector that
contains values, a key, and an operator that relates the key
and values.
properties:
key:
description: key is the label key that the selector applies
to.
type: string
operator:
description: operator represents a key's relationship to
a set of values. Valid operators are In, NotIn, Exists
and DoesNotExist.
type: string
values:
description: values is an array of string values. If the
operator is In or NotIn, the values array must be non-empty.
If the operator is Exists or DoesNotExist, the values
array must be empty. This array is replaced during a strategic
merge patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
matchLabels:
additionalProperties:
type: string
description: matchLabels is a map of {key,value} pairs. A single
{key,value} in the matchLabels map is equivalent to an element
of matchExpressions, whose key field is "key", the operator
is "In", and the values array contains only "value". The requirements
are ANDed.
type: object
type: object
x-kubernetes-map-type: atomic
disableService:
description: By default will build the related service according to
the globalinputs definition.
Expand Down
Loading

0 comments on commit 251932e

Please sign in to comment.