From d54989c2cf70aa79693fe9a1b57bbf40d8999aca Mon Sep 17 00:00:00 2001 From: Roman Dodin Date: Thu, 20 Feb 2025 12:10:45 +0100 Subject: [PATCH] Envsubst for startup configs (#2471) * envsubst for startup configs * correct vars * squash linter errors * template and envsubst partials for sros * added sros templating doc --- clab/config.go | 15 ++--- docs/manual/kinds/vr-sros.md | 11 ++- docs/manual/nodes.md | 127 ++++++++++++++++++----------------- docs/stylesheets/extra.css | 10 +++ nodes/default_node.go | 22 ++---- nodes/srl/srl.go | 9 ++- nodes/vr_sros/vr-sros.go | 8 ++- utils/template.go | 34 +++++++++- 8 files changed, 143 insertions(+), 93 deletions(-) diff --git a/clab/config.go b/clab/config.go index a07c0dcc9..a4759b628 100644 --- a/clab/config.go +++ b/clab/config.go @@ -259,7 +259,7 @@ func (c *CLab) createNodeCfg(nodeName string, nodeDef *types.NodeDefinition, idx // processStartupConfig processes the raw path of the startup-config as it is defined in the topology file. // It handles remote files, local files and embedded configs. -// Returns an absolute path to the startup-config file. +// As a result the `nodeCfg.StartupConfig` will be set to an absPath of the startup config file. func (c *CLab) processStartupConfig(nodeCfg *types.NodeConfig) error { // replace __clabNodeName__ magic var in startup-config path with node short name r := c.magicVarReplacer(nodeCfg.ShortName) @@ -302,7 +302,7 @@ func (c *CLab) processStartupConfig(nodeCfg *types.NodeConfig) error { return err } - // adjust the nodeconfig by pointing startup-config to the local downloaded file + // adjust the NodeConfig by pointing startup-config to the local downloaded file p = absDestFile } } @@ -331,10 +331,8 @@ func (c *CLab) checkTopologyDefinition(ctx context.Context) error { if err = c.verifyDuplicateAddresses(); err != nil { return err } - if err = c.verifyContainersUniqueness(ctx); err != nil { - return err - } - return nil + + return c.verifyContainersUniqueness(ctx) } // verifyRootNetNSLinks makes sure, that there will be no overlap in @@ -376,7 +374,8 @@ func (c *CLab) verifyRootNetNSLinks() error { // appear only once. func (c *CLab) verifyLinks(ctx context.Context) error { var err error - verificationErrors := []error{} + var verificationErrors []error + for _, e := range c.Endpoints { err = e.Verify(ctx, c.globalRuntime().Config().VerifyLinkParams) if err != nil { @@ -468,7 +467,7 @@ func (c *CLab) verifyContainersUniqueness(ctx context.Context) error { return nil } - dups := []string{} + var dups []string for _, n := range c.Nodes { if n.Config().SkipUniquenessCheck { continue diff --git a/docs/manual/kinds/vr-sros.md b/docs/manual/kinds/vr-sros.md index 90938724d..d81ba4716 100644 --- a/docs/manual/kinds/vr-sros.md +++ b/docs/manual/kinds/vr-sros.md @@ -215,10 +215,14 @@ Nokia SR OS nodes come up with a basic "blank" configuration where only the card #### User-defined config -SR OS nodes launched with hellt/vrnetlab come up with some basic configuration that configures the management interfaces, line cards, mdas and power modules. This configuration is applied right after the node is booted. +SR OS nodes launched with [hellt/vrnetlab](https://github.com/hellt/vrnetlab) come up with some basic configuration that configures the management interfaces, line cards, mdas and power modules. This configuration is applied right after the node is booted. Since this initial configuration is meant to provide a bare minimum configuration to make the node operational, users will likely want to apply their own configuration to the node to enable some features or to configure some interfaces. This can be done by providing a user-defined configuration file using [`startup-config`](../nodes.md#startup-config) property of the node/kind. +/// tip +Configuration text can contain Go template logic as well as make use of [environment variables](../topo-def-file.md#environment-variables) allowing for runtime customization of the configuration. +/// + ##### Full startup-config When a user provides a path to a file that has a complete configuration for the node, containerlab will copy that file to the lab directory for that specific node under the `/tftpboot/config.txt` name and mount that dir to the container. This will result in this config to act as a startup-config for the node: @@ -232,8 +236,9 @@ topology: startup-config: myconfig.txt ``` -!!!note - With the above configuration, the node will boot with the configuration specified in `myconfig.txt`, no other configuration will be applied. You have to provision interfaces, cards, power-shelves, etc. yourself. +/// note +With the above configuration, the node will boot with the configuration specified in `myconfig.txt`, no other configuration will be applied. You have to provision interfaces, cards, power-shelves, etc. yourself. +/// ##### Partial startup-config diff --git a/docs/manual/nodes.md b/docs/manual/nodes.md index d6abbbc86..ed6dfc43b 100644 --- a/docs/manual/nodes.md +++ b/docs/manual/nodes.md @@ -125,6 +125,9 @@ For all Network OS kinds, it's possible to provide startup configuration that th 1. As a path to a file that is available on the host machine and contains the config blob that the node understands. 2. As an embedded config blob that is provided as a multiline string. +The environment variables in the config files will be substituted before the config is applied. This gives the user the ability to customize the config at runtime based on the present environment variables. +The env vars are defined as explained in the [Topology Definition section](topo-def-file.md#environment-variables). + #### path to a startup-config file When a path to a startup-config file is provided, containerlab either mounts the file to the container by a path that NOS expects to have its startup-config file, or it will apply the config via using the NOS-dependent method. @@ -138,72 +141,74 @@ topology: Check the particular kind documentation to see if the startup-config is supported and how it is applied. -???info "Startup-config path variable" - By default, the startup-config references are either provided as an absolute or a relative (to the current working dir) path to the file to be used. - - Consider a two-node lab `mylab.clab.yml` with seed configs that the user may wish to use in their lab. A user could create a directory for such files similar to this: - - ``` - . - ├── cfgs - │   ├── node1.partial.cfg - │   └── node2.partial.cfg - └── mylab.clab.yml - - 2 directories, 3 files - ``` - - Then to leverage these configs, the node could be configured with startup-config references like this: - - ```yaml - name: mylab - topology: - nodes: - node1: - startup-config: cfgs/node1.partial.cfg - node2: - startup-config: cfgs/node2.partial.cfg - ``` +/// details | Startup-config path variable + By default, the startup-config references are either provided as an absolute or a relative (to the current working dir) path to the file to be used. + + Consider a two-node lab `mylab.clab.yml` with seed configs that the user may wish to use in their lab. A user could create a directory for such files similar to this: + + ``` + . + ├── cfgs + │   ├── node1.partial.cfg + │   └── node2.partial.cfg + └── mylab.clab.yml + + 2 directories, 3 files + ``` + + Then to leverage these configs, the node could be configured with startup-config references like this: + + ```yaml + name: mylab + topology: + nodes: + node1: + startup-config: cfgs/node1.partial.cfg + node2: + startup-config: cfgs/node2.partial.cfg + ``` + + while this configuration is correct, it might be considered verbose as the number of nodes grows. To remove this verbosity, the users can use a special variable `__clabNodeName__` in their startup-config paths. This variable will expand to the node-name for the parent node that the startup-config reference falls under. + + ```yaml + name: mylab + topology: + nodes: + node1: + startup-config: cfg/__clabNodeName__.partial.cfg + node2: + startup-config: cfgs/__clabNodeName__.partial.cfg + ``` - while this configuration is correct, it might be considered verbose as the number of nodes grows. To remove this verbosity, the users can use a special variable `__clabNodeName__` in their startup-config paths. This variable will expand to the node-name for the parent node that the startup-config reference falls under. + The `__clabNodeName__` variable can also be used in the kind and default sections of the config. Using the same directory structure from the example above, the following shows how to use the magic variable for a kind. - ```yaml - name: mylab - topology: - nodes: - node1: - startup-config: cfg/__clabNodeName__.partial.cfg - node2: - startup-config: cfgs/__clabNodeName__.partial.cfg - ``` - - The `__clabNodeName__` variable can also be used in the kind and default sections of the config. Using the same directory structure from the example above, the following shows how to use the magic variable for a kind. + ```yaml + name: mylab + topology: + defaults: + kind: nokia_srlinux + kinds: + nokia_srlinux: + startup-config: cfgs/__clabNodeName__.partial.cfg + nodes: + node1: + node2: + ``` - ```yaml - name: mylab - topology: - defaults: - kind: nokia_srlinux - kinds: - nokia_srlinux: - startup-config: cfgs/__clabNodeName__.partial.cfg - nodes: - node1: - node2: - ``` + The following example shows how one would do it using defaults. - The following example shows how one would do it using defaults. + ```yaml + name: mylab + topology: + defaults: + kind: nokia_srlinux + startup-config: cfgs/__clabNodeName__.partial.cfg + nodes: + node1: + node2: + ``` - ```yaml - name: mylab - topology: - defaults: - kind: nokia_srlinux - startup-config: cfgs/__clabNodeName__.partial.cfg - nodes: - node1: - node2: - ``` +/// #### embedded startup-config diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 1b4c9b784..7384c2b4b 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -12,6 +12,16 @@ fill: var(--logo-outline-color); } +.md-grid { + /* a slightly increased width for the main content ~1300px */ + max-width: 65rem; +} + +.md-post--excerpt { + /* a slightly increased width for the main content ~1300px */ + max-width: 40rem; +} + .mdx-content__footer { margin-top: 20px; text-align: center; diff --git a/nodes/default_node.go b/nodes/default_node.go index aa1e3836d..31b1c3278 100644 --- a/nodes/default_node.go +++ b/nodes/default_node.go @@ -5,7 +5,6 @@ package nodes import ( - "bytes" "context" "fmt" "os" @@ -13,8 +12,8 @@ import ( "path/filepath" "regexp" "strconv" + "strings" "sync" - "text/template" "github.com/charmbracelet/log" "github.com/containernetworking/plugins/pkg/ns" @@ -349,8 +348,8 @@ func (d *DefaultNode) VerifyStartupConfig(topoDir string) error { } // GenerateConfig generates configuration for the nodes -// out of the template based on the node configuration and saves the result to dst. -func (d *DefaultNode) GenerateConfig(dst, templ string) error { +// out of the template `t` based on the node configuration and saves the result to dst. +func (d *DefaultNode) GenerateConfig(dst, t string) error { // If the config file is already present in the node dir // we do not regenerate the config unless EnforceStartupConfig is explicitly set to true and startup-config points to a file // this will persist the changes that users make to a running config when booted from some startup config @@ -365,27 +364,20 @@ func (d *DefaultNode) GenerateConfig(dst, templ string) error { return nil } - log.Debugf("generating config for node %s from file %s", d.Cfg.ShortName, d.Cfg.StartupConfig) + log.Debug("Generating config", "node", d.Cfg.ShortName, "file", d.Cfg.StartupConfig) - tpl, err := template.New(filepath.Base(d.Cfg.StartupConfig)).Funcs(utils.TemplateFuncs).Parse(templ) + cfgBuf, err := utils.SubstituteEnvsAndTemplate(strings.NewReader(t), d.Cfg) if err != nil { return err } - - dstBytes := new(bytes.Buffer) - - err = tpl.Execute(dstBytes, d.Cfg) - if err != nil { - return err - } - log.Debugf("node '%s' generated config: %s", d.Cfg.ShortName, dstBytes.String()) + log.Debugf("node '%s' generated config: %s", d.Cfg.ShortName, cfgBuf.String()) f, err := os.Create(dst) if err != nil { return err } - _, err = f.Write(dstBytes.Bytes()) + _, err = f.Write(cfgBuf.Bytes()) if err != nil { f.Close() return err diff --git a/nodes/srl/srl.go b/nodes/srl/srl.go index 7e6006180..4749bf7b1 100644 --- a/nodes/srl/srl.go +++ b/nodes/srl/srl.go @@ -445,7 +445,7 @@ func (s *srl) createSRLFiles() error { var cfgTemplate string cfgPath := filepath.Join(s.Cfg.LabDir, "config", "config.json") if s.Cfg.StartupConfig != "" { - log.Debugf("Reading startup-config %s", s.Cfg.StartupConfig) + log.Debug("Reading startup-config", "file", s.Cfg.StartupConfig) c, err := os.ReadFile(s.Cfg.StartupConfig) if err != nil { @@ -461,7 +461,12 @@ func (s *srl) createSRLFiles() error { log.Debugf("startup-config passed to %s is in the CLI format. Will apply it in post-deploy stage", s.Cfg.ShortName) - s.startupCliCfg = c + cBuf, err := utils.SubstituteEnvsAndTemplate(bytes.NewReader(c), s.Cfg) + if err != nil { + return err + } + + s.startupCliCfg = cBuf.Bytes() // no need to generate and mount startup-config passed in a CLI format // as we will apply it over the top of a default config in the post deploy stage diff --git a/nodes/vr_sros/vr-sros.go b/nodes/vr_sros/vr-sros.go index 039e407ff..8a218a287 100644 --- a/nodes/vr_sros/vr-sros.go +++ b/nodes/vr_sros/vr-sros.go @@ -260,12 +260,12 @@ func (s *vrSROS) applyPartialConfig(ctx context.Context, addr, platformName, var err error var d *network.Driver - configContent, err := io.ReadAll(config) + configContent, err := utils.SubstituteEnvsAndTemplate(config, s.Cfg) if err != nil { return err } - configContentStr := string(configContent) + configContentStr := configContent.String() // check file contains content, otherwise exit early if strings.TrimSpace(configContentStr) == "" { @@ -288,7 +288,9 @@ func (s *vrSROS) applyPartialConfig(ctx context.Context, addr, platformName, case <-ctx.Done(): return fmt.Errorf("%s: timed out waiting to accept configs", addr) default: - sl := log.StandardLog() + sl := log.StandardLog(log.StandardLogOptions{ + ForceLevel: log.DebugLevel, + }) li, err := scraplilogging.NewInstance( scraplilogging.WithLevel("debug"), scraplilogging.WithLogger(sl.Print)) diff --git a/utils/template.go b/utils/template.go index d008b41e6..cff8fc282 100644 --- a/utils/template.go +++ b/utils/template.go @@ -1,13 +1,17 @@ package utils import ( + "bytes" "encoding/json" "fmt" + "io" "math" "reflect" "strconv" "strings" "text/template" + + "github.com/hellt/envsubst" ) var TemplateFuncs = template.FuncMap{ @@ -23,7 +27,7 @@ func toJson(v any) string { return string(a) } -func toJsonPretty(v any, prefix string, indent string) string { +func toJsonPretty(v any, prefix, indent string) string { a, _ := json.MarshalIndent(v, prefix, indent) return string(a) } @@ -199,3 +203,31 @@ func strToFloat64(str string) (float64, error) { return float64(iv), nil } + +// SubstituteEnvsAndTemplate substitutes environment variables and template the reader `r` +// with the `data` template data. +func SubstituteEnvsAndTemplate(r io.Reader, data any) (*bytes.Buffer, error) { + b, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + // expand env vars in `b` if any were set + // do not replace vars initialized with defaults + // and do not replace vars that are not set + b, err = envsubst.BytesRestrictedNoReplace(b, false, false, true, true) + if err != nil { + return nil, err + } + + t, err := template.New("template").Funcs(TemplateFuncs).Parse(string(b)) + if err != nil { + return nil, err + } + + buf := new(bytes.Buffer) + + t.Execute(buf, data) + + return buf, nil +}