✨ [environment] Manipulate environment variable collections (#354)

<!--
Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors.
All rights reserved.
SPDX-License-Identifier: Apache-2.0
-->
### Description
- `[platform]` Define a portable way to do parameter substitution
- `[environment]` Add utilities to manipulate environment variables such
as merges, sorting, ensure uniqueness



### Test Coverage

<!--
Please put an `x` in the correct box e.g. `[x]` to indicate the testing
coverage of this change.
-->

- [x]  This change is covered by existing or additional automated tests.
- [ ] Manual testing has been performed (and evidence provided) as
automated testing was not feasible.
- [ ] Additional tests are not required for this change (e.g.
documentation update).

changes/20231029203453.feature

@@ -0,0 +1 @@
+:sparkles: `[platform]` Add parameter substitution utilities for all platform

changes/20231029203455.feature

@@ -0,0 +1 @@
+:sparkles: `[environment]` Add utilities to manipulate environment variable collections

changes/20231030114112.feature

@@ -0,0 +1 @@
+:sparkles: [`platform`] Add portable variable name validation

utils/environment/envvar.go

@@ -3,9 +3,11 @@ package environment
 import (
 	"fmt"
 	"regexp"
+	"sort"
 	"strings"
 
 	validation "github.com/go-ozzo/ozzo-validation/v4"
+	"golang.org/x/exp/maps"
 
 	"github.com/ARM-software/golang-utils/utils/commonerrors"
 	"github.com/ARM-software/golang-utils/utils/platform"
@@ -108,8 +110,8 @@ func NewEnvironmentVariable(key, value string) IEnvironmentVariable {
 
 }
 
-// CloneEnvironementVariable returns a clone of the environment variable.
-func CloneEnvironementVariable(envVar IEnvironmentVariable) IEnvironmentVariable {
+// CloneEnvironmentVariable returns a clone of the environment variable.
+func CloneEnvironmentVariable(envVar IEnvironmentVariable) IEnvironmentVariable {
 	if envVar == nil {
 		return nil
 	}
@@ -159,6 +161,16 @@ func FindEnvironmentVariable(envvar string, envvars ...IEnvironmentVariable) (IE
 	return nil, fmt.Errorf("%w: environment variable '%v' not set", commonerrors.ErrNotFound, envvar)
 }
 
+// FindFoldEnvironmentVariable looks for an environment variable in a list similarly to FindEnvironmentVariable but without case-sensitivity.
+func FindFoldEnvironmentVariable(envvar string, envvars ...IEnvironmentVariable) (IEnvironmentVariable, error) {
+	for i := range envvars {
+		if strings.EqualFold(envvars[i].GetKey(), envvar) {
+			return envvars[i], nil
+		}
+	}
+	return nil, fmt.Errorf("%w: environment variable '%v' not set", commonerrors.ErrNotFound, envvar)
+}
+
 // ExpandEnvironmentVariables returns a list of environment variables with their value being expanded.
 // Expansion assumes that all the variables are present in the envvars list.
 // If recursive is set to true, then expansion is performed recursively over the variable list.
@@ -186,3 +198,45 @@ func ExpandEnvironmentVariable(recursive bool, envVarToExpand IEnvironmentVariab
 	expandedEnvVar = NewEnvironmentVariable(envVarToExpand.GetKey(), platform.ExpandParameter(envVarToExpand.GetValue(), mappingFunc, recursive))
 	return
 }
+
+// UniqueEnvironmentVariables returns a list of unique environment variables.
+// caseSensitive states whether two same keys but with different case should be both considered unique.
+func UniqueEnvironmentVariables(caseSensitive bool, envvars ...IEnvironmentVariable) (uniqueEnvVars []IEnvironmentVariable) {
+	uniqueSet := map[string]IEnvironmentVariable{}
+	recordUniqueEnvVar(caseSensitive, envvars, uniqueSet)
+	uniqueEnvVars = maps.Values(uniqueSet)
+	return
+}
+
+// SortEnvironmentVariables sorts a list of environment variable alphabetically no matter the case.
+func SortEnvironmentVariables(envvars []IEnvironmentVariable) {
+	if len(envvars) == 0 {
+		return
+	}
+	sort.SliceStable(envvars, func(i, j int) bool {
+		return strings.ToLower(envvars[i].GetKey()) < strings.ToLower(envvars[j].GetKey())
+	})
+}
+
+// MergeEnvironmentVariableSets merges two sets of environment variables.
+// If both sets have a same environment variable, its value in set 1 will take precedence.
+// caseSensitive states whether two similar keys with different case should be considered as different
+func MergeEnvironmentVariableSets(caseSensitive bool, envvarSet1 []IEnvironmentVariable, envvarSet2 ...IEnvironmentVariable) (mergedEnvVars []IEnvironmentVariable) {
+	mergeSet := map[string]IEnvironmentVariable{}
+	recordUniqueEnvVar(caseSensitive, envvarSet1, mergeSet)
+	recordUniqueEnvVar(caseSensitive, envvarSet2, mergeSet)
+	mergedEnvVars = maps.Values(mergeSet)
+	return
+}
+
+func recordUniqueEnvVar(caseSensitive bool, envvarSet []IEnvironmentVariable, hashTable map[string]IEnvironmentVariable) {
+	for i := range envvarSet {
+		key := envvarSet[i].GetKey()
+		if !caseSensitive {
+			key = strings.ToLower(key)
+		}
+		if _, contains := hashTable[key]; !contains {
+			hashTable[key] = envvarSet[i]
+		}
+	}
+}

utils/environment/envvar_test.go

@@ -165,14 +165,52 @@ func TestEnvVar_ParseEnvironmentVariables(t *testing.T) {
 	errortest.AssertError(t, err, commonerrors.ErrNotFound)
 }
 
+func TestFindEnvironmentVariable(t *testing.T) {
+	entries := []string{"DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/65357/bus", "HOME=first", "home=second", faker.UUIDHyphenated(), "EDITOR=hx", "logName=josjen01", "LOGNAME=josjen01", "teSt1=Accusantium voluptatem aut sit perferendis consequatur", "TEST1=Perferendis aut accusantium voluptatem sit consequatur.", faker.Word()}
+	environmentVariables := ParseEnvironmentVariables(entries...)
+	home, err := FindEnvironmentVariable("HOME", environmentVariables...)
+	require.NoError(t, err)
+	assert.Equal(t, "first", home.GetValue())
+	home, err = FindEnvironmentVariable("home", environmentVariables...)
+	require.NoError(t, err)
+	assert.Equal(t, "second", home.GetValue())
+	home, err = FindEnvironmentVariable(faker.Username(), environmentVariables...)
+	require.Error(t, err)
+	errortest.AssertError(t, err, commonerrors.ErrNotFound)
+	assert.Empty(t, home)
+	home, err = FindFoldEnvironmentVariable(faker.Username(), environmentVariables...)
+	require.Error(t, err)
+	errortest.AssertError(t, err, commonerrors.ErrNotFound)
+	assert.Empty(t, home)
+	home, err = FindFoldEnvironmentVariable("home", environmentVariables...)
+	require.NoError(t, err)
+	assert.Equal(t, "first", home.GetValue())
+	test1, err := FindEnvironmentVariable("TEST1", environmentVariables...)
+	require.NoError(t, err)
+	assert.True(t, strings.HasPrefix(test1.GetValue(), "Perferendis"))
+	test1, err = FindFoldEnvironmentVariable("TEST1", environmentVariables...)
+	require.NoError(t, err)
+	assert.True(t, strings.HasPrefix(test1.GetValue(), "Accusantium"))
+}
 func TestExpandEnvironmentVariable(t *testing.T) {
+	env1 := NewEnvironmentVariable("test", platform.SubstituteParameter("test"))
+	expanded1 := ExpandEnvironmentVariable(true, env1)
+	require.NotEmpty(t, expanded1)
+	assert.True(t, env1.Equal(expanded1))
+	expanded1 = ExpandEnvironmentVariable(true, env1, env1)
+	require.NotEmpty(t, expanded1)
+	assert.True(t, env1.Equal(expanded1))
+	env2 := NewEnvironmentVariable("test2", platform.SubstituteParameter("test3"))
+	env3 := NewEnvironmentVariable("test3", platform.SubstituteParameter("test"))
+	expanded2 := ExpandEnvironmentVariable(true, env2, env1, env2, env3)
+	require.NotEmpty(t, expanded2)
+	assert.False(t, env1.Equal(expanded2))
+	assert.Equal(t, expanded2.GetValue(), env1.GetValue())
+}
+
+func TestExpandEnvironmentVariables(t *testing.T) {
 	username := faker.Username()
-	var entries []string
-	if platform.IsWindows() {
-		entries = []string{"DBUS_SESSION_BUS_ADDRESS=system:path=/run%HOME%/65357/bus/", "HOME=/home/%LOGNAME%", fmt.Sprintf("LOGNAME=%v", username)}
-	} else {
-		entries = []string{"DBUS_SESSION_BUS_ADDRESS=system:path=/run${HOME}/65357/bus/", "HOME=/home/${LOGNAME}", fmt.Sprintf("LOGNAME=%v", username)}
-	}
+	entries := []string{fmt.Sprintf("DBUS_SESSION_BUS_ADDRESS=system:path=/run%v/65357/bus/", platform.SubstituteParameter("HOME")), fmt.Sprintf("HOME=/home/%v", platform.SubstituteParameter("LOGNAME")), fmt.Sprintf("LOGNAME=%v", username)}
 	expandedEnvironmentVariables := ExpandEnvironmentVariables(true, ParseEnvironmentVariables(entries...)...)
 	require.NotEmpty(t, expandedEnvironmentVariables)
 	logname, err := FindEnvironmentVariable("LOGNAME", expandedEnvironmentVariables...)
@@ -182,3 +220,93 @@ func TestExpandEnvironmentVariable(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, fmt.Sprintf("system:path=/run/home/%v/65357/bus/", username), dbus.GetValue())
 }
+
+func TestSortEnvironmentVariables(t *testing.T) {
+	entries := []string{fmt.Sprintf("ccc%v=%v", faker.Username(), faker.Sentence()), fmt.Sprintf("Aaodasdoah%v=%v", faker.Username(), faker.Sentence()), "b=second", fmt.Sprintf("Za%v=%v", faker.Word(), faker.Sentence())}
+	envVars := ParseEnvironmentVariables(entries...)
+	SortEnvironmentVariables(envVars)
+	require.Len(t, envVars, 4)
+	assert.True(t, strings.HasPrefix(envVars[0].String(), "A"))
+	assert.True(t, strings.HasPrefix(envVars[1].String(), "b"))
+	assert.True(t, strings.HasPrefix(envVars[2].String(), "c"))
+	assert.True(t, strings.HasPrefix(envVars[3].String(), "Z"))
+
+	var empty []IEnvironmentVariable
+	SortEnvironmentVariables(empty)
+}
+
+func TestUniqueEnvironmentVariables(t *testing.T) {
+	randomKey := faker.Word()
+	entries := []string{"DBUS_SESSION_BUS_ADDRESS=system:path=/run/65357/bus/", "HOME=first", "home=second", fmt.Sprintf("%v=%v", randomKey, faker.Sentence())}
+	envVars := ParseEnvironmentVariables(entries...)
+	require.NotEmpty(t, envVars)
+	assert.Len(t, envVars, 4)
+	assert.Empty(t, UniqueEnvironmentVariables(true))
+	uniqueEnvVars := UniqueEnvironmentVariables(false, envVars...)
+	require.NotEmpty(t, uniqueEnvVars)
+	assert.NotEqual(t, uniqueEnvVars, envVars)
+	assert.Len(t, uniqueEnvVars, 3)
+	home, err := FindEnvironmentVariable("HOME", uniqueEnvVars...)
+	require.NoError(t, err)
+	assert.Equal(t, "first", home.GetValue())
+
+	uniqueEnvVars2 := UniqueEnvironmentVariables(false, uniqueEnvVars...)
+	require.NotEmpty(t, uniqueEnvVars2)
+	SortEnvironmentVariables(uniqueEnvVars2)
+	SortEnvironmentVariables(uniqueEnvVars)
+	assert.EqualValues(t, uniqueEnvVars2, uniqueEnvVars)
+
+	uniqueEnvVars3 := UniqueEnvironmentVariables(true, envVars...)
+	require.NotEmpty(t, uniqueEnvVars3)
+	SortEnvironmentVariables(uniqueEnvVars3)
+	assert.Len(t, uniqueEnvVars3, 4)
+	assert.NotEqualValues(t, uniqueEnvVars3, uniqueEnvVars)
+
+}
+
+func TestMergeEnvironmentVariables(t *testing.T) {
+	randomKey := fmt.Sprintf("B%v", faker.Word())
+	entries1 := []string{fmt.Sprintf("ccc%v=%v", faker.Username(), faker.Sentence()), "HOME=first", "home=second", fmt.Sprintf("%v=%v", randomKey, faker.Sentence())}
+	entries2 := []string{"Zabcd=tmp", "HOME=third", fmt.Sprintf("%v=%v", randomKey, faker.Sentence())}
+	envVars := MergeEnvironmentVariableSets(false, ParseEnvironmentVariables(entries1...), ParseEnvironmentVariables(entries2...)...)
+	require.NotEmpty(t, envVars)
+	assert.Len(t, envVars, 4)
+	home, err := FindEnvironmentVariable("HOME", envVars...)
+	require.NoError(t, err)
+	assert.Equal(t, "first", home.GetValue())
+	home, err = FindEnvironmentVariable("home", envVars...)
+	require.Error(t, err)
+	errortest.AssertError(t, err, commonerrors.ErrNotFound)
+	assert.Empty(t, home)
+
+	uniqueEnvVars := UniqueEnvironmentVariables(false, envVars...)
+	require.NotEmpty(t, uniqueEnvVars)
+	SortEnvironmentVariables(envVars)
+	SortEnvironmentVariables(uniqueEnvVars)
+	assert.EqualValues(t, envVars, uniqueEnvVars)
+	assert.True(t, strings.HasPrefix(envVars[0].String(), "B"))
+	assert.True(t, strings.HasPrefix(envVars[1].String(), "c"))
+	assert.True(t, strings.HasPrefix(envVars[2].String(), "H"))
+	assert.True(t, strings.HasPrefix(envVars[3].String(), "Z"))
+
+	envVars = MergeEnvironmentVariableSets(true, ParseEnvironmentVariables(entries1...), ParseEnvironmentVariables(entries2...)...)
+	require.NotEmpty(t, envVars)
+	assert.Len(t, envVars, 5)
+	home, err = FindEnvironmentVariable("HOME", envVars...)
+	require.NoError(t, err)
+	assert.Equal(t, "first", home.GetValue())
+	home, err = FindEnvironmentVariable("home", envVars...)
+	require.NoError(t, err)
+	assert.Equal(t, "second", home.GetValue())
+}
+
+func TestCloneEnvironmentVariable(t *testing.T) {
+	env1 := NewEnvironmentVariable(faker.Word(), faker.Sentence())
+	clone1 := CloneEnvironmentVariable(env1)
+	assert.Equal(t, env1, env1)
+	assert.False(t, clone1 == env1)
+	assert.True(t, clone1.Equal(env1))
+	assert.True(t, env1.Equal(clone1))
+	clone2 := CloneEnvironmentVariable(nil)
+	assert.Nil(t, clone2)
+}

utils/platform/os.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	validation "github.com/go-ozzo/ozzo-validation/v4"
 	"github.com/shirou/gopsutil/v3/host"
 	"github.com/shirou/gopsutil/v3/mem"
 
@@ -24,8 +25,44 @@ var (
 	// https://learn.microsoft.com/en-us/previous-versions/troubleshoot/winautomation/product-documentation/best-practices/variables/percentage-character-usage-in-notations
 	// https://ss64.com/nt/syntax-replace.html
 	windowsVariableExpansionRegexStr = `%(?P<variable>[^:=]*)(:(?P<StrToFind>.*)=(?P<NewString>.*))?%`
+	// UnixVariableNameRegexString defines the schema for variable names on Unix.
+	// See https://www.gnu.org/software/bash/manual/bash.html#index-name and https://mywiki.wooledge.org/BashFAQ/006
+	UnixVariableNameRegexString = "^[a-zA-Z_][a-zA-Z_0-9]*$"
+	// WindowsVariableNameRegexString defines the schema for variable names on Windows.
+	// See https://ss64.com/nt/syntax-variables.html
+	WindowsVariableNameRegexString = "^[A-Za-z#$'()*+,.?@\\[\\]_`{}~][A-Za-z0-9#$'()*+,.?@\\[\\]_`{}~\\s]*$"
+	errVariableNameInvalid         = validation.NewError("validation_is_variable_name", "must be a valid variable name")
+	// IsWindowsVariableName defines a validation rule for variable names on Windows for use with github.com/go-ozzo/ozzo-validation
+	IsWindowsVariableName = validation.NewStringRuleWithError(isWindowsVarName, errVariableNameInvalid)
+	// IsUnixVariableName defines a validation rule for variable names on Unix for use with github.com/go-ozzo/ozzo-validation
+	IsUnixVariableName = validation.NewStringRuleWithError(isUnixVarName, errVariableNameInvalid)
+	// IsVariableName defines a validation rule for variable names for use with github.com/go-ozzo/ozzo-validation
+	IsVariableName = validation.NewStringRuleWithError(isVarName, errVariableNameInvalid)
 )
 
+func isWindowsVarName(value string) bool {
+	if validation.Required.Validate(value) != nil {
+		return false
+	}
+	regex := regexp.MustCompile(WindowsVariableNameRegexString)
+	return regex.MatchString(value)
+}
+
+func isUnixVarName(value string) bool {
+	if validation.Required.Validate(value) != nil {
+		return false
+	}
+	regex := regexp.MustCompile(UnixVariableNameRegexString)
+	return regex.MatchString(value)
+}
+
+func isVarName(value string) bool {
+	if IsWindows() {
+		return isWindowsVarName(value)
+	}
+	return isUnixVarName(value)
+}
+
 // ConvertError converts a platform error into a commonerrors
 func ConvertError(err error) error {
 	switch {
@@ -179,6 +216,44 @@ func GetRAM() (ram RAM, err error) {
 	return
 }
 
+// SubstituteParameter performs parameter substitution on all platforms.
+// - the first element is the parameter to substitute
+// - if find and replace is also wanted, pass the pattern and the replacement as following arguments in that order.
+func SubstituteParameter(parameter ...string) string {
+	if IsWindows() {
+		return SubstituteParameterWindows(parameter...)
+	}
+	return SubstituteParameterUnix(parameter...)
+}
+
+// SubstituteParameterUnix performs Unix parameter substitution:
+// See https://tldp.org/LDP/abs/html/parameter-substitution.html
+// - the first element is the parameter to substitute
+// - if find and replace is also wanted, pass the pattern and the replacement as following arguments in that order.
+func SubstituteParameterUnix(parameter ...string) string {
+	if len(parameter) < 1 || !isUnixVarName(parameter[0]) {
+		return "${}"
+	}
+	if len(parameter) < 3 || parameter[1] == "" {
+		return fmt.Sprintf("${%v}", parameter[0])
+	}
+	return fmt.Sprintf("${%v//%v/%v}", parameter[0], parameter[1], parameter[2])
+}
+
+// SubstituteParameterWindows performs Windows parameter substitution:
+// See https://ss64.com/nt/syntax-replace.html
+// - the first element is the parameter to substitute
+// - if find and replace is also wanted, pass the pattern and the replacement as following arguments in that order.
+func SubstituteParameterWindows(parameter ...string) string {
+	if len(parameter) < 1 || !isWindowsVarName(parameter[0]) {
+		return "%%"
+	}
+	if len(parameter) < 3 || parameter[1] == "" {
+		return "%" + parameter[0] + "%"
+	}
+	return "%" + fmt.Sprintf("%v:%v=%v", parameter[0], parameter[1], parameter[2]) + "%"
+}
+
 // ExpandParameter expands a variable expressed in a string `s` with its value returned by the mapping function.
 // If the mapping function returns a string with variables, it will expand them too if recursive is set to true.
 func ExpandParameter(s string, mappingFunc func(string) (string, bool), recursive bool) string {
@@ -212,6 +287,7 @@ func recursiveMapping(mappingFunc func(string) (string, bool), expansionFunc fun
 
 // ExpandUnixParameter expands a ${param} or $param in `s` based on the mapping function
 // See https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html
+// os.Expand is used under the bonnet and so, only basic parameter substitution is performed.
 // TODO if os.Expand is not good enough, consider using other libraries such as https://github.com/ganbarodigital/go_shellexpand or https://github.com/mvdan/sh
 func ExpandUnixParameter(s string, mappingFunc func(string) (string, bool), recursive bool) string {
 	mapping := newMappingFunc(recursive, mappingFunc, expandUnixParameter)
@@ -229,6 +305,7 @@ func expandUnixParameter(s string, mappingFunc func(string) (string, bool)) stri
 // See https://learn.microsoft.com/en-us/previous-versions/troubleshoot/winautomation/product-documentation/best-practices/variables/percentage-character-usage-in-notations
 // https://devblogs.microsoft.com/oldnewthing/20060823-00/?p=29993
 // https://github.com/golang/go/issues/24848
+// WARNING: currently the function only works with one parameter substitution in `s`.
 func ExpandWindowsParameter(s string, mappingFunc func(string) (string, bool), recursive bool) string {
 	mapping := newMappingFunc(recursive, mappingFunc, expandWindowsParameter)
 	return expandWindowsParameter(s, mapping)