feat(configuration): file filters (#4515)

This adds experimental file filters which are not guaranteed under our stability policies. These filters take effect after reading the files and before parsing their content.

cmd/authelia-gen/templates.go

@@ -5,18 +5,14 @@ import (
 	"fmt"
 	"strings"
 	"text/template"
+
+	"github.com/authelia/authelia/v4/internal/templates"
 )
 
 //go:embed templates/*
 var templatesFS embed.FS
 
 var (
-	funcMap = template.FuncMap{
-		"stringsContains": strings.Contains,
-		"join":            strings.Join,
-		"joinX":           fmJoinX,
-	}
-
 	tmplCodeConfigurationSchemaKeys = template.Must(newTMPL("internal_configuration_schema_keys.go"))
 	tmplGitHubIssueTemplateBug      = template.Must(newTMPL("github_issue_template_bug_report.yml"))
 	tmplIssueTemplateFeature        = template.Must(newTMPL("github_issue_template_feature.yml"))
@@ -27,33 +23,14 @@ var (
 	tmplServer                      = template.Must(newTMPL("server_gen.go"))
 )
 
-func fmJoinX(elems []string, sep string, n int, p string) string {
-	buf := strings.Builder{}
-
-	c := 0
-	e := len(elems) - 1
-
-	for i := 0; i <= e; i++ {
-		if c+len(elems[i])+1 > n {
-			c = 0
-
-			buf.WriteString(p)
-		}
-
-		c += len(elems[i]) + 1
-
-		buf.WriteString(elems[i])
-
-		if i < e {
-			buf.WriteString(sep)
-		}
-	}
-
-	return buf.String()
-}
-
 func newTMPL(name string) (tmpl *template.Template, err error) {
-	return template.New(name).Funcs(funcMap).Parse(mustLoadTmplFS(name))
+	return template.New(name).
+		Funcs(template.FuncMap{
+			"stringsContains": strings.Contains,
+			"join":            strings.Join,
+			"joinX":           templates.StringJoinXFunc,
+		}).
+		Parse(mustLoadTmplFS(name))
 }
 
 func mustLoadTmplFS(tmpl string) string {

docs/content/en/configuration/methods/files.md

@@ -124,3 +124,163 @@ spec:
 See the Kubernetes [workloads documentation](https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates) or the
 [Container API docs](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#container-v1-core) for more
 information.
+
+## File Filters
+
+Experimental file filters exist which allow modification of all configuration files after reading them from the
+filesystem but before parsing their content. These filters are _**NOT**_ covered by our
+[Standard Versioning Policy](../../policies/versioning.md). There __*WILL*__ be a point where the name of the CLI
+argument or environment variable will change and usage of these will either break or just not work.
+
+The filters are configured as a list of filter names by the `--config.experimental.filters` CLI argument and
+`X_AUTHELIA_CONFIG_EXPERIMENTAL_FILTERS` environment variable. We recommend using the environment variable as it ensures
+commands executed from the container use the same filters. If both the CLI argument and environment variable are used
+the environment variable is completely ignored.
+
+Filters can either be used on their own, in combination, or not at all. The filters are processed in order as they are
+defined.
+
+Examples:
+
+```bash
+authelia --config config.yml --config.experimental.filters expand-env,template
+```
+
+```text
+X_AUTHELIA_CONFIG_EXPERIMENTAL_FILTERS=expand-env,template
+```
+
+### Expand Environment Variable Filter
+
+The name used to enable this filter is `expand-env`.
+
+This filter is the most common filter type used by many other applications. It is similar to using `envsubst` where it
+replaces a string like `$EXAMPLE` or `${EXAMPLE}` with the value of the `EXAMPLE` environment variable.
+
+### Go Template Filter
+
+The name used to enable this filter is `template`.
+
+This filter uses the [Go template engine](https://pkg.go.dev/text/template) to render the configuration files. It uses
+similar syntax to Jinja2 templates with different function names.
+
+#### Functions
+
+In addition to the standard builtin functions we support several other functions.
+
+##### iterate
+
+The `iterate` function generates a list of numbers from 0 to the input provided. Useful for ranging over a list of
+numbers.
+
+Example:
+
+```yaml
+numbers:
+{{- range $i := iterate 5 }}
+  -  {{ $i }}
+{{- end }}
+```
+
+##### env
+
+The `env` function returns the value of an environment variable or a blank string.
+
+Example:
+
+```yaml
+default_redirection_url: 'https://{{ env "DOMAIN" }}'
+```
+
+##### split
+
+The `split` function splits a string by the separator.
+
+Example:
+
+```yaml
+access_control:
+  rules:
+    - domain: 'app.{{ env "DOMAIN" }}'
+      policy: bypass
+      methods:
+      {{ range _, $method := split "GET,POST" "," }}
+        - {{ $method }}
+      {{ end }}
+```
+
+##### join
+
+The `join` function is similar to [split](#split) but does the complete oppiste, joining an array of strings with a
+separator.
+
+Example:
+
+```yaml
+access_control:
+  rules:
+    - domain: ['app.{{ join (split (env "DOMAINS") ",") "', 'app." }}']
+      policy: bypass
+```
+
+##### contains
+
+The `contains` function is a test function which checks if one string contains another string.
+
+Example:
+
+```yaml
+{{ if contains (env "DOMAIN") "https://" }}
+default_redirection_url: '{{ env "DOMAIN" }}'
+{{ else}}
+default_redirection_url: 'https://{{ env "DOMAIN" }}'
+  {{ end }}
+```
+
+##### hasPrefix
+
+The `hasPrefix` function is a test function which checks if one string is prefixed with another string.
+
+Example:
+
+```yaml
+{{ if hasPrefix (env "DOMAIN") "https://" }}
+default_redirection_url: '{{ env "DOMAIN" }}'
+{{ else}}
+default_redirection_url: 'https://{{ env "DOMAIN" }}'
+{{ end }}
+```
+
+##### hasSuffix
+
+The `hasSuffix` function is a test function which checks if one string is suffixed with another string.
+
+Example:
+
+```yaml
+{{ if hasSuffix (env "DOMAIN") "/" }}
+default_redirection_url: 'https://{{ env "DOMAIN" }}'
+{{ else}}
+default_redirection_url: 'https://{{ env "DOMAIN" }}/'
+{{ end }}
+```
+
+##### lower
+
+The `lower` function is a conversion function which converts a string to all lowercase.
+
+Example:
+
+```yaml
+default_redirection_url: 'https://{{ env "DOMAIN" | lower }}'
+```
+
+##### upper
+
+The `upper` function is a conversion function which converts a string to all uppercase.
+
+Example:
+
+```yaml
+default_redirection_url: 'https://{{ env "DOMAIN" | upper }}'
+```

docs/content/en/policies/versioning.md

@@ -0,0 +1,39 @@
+---
+title: "Versioning Policy"
+description: "The Authelia Versioning Policy which is important reading for administrators"
+date: 2022-12-21T16:46:42+11:00
+draft: false
+images: []
+aliases:
+  - /versioning-policy
+  - /versioning
+---
+
+The __Authelia__ team aims to abide by the [Semantic Versioning 2.0.0](https://semver.org/spec/v2.0.0.html) policy. This
+means that we use the format `major.minor.patch` for our version numbers, where a change to `major` denotes a breaking
+change which will likely require user interaction to upgrade, `minor` which denotes a new feature, and `patch` denotes a
+fix.
+
+It is therefore recommended users do not automatically upgrade the `minor` version without reading the patch notes, and
+it's critically important users do not upgrade the `major` version without reading the patch notes. You should pin your
+version to `4.37` for example to prevent automatic upgrades from negatively affecting you.
+
+## Exceptions
+
+There are exceptions to this versioning policy.
+
+### Breaking Changes
+
+All features which are marked as:
+
+- beta
+- experimental
+
+Notable examples:
+
+- OpenID Connect 1.0
+- File Filters
+
+The reasoning is as we develop these features there may be mistakes and we may need to make a change that should be
+considered breaking. As these features graduate from their status to generally available they will move into our
+standard versioning policy from this exception.

docs/content/en/reference/cli/authelia/authelia.md

@@ -42,6 +42,7 @@ authelia --config /etc/authelia/config/
 
 ```
   -c, --config strings                        configuration files to load
+      --config.experimental.filters strings   Applies filters in order to the configuration file before the YAML parser. Options are 'template', 'expand-env'
   -h, --help                                  help for authelia
 ```
 

internal/commands/configuration.go

@@ -27,8 +27,11 @@ func newCmdWithConfigPreRun(ensureConfigExists, validateKeys, validateConfigurat
 	return func(cmd *cobra.Command, _ []string) {
 		var (
 			logger *logrus.Logger
-			configs []string
 			err    error
+
+			configs, filterNames []string
+
+			filters []configuration.FileFilter
 		)
 
 		logger = logging.Logger()
@@ -37,6 +40,14 @@ func newCmdWithConfigPreRun(ensureConfigExists, validateKeys, validateConfigurat
 			logger.Fatalf("Error reading flags: %v", err)
 		}
 
+		if filterNames, err = cmd.Flags().GetStringSlice(cmdFlagNameConfigExpFilters); err != nil {
+			logger.Fatalf("Error reading flags: %v", err)
+		}
+
+		if filters, err = configuration.NewFileFilters(filterNames); err != nil {
+			logger.Fatalf("Error occurred loading configuration: flag '--%s' is invalid: %v", cmdFlagNameConfigExpFilters, err)
+		}
+
 		if ensureConfigExists && len(configs) == 1 {
 			created, err := configuration.EnsureConfigurationExists(configs[0])
 			if err != nil {
@@ -53,7 +64,7 @@ func newCmdWithConfigPreRun(ensureConfigExists, validateKeys, validateConfigurat
 			val *schema.StructValidator
 		)
 
-		config, val, err = loadConfig(configs, validateKeys, validateConfiguration)
+		config, val, err = loadConfig(configs, validateKeys, validateConfiguration, filters...)
 		if err != nil {
 			logger.Fatalf("Error occurred loading configuration: %v", err)
 		}
@@ -76,14 +87,15 @@ func newCmdWithConfigPreRun(ensureConfigExists, validateKeys, validateConfigurat
 	}
 }
 
-func loadConfig(configs []string, validateKeys, validateConfiguration bool) (c *schema.Configuration, val *schema.StructValidator, err error) {
+func loadConfig(configs []string, validateKeys, validateConfiguration bool, filters ...configuration.FileFilter) (c *schema.Configuration, val *schema.StructValidator, err error) {
 	var keys []string
 
 	val = schema.NewStructValidator()
 
 	if keys, c, err = configuration.Load(val,
-		configuration.NewDefaultSources(
+		configuration.NewDefaultSourcesFiltered(
 			configs,
+			filters,
 			configuration.DefaultEnvPrefix,
 			configuration.DefaultEnvDelimiter)...); err != nil {
 		return nil, nil, err

internal/commands/const.go

@@ -547,6 +547,8 @@ const (
 	cmdFlagNameSHA512           = "sha512"
 	cmdFlagNameConfig           = "config"
 
+	cmdFlagNameConfigExpFilters = "config.experimental.filters"
+
 	cmdFlagNameCharSet     = "charset"
 	cmdFlagValueCharSet    = "alphanumeric"
 	cmdFlagUsageCharset    = "sets the charset for the random password, options are 'ascii', 'alphanumeric', 'alphabetic', 'numeric', 'numeric-hex', and 'rfc3986'"

internal/commands/crypto_hash.go

@@ -367,7 +367,9 @@ func cmdCryptoHashGetConfig(algorithm string, configs []string, flags *pflag.Fla
 		prefixFilePassword + ".scrypt.salt_length":    schema.DefaultPasswordConfig.SCrypt.SaltLength,
 	}
 
-	sources := configuration.NewDefaultSourcesWithDefaults(configs,
+	sources := configuration.NewDefaultSourcesWithDefaults(
+		configs,
+		nil,
 		configuration.DefaultEnvPrefix, configuration.DefaultEnvDelimiter,
 		configuration.NewMapSource(mapDefaults),
 		configuration.NewCommandLineSourceWithMapping(flags, flagsMap, false, false),

internal/commands/root.go

@@ -44,6 +44,8 @@ func NewRootCmd() (cmd *cobra.Command) {
 
 	cmdWithConfigFlags(cmd, false, []string{})
 
+	cmd.Flags().StringSlice(cmdFlagNameConfigExpFilters, nil, "Applies filters in order to the configuration file before the YAML parser. Options are 'template', 'expand-env'")
+
 	cmd.AddCommand(
 		newAccessControlCommand(),
 		newBuildInfoCmd(),

internal/commands/validate.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/authelia/authelia/v4/internal/configuration"
 	"github.com/authelia/authelia/v4/internal/configuration/schema"
 )
 
@@ -35,7 +36,7 @@ func cmdValidateConfigRunE(cmd *cobra.Command, _ []string) (err error) {
 		return err
 	}
 
-	config, val, err = loadConfig(configs, true, true)
+	config, val, err = loadConfig(configs, true, true, configuration.NewFileFiltersDefault()...)
 	if err != nil {
 		return fmt.Errorf("error occurred loading configuration: %v", err)
 	}

internal/configuration/koanf_provider_filtered_file.go

@@ -0,0 +1,170 @@
+package configuration
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/authelia/authelia/v4/internal/logging"
+	"github.com/authelia/authelia/v4/internal/templates"
+)
+
+// FilteredFile implements a koanf.Provider.
+type FilteredFile struct {
+	path    string
+	filters []FileFilter
+}
+
+// FilteredFileProvider returns a koanf.Provider which provides filtered file output.
+func FilteredFileProvider(path string, filters ...FileFilter) *FilteredFile {
+	return &FilteredFile{
+		path:    filepath.Clean(path),
+		filters: filters,
+	}
+}
+
+// ReadBytes reads the contents of a file on disk, passes it through any configured filters, and returns the bytes.
+func (f *FilteredFile) ReadBytes() (data []byte, err error) {
+	if data, err = os.ReadFile(f.path); err != nil {
+		return nil, err
+	}
+
+	if len(data) == 0 || len(f.filters) == 0 {
+		return data, nil
+	}
+
+	for _, filter := range f.filters {
+		if data, err = filter(data); err != nil {
+			return nil, err
+		}
+	}
+
+	return data, nil
+}
+
+// Read is not supported by the filtered file koanf.Provider.
+func (f *FilteredFile) Read() (map[string]interface{}, error) {
+	return nil, errors.New("filtered file provider does not support this method")
+}
+
+// FileFilter describes a func used to filter files.
+type FileFilter func(in []byte) (out []byte, err error)
+
+// NewFileFiltersDefault returns the default list of FileFilter.
+func NewFileFiltersDefault() []FileFilter {
+	return []FileFilter{
+		NewTemplateFileFilter(),
+		NewExpandEnvFileFilter(),
+	}
+}
+
+// NewFileFilters returns a list of FileFilter provided they are valid.
+func NewFileFilters(names []string) (filters []FileFilter, err error) {
+	filters = make([]FileFilter, len(names))
+
+	filterMap := map[string]int{}
+
+	for i, name := range names {
+		name = strings.ToLower(name)
+
+		switch name {
+		case "template":
+			filters[i] = NewTemplateFileFilter()
+		case "expand-env":
+			filters[i] = NewExpandEnvFileFilter()
+		default:
+			return nil, fmt.Errorf("invalid filter named '%s'", name)
+		}
+
+		if _, ok := filterMap[name]; ok {
+			return nil, fmt.Errorf("duplicate filter named '%s'", name)
+		} else {
+			filterMap[name] = 1
+		}
+	}
+
+	return filters, nil
+}
+
+// NewExpandEnvFileFilter is a FileFilter which passes the bytes through os.ExpandEnv.
+func NewExpandEnvFileFilter() FileFilter {
+	log := logging.Logger()
+
+	return func(in []byte) (out []byte, err error) {
+		out = []byte(os.ExpandEnv(string(in)))
+
+		if log.Level >= logrus.TraceLevel {
+			log.
+				WithField("content", base64.RawStdEncoding.EncodeToString(out)).
+				Trace("Expanded Env File Filter completed successfully")
+		}
+
+		return out, nil
+	}
+}
+
+// NewTemplateFileFilter is a FileFilter which passes the bytes through text/template.
+func NewTemplateFileFilter() FileFilter {
+	data := &TemplateFileFilterData{
+		Env: map[string]string{},
+	}
+
+	for _, e := range os.Environ() {
+		kv := strings.SplitN(e, "=", 2)
+
+		if len(kv) != 2 {
+			continue
+		}
+
+		data.Env[kv[0]] = kv[1]
+	}
+
+	t := template.New("config.template").
+		Funcs(template.FuncMap{
+			"env":       templates.StringMapLookupDefaultEmptyFunc(data.Env),
+			"split":     templates.StringsSplitFunc,
+			"iterate":   templates.IterateFunc,
+			"join":      strings.Join,
+			"contains":  strings.Contains,
+			"hasPrefix": strings.HasPrefix,
+			"hasSuffix": strings.HasSuffix,
+			"lower":     strings.ToLower,
+			"upper":     strings.ToUpper,
+		})
+
+	log := logging.Logger()
+
+	return func(in []byte) (out []byte, err error) {
+		if t, err = t.Parse(string(in)); err != nil {
+			return nil, err
+		}
+
+		buf := &bytes.Buffer{}
+
+		if err = t.Execute(buf, data); err != nil {
+			return nil, err
+		}
+
+		out = buf.Bytes()
+
+		if log.Level >= logrus.TraceLevel {
+			log.
+				WithField("content", base64.RawStdEncoding.EncodeToString(out)).
+				Trace("Templated File Filter completed successfully")
+		}
+
+		return out, nil
+	}
+}
+
+// TemplateFileFilterData is the data available to the Go Template FileFilter.
+type TemplateFileFilterData struct {
+	Env map[string]string
+}

internal/configuration/koanf_provider_filtered_file_test.go

@@ -0,0 +1,80 @@
+package configuration
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewFileFilters(t *testing.T) {
+	testCases := []struct {
+		name   string
+		have   []string
+		expect string
+	}{
+		{
+			"ShouldErrorOnInvalidFilterName",
+			[]string{"abc"},
+			"invalid filter named 'abc'",
+		},
+		{
+			"ShouldErrorOnInvalidFilterNameWithDuplicates",
+			[]string{"abc", "abc"},
+			"invalid filter named 'abc'",
+		},
+		{
+			"ShouldErrorOnInvalidFilterNameWithDuplicatesCaps",
+			[]string{"ABC", "abc"},
+			"invalid filter named 'abc'",
+		},
+		{
+			"ShouldErrorOnDuplicateFilterName",
+			[]string{"expand-env", "expand-env"},
+			"duplicate filter named 'expand-env'",
+		},
+		{
+			"ShouldErrorOnDuplicateFilterNameCaps",
+			[]string{"expand-ENV", "expand-env"},
+			"duplicate filter named 'expand-env'",
+		},
+		{
+			"ShouldNotErrorOnValidFilters",
+			[]string{"expand-env", "template"},
+			"",
+		},
+		{
+			"ShouldNotErrorOnExpandEnvFilter",
+			[]string{"expand-env"},
+			"",
+		},
+		{
+			"ShouldNotErrorOnExpandEnvFilterCaps",
+			[]string{"EXPAND-env"},
+			"",
+		},
+		{
+			"ShouldNotErrorOnTemplateFilter",
+			[]string{"template"},
+			"",
+		},
+		{
+			"ShouldNotErrorOnTemplateFilterCaps",
+			[]string{"TEMPLATE"},
+			"",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual, theError := NewFileFilters(tc.have)
+
+			switch tc.expect {
+			case "":
+				assert.NoError(t, theError)
+				assert.Len(t, actual, len(tc.have))
+			default:
+				assert.EqualError(t, theError, tc.expect)
+			}
+		})
+	}
+}

internal/configuration/provider_test.go

@@ -105,6 +105,29 @@ func TestShouldValidateConfigurationWithEnv(t *testing.T) {
 	assert.Len(t, val.Warnings(), 0)
 }
 
+func TestShouldValidateConfigurationWithFilters(t *testing.T) {
+	testReset()
+
+	testSetEnv(t, "SESSION_SECRET", "abc")
+	testSetEnv(t, "STORAGE_MYSQL_PASSWORD", "abc")
+	testSetEnv(t, "JWT_SECRET", "abc")
+	testSetEnv(t, "AUTHENTICATION_BACKEND_LDAP_PASSWORD", "abc")
+
+	_ = os.Setenv("SERVICES_SERVER", "10.10.10.10")
+	_ = os.Setenv("ROOT_DOMAIN", "example.org")
+
+	val := schema.NewStructValidator()
+	_, config, err := Load(val, NewDefaultSourcesFiltered([]string{"./test_resources/config.filtered.yml"}, NewFileFiltersDefault(), DefaultEnvPrefix, DefaultEnvDelimiter)...)
+
+	assert.NoError(t, err)
+	require.Len(t, val.Errors(), 0)
+	require.Len(t, val.Warnings(), 0)
+
+	assert.Equal(t, "api-123456789.example.org", config.DuoAPI.Hostname)
+	assert.Equal(t, "10.10.10.10", config.Notifier.SMTP.Host)
+	assert.Equal(t, "10.10.10.10", config.Session.Redis.Host)
+}
+
 func TestShouldNotIgnoreInvalidEnvs(t *testing.T) {
 	testReset()
 

internal/configuration/sources.go

@@ -8,15 +8,14 @@ import (
 	"github.com/knadh/koanf/parsers/yaml"
 	"github.com/knadh/koanf/providers/confmap"
 	"github.com/knadh/koanf/providers/env"
-	"github.com/knadh/koanf/providers/file"
 	"github.com/knadh/koanf/providers/posflag"
 	"github.com/spf13/pflag"
 
 	"github.com/authelia/authelia/v4/internal/configuration/schema"
 )
 
-// NewYAMLFileSource returns a Source configured to load from a specified YAML path. If there is an issue accessing this
+// NewYAMLFileSource returns a configuration.Source configured to load from a specified YAML path. If there is an issue
-// path it also returns an error.
+// accessing this path it also returns an error.
 func NewYAMLFileSource(path string) (source *YAMLFileSource) {
 	return &YAMLFileSource{
 		koanf: koanf.New(constDelimiter),
@@ -24,7 +23,17 @@ func NewYAMLFileSource(path string) (source *YAMLFileSource) {
 	}
 }
 
-// NewYAMLFileSources returns a slice of Source configured to load from specified YAML files.
+// NewYAMLFileTemplatedSource returns a configuration.Source configured to load from a specified YAML path. If there is
+// an issue accessing this path it also returns an error.
+func NewYAMLFileTemplatedSource(path string, filters ...FileFilter) (source *YAMLFileSource) {
+	return &YAMLFileSource{
+		koanf:   koanf.New(constDelimiter),
+		path:    path,
+		filters: filters,
+	}
+}
+
+// NewYAMLFileSources returns a slice of configuration.Source configured to load from specified YAML files.
 func NewYAMLFileSources(paths []string) (sources []*YAMLFileSource) {
 	for _, path := range paths {
 		source := NewYAMLFileSource(path)
@@ -35,6 +44,17 @@ func NewYAMLFileSources(paths []string) (sources []*YAMLFileSource) {
 	return sources
 }
 
+// NewYAMLFilteredFileSources returns a slice of configuration.Source configured to load from specified YAML files.
+func NewYAMLFilteredFileSources(paths []string, filters []FileFilter) (sources []*YAMLFileSource) {
+	for _, path := range paths {
+		source := NewYAMLFileTemplatedSource(path, filters...)
+
+		sources = append(sources, source)
+	}
+
+	return sources
+}
+
 // Name of the Source.
 func (s *YAMLFileSource) Name() (name string) {
 	return fmt.Sprintf("yaml file(%s)", s.path)
@@ -51,7 +71,7 @@ func (s *YAMLFileSource) Load(_ *schema.StructValidator) (err error) {
 		return errors.New("invalid yaml path source configuration")
 	}
 
-	return s.koanf.Load(file.Provider(s.path), yaml.Parser())
+	return s.koanf.Load(FilteredFileProvider(s.path, s.filters...), yaml.Parser())
 }
 
 // NewEnvironmentSource returns a Source configured to load from environment variables.
@@ -185,11 +205,32 @@ func NewDefaultSources(filePaths []string, prefix, delimiter string, additionalS
 	return sources
 }
 
-// NewDefaultSourcesWithDefaults returns a slice of Source configured to load from specified YAML files with additional sources.
+// NewDefaultSourcesFiltered returns a slice of Source configured to load from specified YAML files.
-func NewDefaultSourcesWithDefaults(filePaths []string, prefix, delimiter string, defaults Source, additionalSources ...Source) (sources []Source) {
+func NewDefaultSourcesFiltered(files []string, filters []FileFilter, prefix, delimiter string, additionalSources ...Source) (sources []Source) {
-	sources = []Source{defaults}
+	fileSources := NewYAMLFilteredFileSources(files, filters)
+	for _, source := range fileSources {
+		sources = append(sources, source)
+	}
 
-	sources = append(sources, NewDefaultSources(filePaths, prefix, delimiter, additionalSources...)...)
+	sources = append(sources, NewEnvironmentSource(prefix, delimiter))
+	sources = append(sources, NewSecretsSource(prefix, delimiter))
+
+	if len(additionalSources) != 0 {
+		sources = append(sources, additionalSources...)
+	}
+
+	return sources
+}
+
+// NewDefaultSourcesWithDefaults returns a slice of Source configured to load from specified YAML files with additional sources.
+func NewDefaultSourcesWithDefaults(files []string, filters []FileFilter, prefix, delimiter string, defaults Source, additionalSources ...Source) (sources []Source) {
+	sources = []Source{defaults}
+
+	if len(filters) == 0 {
+		sources = append(sources, NewDefaultSources(files, prefix, delimiter, additionalSources...)...)
+	} else {
+		sources = append(sources, NewDefaultSourcesFiltered(files, filters, prefix, delimiter, additionalSources...)...)
+	}
 
 	return sources
 }

internal/configuration/template.go

@@ -7,7 +7,7 @@ import (
 )
 
 //go:embed config.template.yml
-var template []byte
+var conftemplate []byte
 
 // EnsureConfigurationExists is an auxiliary function to the main Configuration tools that ensures the Configuration
 // template is created if it doesn't already exist.
@@ -15,7 +15,7 @@ func EnsureConfigurationExists(path string) (created bool, err error) {
 	_, err = os.Stat(path)
 	if err != nil {
 		if os.IsNotExist(err) {
-			if err = os.WriteFile(path, template, 0600); err != nil {
+			if err = os.WriteFile(path, conftemplate, 0600); err != nil {
 				return false, fmt.Errorf(errFmtGenerateConfiguration, err)
 			}
 

internal/configuration/test_resources/config.filtered.yml

@@ -0,0 +1,177 @@
+---
+default_redirection_url: 'https://home.{{ env "ROOT_DOMAIN" }}:8080/'
+
+server:
+  host: '{{ env "SERVICES_SERVER" }}'
+  port: 9091
+
+log:
+  level: debug
+
+totp:
+  issuer: authelia.com
+
+duo_api:
+  hostname: 'api-123456789.{{ env "ROOT_DOMAIN" }}'
+  integration_key: ABCDEF
+
+authentication_backend:
+  ldap:
+    url: 'ldap://{{ env "SERVICES_SERVER" }}'
+    tls:
+      private_key: |
+        -----BEGIN RSA PRIVATE KEY-----
+        MIIEpAIBAAKCAQEA6z1LOg1ZCqb0lytXWZ+MRBpMHEXOoTOLYgfZXt1IYyE3Z758
+        cyalk0NYQhY5cZDsXPYWPvAHiPMUxutWkoxFwby56S+AbIMa3/Is+ILrHRJs8Exn
+        ZkpyrYFxPX12app2kErdmAkHSx0Z5/kuXiz96PHs8S8/ZbyZolLHzdfLtSzjvRm5
+        Zue5iFzsf19NJz5CIBfv8g5lRwtE8wNJoRSpn1xq7fqfuA0weDNFPzjlNWRLy6aa
+        rK7qJexRkmkCs4sLgyl+9NODYJpvmN8E1yhyC27E0joI6rBFVW7Ihv+cSPCdDzGp
+        EWe81x3AeqAa3mjVqkiq4u4Z2i8JDgBaPboqJwIDAQABAoIBAAFdLZ58jVOefDSU
+        L8F5R1rtvBs93GDa56f926jNJ6pLewLC+/2+757W+SAI+PRLntM7Kg3bXm/Q2QH+
+        Q1Y+MflZmspbWCdI61L5GIGoYKyeers59i+FpvySj5GHtLQRiTZ0+Kv1AXHSDWBm
+        9XneUOqU3IbZe0ifu1RRno72/VtjkGXbW8Mkkw+ohyGbIeTx/0/JQ6sSNZTT3Vk7
+        8i4IXptq3HSF0/vqZuah8rShoeNq72pD1YLM9YPdL5by1QkDLnqATDiCpLBTCaNV
+        I8sqYEun+HYbQzBj8ZACG2JVZpEEidONWQHw5BPWO95DSZYrVnEkuCqeH+u5vYt7
+        CHuJ3AECgYEA+W3v5z+j91w1VPHS0VB3SCDMouycAMIUnJPAbt+0LPP0scUFsBGE
+        hPAKddC54pmMZRQ2KIwBKiyWfCrJ8Xz8Yogn7fJgmwTHidJBr2WQpIEkNGlK3Dzi
+        jXL2sh0yC7sHvn0DqiQ79l/e7yRbSnv2wrTJEczOOH2haD7/tBRyCYECgYEA8W+q
+        E9YyGvEltnPFaOxofNZ8LHVcZSsQI5b6fc0iE7fjxFqeXPXEwGSOTwqQLQRiHn9b
+        CfPmIG4Vhyq0otVmlPvUnfBZ2OK+tl5X2/mQFO3ROMdvpi0KYa994uqfJdSTaqLn
+        jjoKFB906UFHnDQDLZUNiV1WwnkTglgLc+xrd6cCgYEAqqthyv6NyBTM3Tm2gcio
+        Ra9Dtntl51LlXZnvwy3IkDXBCd6BHM9vuLKyxZiziGx+Vy90O1xI872cnot8sINQ
+        Am+dur/tAEVN72zxyv0Y8qb2yfH96iKy9gxi5s75TnOEQgAygLnYWaWR2lorKRUX
+        bHTdXBOiS58S0UzCFEslGIECgYBqkO4SKWYeTDhoKvuEj2yjRYyzlu28XeCWxOo1
+        otiauX0YSyNBRt2cSgYiTzhKFng0m+QUJYp63/wymB/5C5Zmxi0XtWIDADpLhqLj
+        HmmBQ2Mo26alQ5YkffBju0mZyhVzaQop1eZi8WuKFV1FThPlB7hc3E0SM5zv2Grd
+        tQnOWwKBgQC40yZY0PcjuILhy+sIc0Wvh7LUA7taSdTye149kRvbvsCDN7Jh75lM
+        USjhLXY0Nld2zBm9r8wMb81mXH29uvD+tDqqsICvyuKlA/tyzXR+QTr7dCVKVwu0
+        1YjCJ36UpTsLre2f8nOSLtNmRfDPtbOE2mkOoO9dD9UU0XZwnvn9xw==
+        -----END RSA PRIVATE KEY-----
+    base_dn: dc=example,dc=com
+    username_attribute: uid
+    additional_users_dn: ou=users
+    users_filter: (&({username_attribute}={input})(objectCategory=person)(objectClass=user))
+    additional_groups_dn: ou=groups
+    groups_filter: (&(member={dn})(objectClass=groupOfNames))
+    group_name_attribute: cn
+    mail_attribute: mail
+    user: cn=admin,dc=example,dc=com
+
+access_control:
+  default_policy: deny
+
+  rules:
+    # Rules applied to everyone
+    - domain:
+        - 'public.{{ env "ROOT_DOMAIN" }}'
+      policy: bypass
+
+    - domain:
+        - 'secure.{{ env "ROOT_DOMAIN" }}'
+      policy: one_factor
+      # Network based rule, if not provided any network matches.
+      networks:
+        - 192.168.1.0/24
+    - domain:
+        - 'secure.{{ env "ROOT_DOMAIN" }}'
+      policy: two_factor
+
+    - domain:
+        - 'singlefactor.{{ env "ROOT_DOMAIN" }}'
+        - 'onefactor.{{ env "ROOT_DOMAIN" }}'
+      policy: one_factor
+
+    # Rules applied to 'admins' group
+    - domain:
+        - 'mx2.mail.{{ env "ROOT_DOMAIN" }}'
+      subject:
+        - 'group:admins'
+      policy: deny
+    - domain:
+        - '*.{{ env "ROOT_DOMAIN" }}'
+      subject:
+        - ['group:admins']
+      policy: two_factor
+
+    # Rules applied to 'dev' group
+    - domain:
+        - 'dev.{{ env "ROOT_DOMAIN" }}'
+      resources:
+        - '^/groups/dev/.*$'
+      subject:
+        - ['group:dev']
+      policy: two_factor
+
+    # Rules applied to user 'john'
+    - domain:
+        - 'dev.{{ env "ROOT_DOMAIN" }}'
+      resources:
+        - '^/users/john/.*$'
+      subject:
+        - ['user:john']
+      policy: two_factor
+
+    # Rules applied to 'dev' group and user 'john'
+    - domain:
+        - 'dev.{{ env "ROOT_DOMAIN" }}'
+      resources:
+        - "^/deny-all.*$"
+      subject:
+        - ['group:dev']
+        - ['user:john']
+      policy: deny
+
+    # Rules applied to user 'harry'
+    - domain:
+        - 'dev.{{ env "ROOT_DOMAIN" }}'
+      resources:
+        - '^/users/harry/.*$'
+      subject:
+        - ['user:harry']
+      policy: two_factor
+
+    # Rules applied to user 'bob'
+    - domain:
+        - '*.mail.{{ env "ROOT_DOMAIN" }}'
+      subject:
+        - ['user:bob']
+      policy: two_factor
+    - domain:
+        - 'dev.{{ env "ROOT_DOMAIN" }}'
+      resources:
+        - '^/users/bob/.*$'
+      subject:
+        - ['user:bob']
+      policy: two_factor
+
+session:
+  name: authelia_session
+  expiration: 3600000  # 1 hour
+  inactivity: 300000  # 5 minutes
+  domain: '{{ env "ROOT_DOMAIN" }}'
+  redis:
+    host: ${SERVICES_SERVER}
+    port: 6379
+    high_availability:
+      sentinel_name: test
+
+regulation:
+  max_retries: 3
+  find_time: 120
+  ban_time: 300
+
+storage:
+  mysql:
+    host: '{{ env "SERVICES_SERVER" }}'
+    port: 3306
+    database: authelia
+    username: authelia
+
+notifier:
+  smtp:
+    username: test
+    host: '{{ env "SERVICES_SERVER" }}'
+    port: 1025
+    sender: 'admin@{{ env "ROOT_DOMAIN" }}'
+    disable_require_tls: true
+...

internal/configuration/types.go

@@ -7,17 +7,18 @@ import (
 	"github.com/authelia/authelia/v4/internal/configuration/schema"
 )
 
-// Source is an abstract representation of a configuration Source implementation.
+// Source is an abstract representation of a configuration configuration.Source implementation.
 type Source interface {
 	Name() (name string)
 	Merge(ko *koanf.Koanf, val *schema.StructValidator) (err error)
 	Load(val *schema.StructValidator) (err error)
 }
 
-// YAMLFileSource is a configuration Source with a YAML File.
+// YAMLFileSource is a YAML file configuration.Source.
 type YAMLFileSource struct {
 	koanf   *koanf.Koanf
 	path    string
+	filters []FileFilter
 }
 
 // EnvironmentSource is a configuration Source which loads values from the environment.

internal/templates/funcs.go

@@ -0,0 +1,78 @@
+package templates
+
+import (
+	"fmt"
+	"strings"
+)
+
+// StringMapLookupDefaultEmptyFunc is function which takes a map[string]string and returns a template function which
+// takes a string which is used as a key lookup for the map[string]string. If the value isn't found it returns an empty
+// string.
+func StringMapLookupDefaultEmptyFunc(m map[string]string) func(key string) (value string) {
+	return func(key string) (value string) {
+		var ok bool
+
+		if value, ok = m[key]; !ok {
+			return ""
+		}
+
+		return value
+	}
+}
+
+// StringMapLookupFunc is function which takes a map[string]string and returns a template function which
+// takes a string which is used as a key lookup for the map[string]string. If the value isn't found it returns an error.
+func StringMapLookupFunc(m map[string]string) func(key string) (value string, err error) {
+	return func(key string) (value string, err error) {
+		var ok bool
+
+		if value, ok = m[key]; !ok {
+			return value, fmt.Errorf("failed to lookup key '%s' from map", key)
+		}
+
+		return value, nil
+	}
+}
+
+// IterateFunc is a template function which takes a single uint returning a slice of units from 0 up to that number.
+func IterateFunc(count *uint) (out []uint) {
+	var i uint
+
+	for i = 0; i < (*count); i++ {
+		out = append(out, i)
+	}
+
+	return
+}
+
+// StringsSplitFunc is a template function which takes sep and value, splitting the value by the sep into a slice.
+func StringsSplitFunc(value, sep string) []string {
+	return strings.Split(value, sep)
+}
+
+// StringJoinXFunc takes a list of string elements, joins them by the sep string, before every int n characters are
+// written it writes string p. This is useful for line breaks mostly.
+func StringJoinXFunc(elems []string, sep string, n int, p string) string {
+	buf := strings.Builder{}
+
+	c := 0
+	e := len(elems) - 1
+
+	for i := 0; i <= e; i++ {
+		if c+len(elems[i])+1 > n {
+			c = 0
+
+			buf.WriteString(p)
+		}
+
+		c += len(elems[i]) + 1
+
+		buf.WriteString(elems[i])
+
+		if i < e {
+			buf.WriteString(sep)
+		}
+	}
+
+	return buf.String()
+}