feat(authentication): file provider hot reload (#4188)

This adds hot reloading to the file auth provider.

docs/content/en/configuration/first-factor/file.md

@@ -20,6 +20,7 @@ aliases:
 authentication_backend:
   file:
     path: /config/users.yml
+    watch: false
     password:
       algorithm: argon2
       argon2:
@@ -58,6 +59,12 @@ The path to the file with the user details list. Supported file types are:
 
 * [YAML File](../../reference/guides/passwords.md#yaml-format)
 
+### watch
+
+{{< confkey type="boolean" default="false" required="no" >}}
+
+Enables reloading the database by watching it for changes.
+
 ## Password Options
 
 A [reference guide](../../reference/guides/passwords.md) exists specifically for choosing password hashing values. This

docs/content/en/reference/cli/authelia/authelia_crypto_hash.md

@@ -2,7 +2,7 @@
 title: "authelia crypto hash"
 description: "Reference for the authelia crypto hash command."
 lead: ""
-date: 2022-08-27T10:46:58+10:00
+date: 2022-10-17T21:51:59+11:00
 draft: false
 images: []
 menu:

docs/content/en/reference/cli/authelia/authelia_crypto_hash_generate.md

@@ -2,7 +2,7 @@
 title: "authelia crypto hash generate"
 description: "Reference for the authelia crypto hash generate command."
 lead: ""
-date: 2022-08-27T10:46:58+10:00
+date: 2022-10-17T21:51:59+11:00
 draft: false
 images: []
 menu:

docs/content/en/reference/cli/authelia/authelia_crypto_hash_generate_argon2.md

@@ -2,7 +2,7 @@
 title: "authelia crypto hash generate argon2"
 description: "Reference for the authelia crypto hash generate argon2 command."
 lead: ""
-date: 2022-08-27T10:46:58+10:00
+date: 2022-10-17T21:51:59+11:00
 draft: false
 images: []
 menu:

docs/content/en/reference/cli/authelia/authelia_crypto_hash_generate_bcrypt.md

@@ -2,7 +2,7 @@
 title: "authelia crypto hash generate bcrypt"
 description: "Reference for the authelia crypto hash generate bcrypt command."
 lead: ""
-date: 2022-08-27T10:46:58+10:00
+date: 2022-10-17T21:51:59+11:00
 draft: false
 images: []
 menu:

docs/content/en/reference/cli/authelia/authelia_crypto_hash_generate_pbkdf2.md

@@ -2,7 +2,7 @@
 title: "authelia crypto hash generate pbkdf2"
 description: "Reference for the authelia crypto hash generate pbkdf2 command."
 lead: ""
-date: 2022-08-27T10:46:58+10:00
+date: 2022-10-17T21:51:59+11:00
 draft: false
 images: []
 menu:

docs/content/en/reference/cli/authelia/authelia_crypto_hash_generate_scrypt.md

@@ -2,7 +2,7 @@
 title: "authelia crypto hash generate scrypt"
 description: "Reference for the authelia crypto hash generate scrypt command."
 lead: ""
-date: 2022-08-27T10:46:58+10:00
+date: 2022-10-17T21:51:59+11:00
 draft: false
 images: []
 menu:

docs/content/en/reference/cli/authelia/authelia_crypto_hash_generate_sha2crypt.md

@@ -2,7 +2,7 @@
 title: "authelia crypto hash generate sha2crypt"
 description: "Reference for the authelia crypto hash generate sha2crypt command."
 lead: ""
-date: 2022-08-27T10:46:58+10:00
+date: 2022-10-17T21:51:59+11:00
 draft: false
 images: []
 menu:

docs/content/en/reference/cli/authelia/authelia_crypto_hash_validate.md

@@ -2,7 +2,7 @@
 title: "authelia crypto hash validate"
 description: "Reference for the authelia crypto hash validate command."
 lead: ""
-date: 2022-08-27T10:46:58+10:00
+date: 2022-10-17T21:51:59+11:00
 draft: false
 images: []
 menu:

docs/content/en/reference/cli/authelia/authelia_crypto_rand.md

@@ -2,7 +2,7 @@
 title: "authelia crypto rand"
 description: "Reference for the authelia crypto rand command."
 lead: ""
-date: 2022-08-27T10:46:58+10:00
+date: 2022-10-17T21:51:59+11:00
 draft: false
 images: []
 menu:

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/duosecurity/duo_api_golang v0.0.0-20220927171823-f4576e85b96c
 	github.com/fasthttp/router v1.4.12
 	github.com/fasthttp/session/v2 v2.4.13
+	github.com/fsnotify/fsnotify v1.6.0
 	github.com/go-asn1-ber/asn1-ber v1.5.4
 	github.com/go-crypt/crypt v0.1.13
 	github.com/go-ldap/ldap/v3 v3.4.4
@@ -56,7 +57,6 @@ require (
 	github.com/dlclark/regexp2 v1.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
 	github.com/go-crypt/x v0.1.3 // indirect
 	github.com/go-redis/redis/v8 v8.11.5 // indirect

go.sum

@@ -229,8 +229,8 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
-github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88=
 github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -1715,12 +1715,12 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 h1:OK7RB6t2WQX54srQQYSXMW8dF5C6/8+oA/s5QBmmto4=
 golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=

internal/authentication/const.go

@@ -87,8 +87,13 @@ const (
 	hashBCrypt    = "bcrypt"
 )
 
-// ErrUserNotFound indicates the user wasn't found in the authentication backend.
+var (
-var ErrUserNotFound = errors.New("user not found")
+	// ErrUserNotFound indicates the user wasn't found in the authentication backend.
+	ErrUserNotFound = errors.New("user not found")
+
+	// ErrNoContent is returned when the file is empty.
+	ErrNoContent = errors.New("no file content")
+)
 
 const fileAuthenticationMode = 0600
 

internal/authentication/file_user_provider.go

@@ -2,8 +2,11 @@ package authentication
 
 import (
 	_ "embed" // Embed users_database.template.yml.
+	"errors"
 	"fmt"
 	"os"
+	"sync"
+	"time"
 
 	"github.com/go-crypt/crypt"
 
@@ -13,18 +16,48 @@ import (
 
 // FileUserProvider is a provider reading details from a file.
 type FileUserProvider struct {
-	config   *schema.FileAuthenticationBackend
+	config        *schema.FileAuthenticationBackend
-	hash     crypt.Hash
+	hash          crypt.Hash
-	database *FileUserDatabase
+	database      *FileUserDatabase
+	mutex         *sync.Mutex
+	timeoutReload time.Time
 }
 
 // NewFileUserProvider creates a new instance of FileUserProvider.
 func NewFileUserProvider(config *schema.FileAuthenticationBackend) (provider *FileUserProvider) {
 	return &FileUserProvider{
-		config: config,
+		config:        config,
+		mutex:         &sync.Mutex{},
+		timeoutReload: time.Now().Add(-1 * time.Second),
 	}
 }
 
+// Reload the database.
+func (p *FileUserProvider) Reload() (reloaded bool, err error) {
+	now := time.Now()
+
+	p.mutex.Lock()
+
+	defer p.mutex.Unlock()
+
+	if now.Before(p.timeoutReload) {
+		return false, nil
+	}
+
+	switch err = p.database.Load(); {
+	case err == nil:
+		p.setTimeoutReload(now)
+	case errors.Is(err, ErrNoContent):
+		return false, nil
+	default:
+		return false, fmt.Errorf("failed to reload: %w", err)
+	}
+
+	p.setTimeoutReload(now)
+
+	return true, nil
+}
+
 // CheckUserPassword checks if provided password matches for the given user.
 func (p *FileUserProvider) CheckUserPassword(username string, password string) (match bool, err error) {
 	var details DatabaseUserDetails
@@ -73,6 +106,12 @@ func (p *FileUserProvider) UpdatePassword(username string, newPassword string) (
 
 	p.database.SetUserDetails(details.Username, &details)
 
+	p.mutex.Lock()
+
+	p.setTimeoutReload(time.Now())
+
+	p.mutex.Unlock()
+
 	if err = p.database.Save(); err != nil {
 		return err
 	}
@@ -101,6 +140,10 @@ func (p *FileUserProvider) StartupCheck() (err error) {
 	return nil
 }
 
+func (p *FileUserProvider) setTimeoutReload(now time.Time) {
+	p.timeoutReload = now.Add(time.Second / 2)
+}
+
 // NewFileCryptoHashFromConfig returns a crypt.Hash given a valid configuration.
 func NewFileCryptoHashFromConfig(config schema.Password) (hash crypt.Hash, err error) {
 	switch config.Algorithm {

internal/authentication/file_user_provider_database.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/asaskevich/govalidator"
 	"github.com/go-crypt/crypt"
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 )
 
 // NewFileUserDatabase creates a new FileUserDatabase.
@@ -168,6 +168,10 @@ func (m *DatabaseModel) Read(filePath string) (err error) {
 		return fmt.Errorf("failed to read the '%s' file: %w", filePath, err)
 	}
 
+	if len(content) == 0 {
+		return ErrNoContent
+	}
+
 	if err = yaml.Unmarshal(content, m); err != nil {
 		return fmt.Errorf("could not parse the YAML database: %w", err)
 	}

internal/commands/root.go

@@ -6,14 +6,17 @@ import (
 	"net"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strings"
 	"syscall"
 
+	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/valyala/fasthttp"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/authelia/authelia/v4/internal/authentication"
 	"github.com/authelia/authelia/v4/internal/configuration/schema"
 	"github.com/authelia/authelia/v4/internal/logging"
 	"github.com/authelia/authelia/v4/internal/middlewares"
@@ -83,11 +86,11 @@ func cmdRootRun(_ *cobra.Command, _ []string) {
 
 	doStartupChecks(config, &providers, logger)
 
-	runServers(config, providers, logger)
+	runServices(config, providers, logger)
 }
 
 //nolint:gocyclo // Complexity is required in this function.
-func runServers(config *schema.Configuration, providers middlewares.Providers, log *logrus.Logger) {
+func runServices(config *schema.Configuration, providers middlewares.Providers, log *logrus.Logger) {
 	ctx := context.Background()
 
 	ctx, cancel := context.WithCancel(ctx)
@@ -151,6 +154,15 @@ func runServers(config *schema.Configuration, providers middlewares.Providers, l
 		return nil
 	})
 
+	if config.AuthenticationBackend.File != nil && config.AuthenticationBackend.File.Watch {
+		provider := providers.UserProvider.(*authentication.FileUserProvider)
+		if watcher, err := runServiceFileWatcher(g, log, config.AuthenticationBackend.File.Path, provider); err != nil {
+			log.WithError(err).Errorf("Error opening file watcher")
+		} else {
+			defer watcher.Close()
+		}
+	}
+
 	select {
 	case s := <-quit:
 		switch s {
@@ -186,6 +198,75 @@ func runServers(config *schema.Configuration, providers middlewares.Providers, l
 	}
 }
 
+type ReloadFilter func(path string) (skipped bool)
+
+type ProviderReload interface {
+	Reload() (reloaded bool, err error)
+}
+
+func runServiceFileWatcher(g *errgroup.Group, log *logrus.Logger, path string, reload ProviderReload) (watcher *fsnotify.Watcher, err error) {
+	if watcher, err = fsnotify.NewWatcher(); err != nil {
+		return nil, err
+	}
+
+	failed := make(chan struct{})
+
+	var directory, filename string
+
+	if path != "" {
+		directory, filename = filepath.Dir(path), filepath.Base(path)
+	}
+
+	g.Go(func() error {
+		for {
+			select {
+			case <-failed:
+				return nil
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return nil
+				}
+
+				if filename != filepath.Base(event.Name) {
+					log.WithField("file", event.Name).WithField("op", event.Op).Tracef("File modification detected to irrelevant file")
+					break
+				}
+
+				switch {
+				case event.Op&fsnotify.Write == fsnotify.Write, event.Op&fsnotify.Create == fsnotify.Create:
+					log.WithField("file", event.Name).WithField("op", event.Op).Debug("File modification detected")
+
+					switch reloaded, err := reload.Reload(); {
+					case err != nil:
+						log.WithField("file", event.Name).WithField("op", event.Op).WithError(err).Error("Error occurred reloading file")
+					case reloaded:
+						log.WithField("file", event.Name).Info("Reloaded file successfully")
+					default:
+						log.WithField("file", event.Name).Debug("Reload of file was triggered but it was skipped")
+					}
+				case event.Op&fsnotify.Remove == fsnotify.Remove:
+					log.WithField("file", event.Name).WithField("op", event.Op).Debug("Remove of file was detected")
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return nil
+				}
+				log.WithError(err).Errorf("Error while watching files")
+			}
+		}
+	})
+
+	if err := watcher.Add(directory); err != nil {
+		failed <- struct{}{}
+
+		return nil, err
+	}
+
+	log.WithField("directory", directory).WithField("file", filename).Debug("Directory is being watched for changes to the file")
+
+	return watcher, nil
+}
+
 func doStartupChecks(config *schema.Configuration, providers *middlewares.Providers, log *logrus.Logger) {
 	var (
 		failures []string

internal/configuration/schema/authentication.go

@@ -24,6 +24,7 @@ type PasswordResetAuthenticationBackend struct {
 // FileAuthenticationBackend represents the configuration related to file-based backend.
 type FileAuthenticationBackend struct {
 	Path     string   `koanf:"path"`
+	Watch    bool     `koanf:"watch"`
 	Password Password `koanf:"password"`
 }
 

internal/configuration/schema/keys.go

@@ -50,6 +50,7 @@ var Keys = []string{
 	"authentication_backend.password_reset.custom_url",
 	"authentication_backend.refresh_interval",
 	"authentication_backend.file.path",
+	"authentication_backend.file.watch",
 	"authentication_backend.file.password.algorithm",
 	"authentication_backend.file.password.argon2.variant",
 	"authentication_backend.file.password.argon2.iterations",