[FEATURE] Support MySQL as a storage backend. (#678)

* [FEATURE] Support MySQL as a storage backend.

Fixes #512.

* Fix integration tests and include MySQL in docs.

docs/configuration/storage/index.md

@@ -16,5 +16,5 @@ The available options are:
 
 * [SQLite](./sqlite.md)
 * [MariaDB](./mariadb.md)
-* ~~MySQL~~ ([#512](https://github.com/authelia/authelia/issues/512))
+* [MySQL](./mysql.md)
 * [Postgres](./postgres.md)

docs/configuration/storage/mysql.md

@@ -0,0 +1,20 @@
+---
+layout: default
+title: MySQL
+parent: Storage backends
+grand_parent: Configuration
+nav_order: 3
+---
+
+# MySQL
+
+```yaml
+storage:
+    mysql:
+        host: 127.0.0.1
+        port: 3306
+        database: authelia
+        username: authelia
+        # This secret can also be set using the env variables AUTHELIA_STORAGE_MYSQL_PASSWORD
+        password: mypassword
+```

docs/configuration/storage/sqlite.md

@@ -3,7 +3,7 @@ layout: default
 title: SQLite
 parent: Storage backends
 grand_parent: Configuration
-nav_order: 3
+nav_order: 4
 ---
 
 # SQLite

internal/storage/constants.go

@@ -1,8 +1,44 @@
 package storage
 
+import "fmt"
+
 // Keep table names in lower case because some DB does not support upper case.
 var preferencesTableName = "user_preferences"
 var identityVerificationTokensTableName = "identity_verification_tokens"
 var totpSecretsTableName = "totp_secrets"
 var u2fDeviceHandlesTableName = "u2f_devices"
 var authenticationLogsTableName = "authentication_logs"
+
+// SQLCreateUserPreferencesTable common SQL query to create user_preferences table
+var SQLCreateUserPreferencesTable = fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s (
+	username VARCHAR(100) PRIMARY KEY,
+	second_factor_method VARCHAR(11)
+)`, preferencesTableName)
+
+// SQLCreateIdentityVerificationTokensTable common SQL query to create identity_verification_tokens table
+var SQLCreateIdentityVerificationTokensTable = fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s (token VARCHAR(512))
+`, identityVerificationTokensTableName)
+
+// SQLCreateTOTPSecretsTable common SQL query to create totp_secrets table
+var SQLCreateTOTPSecretsTable = fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100) PRIMARY KEY, secret VARCHAR(64))
+`, totpSecretsTableName)
+
+// SQLCreateU2FDeviceHandlesTable common SQL query to create u2f_device_handles table
+var SQLCreateU2FDeviceHandlesTable = fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s (
+	username VARCHAR(100) PRIMARY KEY,
+	keyHandle TEXT,
+	publicKey TEXT
+)`, u2fDeviceHandlesTableName)
+
+// SQLCreateAuthenticationLogsTable common SQL query to create authentication_logs table
+var SQLCreateAuthenticationLogsTable = fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s (
+	username VARCHAR(100),
+	successful BOOL,
+	time INTEGER,
+	INDEX usr_time_idx (username, time)
+)`, authenticationLogsTableName)

internal/storage/mysql_provider.go

@@ -43,6 +43,12 @@ func NewMySQLProvider(configuration schema.MySQLStorageConfiguration) *MySQLProv
 
 	provider := MySQLProvider{
 		SQLProvider{
+			sqlCreateUserPreferencesTable:            SQLCreateUserPreferencesTable,
+			sqlCreateIdentityVerificationTokensTable: SQLCreateIdentityVerificationTokensTable,
+			sqlCreateTOTPSecretsTable:                SQLCreateTOTPSecretsTable,
+			sqlCreateU2FDeviceHandlesTable:           SQLCreateU2FDeviceHandlesTable,
+			sqlCreateAuthenticationLogsTable:         SQLCreateAuthenticationLogsTable,
+
 			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=?", preferencesTableName),
 			sqlUpsertSecondFactorPreference: fmt.Sprintf("REPLACE INTO %s (username, second_factor_method) VALUES (?, ?)", preferencesTableName),
 

internal/storage/postgres_provider.go

@@ -51,6 +51,13 @@ func NewPostgreSQLProvider(configuration schema.PostgreSQLStorageConfiguration)
 
 	provider := PostgreSQLProvider{
 		SQLProvider{
+			sqlCreateUserPreferencesTable:            SQLCreateUserPreferencesTable,
+			sqlCreateIdentityVerificationTokensTable: SQLCreateIdentityVerificationTokensTable,
+			sqlCreateTOTPSecretsTable:                SQLCreateTOTPSecretsTable,
+			sqlCreateU2FDeviceHandlesTable:           SQLCreateU2FDeviceHandlesTable,
+			sqlCreateAuthenticationLogsTable:         fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100), successful BOOL, time INTEGER)", authenticationLogsTableName),
+			sqlCreateAuthenticationLogsUserTimeIndex: fmt.Sprintf("CREATE INDEX IF NOT EXISTS usr_time_idx ON %s (username, time)", authenticationLogsTableName),
+
 			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=$1", preferencesTableName),
 			sqlUpsertSecondFactorPreference: fmt.Sprintf("INSERT INTO %s (username, second_factor_method) VALUES ($1, $2) ON CONFLICT (username) DO UPDATE SET second_factor_method=$2", preferencesTableName),
 

internal/storage/sql_provider.go

@@ -13,6 +13,13 @@ import (
 type SQLProvider struct {
 	db *sql.DB
 
+	sqlCreateUserPreferencesTable            string
+	sqlCreateIdentityVerificationTokensTable string
+	sqlCreateTOTPSecretsTable                string
+	sqlCreateU2FDeviceHandlesTable           string
+	sqlCreateAuthenticationLogsTable         string
+	sqlCreateAuthenticationLogsUserTimeIndex string
+
 	sqlGetPreferencesByUsername     string
 	sqlUpsertSecondFactorPreference string
 
@@ -34,40 +41,39 @@ type SQLProvider struct {
 func (p *SQLProvider) initialize(db *sql.DB) error {
 	p.db = db
 
-	_, err := db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100) PRIMARY KEY, second_factor_method VARCHAR(11))", preferencesTableName))
+	_, err := db.Exec(p.sqlCreateUserPreferencesTable)
 	if err != nil {
-		return err
+		return fmt.Errorf("Unable to create table %s: %v", preferencesTableName, err)
 	}
 
-	_, err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (token VARCHAR(512))", identityVerificationTokensTableName))
+	_, err = db.Exec(p.sqlCreateIdentityVerificationTokensTable)
 	if err != nil {
-		return err
+		return fmt.Errorf("Unable to create table %s: %v", identityVerificationTokensTableName, err)
 	}
 
-	_, err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100) PRIMARY KEY, secret VARCHAR(64))", totpSecretsTableName))
+	_, err = db.Exec(p.sqlCreateTOTPSecretsTable)
 	if err != nil {
-		return err
+		return fmt.Errorf("Unable to create table %s: %v", totpSecretsTableName, err)
 	}
 
 	// keyHandle and publicKey are stored in base64 format
-	_, err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100) PRIMARY KEY, keyHandle TEXT, publicKey TEXT)", u2fDeviceHandlesTableName))
+	_, err = db.Exec(p.sqlCreateU2FDeviceHandlesTable)
 	if err != nil {
-		return err
+		return fmt.Errorf("Unable to create table %s: %v", u2fDeviceHandlesTableName, err)
 	}
 
-	_, err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100), successful BOOL, time INTEGER)", authenticationLogsTableName))
+	_, err = db.Exec(p.sqlCreateAuthenticationLogsTable)
 	if err != nil {
-		return err
+		return fmt.Errorf("Unable to create table %s: %v", authenticationLogsTableName, err)
 	}
 
-	_, err = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS time ON %s (time);", authenticationLogsTableName))
+	// Create an index on (username, time) because this couple is highly used by the regulation module
-	if err != nil {
+	// to check whether a user is banned.
-		return err
+	if p.sqlCreateAuthenticationLogsUserTimeIndex != "" {
-	}
+		_, err = db.Exec(p.sqlCreateAuthenticationLogsUserTimeIndex)
-
+		if err != nil {
-	_, err = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS username ON %s (username);", authenticationLogsTableName))
+			return fmt.Errorf("Unable to create table %s: %v", authenticationLogsTableName, err)
-	if err != nil {
+		}
-		return err
 	}
 	return nil
 }

internal/storage/sqlite_provider.go

@@ -22,6 +22,13 @@ func NewSQLiteProvider(path string) *SQLiteProvider {
 
 	provider := SQLiteProvider{
 		SQLProvider{
+			sqlCreateUserPreferencesTable:            SQLCreateUserPreferencesTable,
+			sqlCreateIdentityVerificationTokensTable: SQLCreateIdentityVerificationTokensTable,
+			sqlCreateTOTPSecretsTable:                SQLCreateTOTPSecretsTable,
+			sqlCreateU2FDeviceHandlesTable:           SQLCreateU2FDeviceHandlesTable,
+			sqlCreateAuthenticationLogsTable:         fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100), successful BOOL, time INTEGER)", authenticationLogsTableName),
+			sqlCreateAuthenticationLogsUserTimeIndex: fmt.Sprintf("CREATE INDEX IF NOT EXISTS usr_time_idx ON %s (username, time)", authenticationLogsTableName),
+
 			sqlGetPreferencesByUsername:     fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=?", preferencesTableName),
 			sqlUpsertSecondFactorPreference: fmt.Sprintf("REPLACE INTO %s (username, second_factor_method) VALUES (?, ?)", preferencesTableName),
 

internal/suites/MySQL/configuration.yml

@@ -0,0 +1,68 @@
+###############################################################
+#                Authelia minimal configuration               #
+###############################################################
+
+port: 9091
+
+logs_level: debug
+
+default_redirection_url: https://home.example.com:8080/
+
+jwt_secret: very_important_secret
+
+authentication_backend:
+  file:
+    path: /var/lib/authelia/users.yml
+
+session:
+  secret: unsecure_session_secret
+  domain: example.com
+  expiration: 3600 # 1 hour
+  inactivity: 300 # 5 minutes
+
+# Configuration of the storage backend used to store data and secrets. i.e. totp data
+storage:
+  mysql:
+    host: mysql
+    port: 3306
+    database: authelia
+    username: admin
+    password: password
+
+# TOTP Issuer Name
+#
+# This will be the issuer name displayed in Google Authenticator
+# See: https://github.com/google/google-authenticator/wiki/Key-Uri-Format for more info on issuer names
+totp:
+  issuer: example.com
+
+access_control:
+  default_policy: deny
+  rules:
+    - domain: "public.example.com"
+      policy: bypass
+    - domain: "admin.example.com"
+      policy: two_factor
+    - domain: "secure.example.com"
+      policy: two_factor
+    - domain: "singlefactor.example.com"
+      policy: one_factor
+
+# Configuration of the authentication regulation mechanism.
+regulation:
+  # Set it to 0 to disable max_retries.
+  max_retries: 3
+
+  # The user is banned if the authentication failed `max_retries` times in a `find_time` seconds window.
+  find_time: 8
+
+  # The length of time before a banned user can login again.
+  ban_time: 10
+
+notifier:
+  # Use a SMTP server for sending notifications
+  smtp:
+    host: smtp
+    port: 1025
+    sender: admin@example.com
+    disable_require_tls: true

internal/suites/MySQL/docker-compose.yml

@@ -0,0 +1,6 @@
+version: '3'
+services:
+  authelia-backend:
+    volumes:
+      - './MySQL/configuration.yml:/etc/authelia/configuration.yml:ro'
+      - './MySQL/users.yml:/var/lib/authelia/users.yml'

internal/suites/MySQL/users.yml

@@ -0,0 +1,29 @@
+###############################################################
+#                         Users Database                      #
+###############################################################
+
+# This file can be used if you do not have an LDAP set up.
+
+# List of users
+users:
+  john:
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"
+    email: john.doe@authelia.com
+    groups:
+      - admins
+      - dev
+
+  harry:
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"
+    email: harry.potter@authelia.com
+    groups: []
+
+  bob:
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"
+    email: bob.dylan@authelia.com
+    groups:
+      - dev
+
+  james:
+    password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/"
+    email: james.dean@authelia.com

internal/suites/example/compose/mysql/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3'
+services:
+  mysql:
+    image: mysql:8.0
+    environment:
+      - MYSQL_ROOT_PASSWORD=rootpassword
+      - MYSQL_USER=admin
+      - MYSQL_PASSWORD=password
+      - MYSQL_DATABASE=authelia
+    networks:
+      - authelianet

internal/suites/suite_mysql.go

@@ -0,0 +1,58 @@
+package suites
+
+import (
+	"fmt"
+	"time"
+)
+
+var mysqlSuiteName = "MySQL"
+
+func init() {
+	dockerEnvironment := NewDockerEnvironment([]string{
+		"internal/suites/docker-compose.yml",
+		"internal/suites/MySQL/docker-compose.yml",
+		"internal/suites/example/compose/authelia/docker-compose.backend.{}.yml",
+		"internal/suites/example/compose/authelia/docker-compose.frontend.{}.yml",
+		"internal/suites/example/compose/nginx/backend/docker-compose.yml",
+		"internal/suites/example/compose/nginx/portal/docker-compose.yml",
+		"internal/suites/example/compose/smtp/docker-compose.yml",
+		"internal/suites/example/compose/mysql/docker-compose.yml",
+		"internal/suites/example/compose/ldap/docker-compose.yml",
+	})
+
+	setup := func(suitePath string) error {
+		if err := dockerEnvironment.Up(); err != nil {
+			return err
+		}
+
+		return waitUntilAutheliaBackendIsReady(dockerEnvironment)
+	}
+
+	onSetupTimeout := func() error {
+		backendLogs, err := dockerEnvironment.Logs("authelia-backend", nil)
+		if err != nil {
+			return err
+		}
+		fmt.Println(backendLogs)
+
+		frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil)
+		if err != nil {
+			return err
+		}
+		fmt.Println(frontendLogs)
+		return nil
+	}
+
+	teardown := func(suitePath string) error {
+		err := dockerEnvironment.Down()
+		return err
+	}
+
+	GlobalRegistry.Register(mysqlSuiteName, Suite{
+		SetUp:           setup,
+		SetUpTimeout:    5 * time.Minute,
+		OnSetupTimeout:  onSetupTimeout,
+		TearDown:        teardown,
+		TearDownTimeout: 2 * time.Minute,
+	})
+}

internal/suites/suite_mysql_test.go

@@ -0,0 +1,27 @@
+package suites
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type MySQLSuite struct {
+	*SeleniumSuite
+}
+
+func NewMySQLSuite() *MySQLSuite {
+	return &MySQLSuite{SeleniumSuite: new(SeleniumSuite)}
+}
+
+func (s *MySQLSuite) TestOneFactorScenario() {
+	suite.Run(s.T(), NewOneFactorScenario())
+}
+
+func (s *MySQLSuite) TestTwoFactorScenario() {
+	suite.Run(s.T(), NewTwoFactorScenario())
+}
+
+func TestMySQLSuite(t *testing.T) {
+	suite.Run(t, NewMySQLSuite())
+}

internal/suites/webdriver.go

@@ -27,8 +27,13 @@ func StartWebDriverWithProxy(proxy string, port int) (*WebDriverSession, error)
 		return nil, err
 	}
 
+	browserPath := os.Getenv("BROWSER_PATH")
+	if browserPath == "" {
+		browserPath = "/usr/bin/chromium-browser"
+	}
+
 	chromeCaps := chrome.Capabilities{
-		Path: "/usr/bin/chromium-browser",
+		Path: browserPath,
 	}
 
 	chromeCaps.Args = append(chromeCaps.Args, "--ignore-certificate-errors")