From 0dea0fc82ebab1276fd671a7570f627ba3dc3113 Mon Sep 17 00:00:00 2001 From: Amir Zarrinkafsh Date: Thu, 5 Mar 2020 10:25:52 +1100 Subject: [PATCH] [FEATURE] Support MySQL as a storage backend. (#678) * [FEATURE] Support MySQL as a storage backend. Fixes #512. * Fix integration tests and include MySQL in docs. --- docs/configuration/storage/index.md | 2 +- docs/configuration/storage/mysql.md | 20 ++++++ docs/configuration/storage/sqlite.md | 2 +- internal/storage/constants.go | 36 ++++++++++ internal/storage/mysql_provider.go | 6 ++ internal/storage/postgres_provider.go | 7 ++ internal/storage/sql_provider.go | 42 +++++++----- internal/storage/sqlite_provider.go | 7 ++ internal/suites/MySQL/configuration.yml | 68 +++++++++++++++++++ internal/suites/MySQL/docker-compose.yml | 6 ++ internal/suites/MySQL/users.yml | 29 ++++++++ .../example/compose/mysql/docker-compose.yml | 11 +++ internal/suites/suite_mysql.go | 58 ++++++++++++++++ internal/suites/suite_mysql_test.go | 27 ++++++++ internal/suites/webdriver.go | 7 +- 15 files changed, 307 insertions(+), 21 deletions(-) create mode 100644 docs/configuration/storage/mysql.md create mode 100644 internal/suites/MySQL/configuration.yml create mode 100644 internal/suites/MySQL/docker-compose.yml create mode 100644 internal/suites/MySQL/users.yml create mode 100644 internal/suites/example/compose/mysql/docker-compose.yml create mode 100644 internal/suites/suite_mysql.go create mode 100644 internal/suites/suite_mysql_test.go diff --git a/docs/configuration/storage/index.md b/docs/configuration/storage/index.md index 5025973c7..a04e84031 100644 --- a/docs/configuration/storage/index.md +++ b/docs/configuration/storage/index.md @@ -16,5 +16,5 @@ The available options are: * [SQLite](./sqlite.md) * [MariaDB](./mariadb.md) -* ~~MySQL~~ ([#512](https://github.com/authelia/authelia/issues/512)) +* [MySQL](./mysql.md) * [Postgres](./postgres.md) \ No newline at end of file diff --git a/docs/configuration/storage/mysql.md b/docs/configuration/storage/mysql.md new file mode 100644 index 000000000..12c920adf --- /dev/null +++ b/docs/configuration/storage/mysql.md @@ -0,0 +1,20 @@ +--- +layout: default +title: MySQL +parent: Storage backends +grand_parent: Configuration +nav_order: 3 +--- + +# MySQL + +```yaml +storage: + mysql: + host: 127.0.0.1 + port: 3306 + database: authelia + username: authelia + # This secret can also be set using the env variables AUTHELIA_STORAGE_MYSQL_PASSWORD + password: mypassword +``` diff --git a/docs/configuration/storage/sqlite.md b/docs/configuration/storage/sqlite.md index ebb37d335..d299e3500 100644 --- a/docs/configuration/storage/sqlite.md +++ b/docs/configuration/storage/sqlite.md @@ -3,7 +3,7 @@ layout: default title: SQLite parent: Storage backends grand_parent: Configuration -nav_order: 3 +nav_order: 4 --- # SQLite diff --git a/internal/storage/constants.go b/internal/storage/constants.go index 9fffde961..05f46553a 100644 --- a/internal/storage/constants.go +++ b/internal/storage/constants.go @@ -1,8 +1,44 @@ package storage +import "fmt" + // Keep table names in lower case because some DB does not support upper case. var preferencesTableName = "user_preferences" var identityVerificationTokensTableName = "identity_verification_tokens" var totpSecretsTableName = "totp_secrets" var u2fDeviceHandlesTableName = "u2f_devices" var authenticationLogsTableName = "authentication_logs" + +// SQLCreateUserPreferencesTable common SQL query to create user_preferences table +var SQLCreateUserPreferencesTable = fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS %s ( + username VARCHAR(100) PRIMARY KEY, + second_factor_method VARCHAR(11) +)`, preferencesTableName) + +// SQLCreateIdentityVerificationTokensTable common SQL query to create identity_verification_tokens table +var SQLCreateIdentityVerificationTokensTable = fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS %s (token VARCHAR(512)) +`, identityVerificationTokensTableName) + +// SQLCreateTOTPSecretsTable common SQL query to create totp_secrets table +var SQLCreateTOTPSecretsTable = fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100) PRIMARY KEY, secret VARCHAR(64)) +`, totpSecretsTableName) + +// SQLCreateU2FDeviceHandlesTable common SQL query to create u2f_device_handles table +var SQLCreateU2FDeviceHandlesTable = fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS %s ( + username VARCHAR(100) PRIMARY KEY, + keyHandle TEXT, + publicKey TEXT +)`, u2fDeviceHandlesTableName) + +// SQLCreateAuthenticationLogsTable common SQL query to create authentication_logs table +var SQLCreateAuthenticationLogsTable = fmt.Sprintf(` +CREATE TABLE IF NOT EXISTS %s ( + username VARCHAR(100), + successful BOOL, + time INTEGER, + INDEX usr_time_idx (username, time) +)`, authenticationLogsTableName) diff --git a/internal/storage/mysql_provider.go b/internal/storage/mysql_provider.go index 2f2547f8d..fa270e0c1 100644 --- a/internal/storage/mysql_provider.go +++ b/internal/storage/mysql_provider.go @@ -43,6 +43,12 @@ func NewMySQLProvider(configuration schema.MySQLStorageConfiguration) *MySQLProv provider := MySQLProvider{ SQLProvider{ + sqlCreateUserPreferencesTable: SQLCreateUserPreferencesTable, + sqlCreateIdentityVerificationTokensTable: SQLCreateIdentityVerificationTokensTable, + sqlCreateTOTPSecretsTable: SQLCreateTOTPSecretsTable, + sqlCreateU2FDeviceHandlesTable: SQLCreateU2FDeviceHandlesTable, + sqlCreateAuthenticationLogsTable: SQLCreateAuthenticationLogsTable, + sqlGetPreferencesByUsername: fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=?", preferencesTableName), sqlUpsertSecondFactorPreference: fmt.Sprintf("REPLACE INTO %s (username, second_factor_method) VALUES (?, ?)", preferencesTableName), diff --git a/internal/storage/postgres_provider.go b/internal/storage/postgres_provider.go index 324b77a14..d19919086 100644 --- a/internal/storage/postgres_provider.go +++ b/internal/storage/postgres_provider.go @@ -51,6 +51,13 @@ func NewPostgreSQLProvider(configuration schema.PostgreSQLStorageConfiguration) provider := PostgreSQLProvider{ SQLProvider{ + sqlCreateUserPreferencesTable: SQLCreateUserPreferencesTable, + sqlCreateIdentityVerificationTokensTable: SQLCreateIdentityVerificationTokensTable, + sqlCreateTOTPSecretsTable: SQLCreateTOTPSecretsTable, + sqlCreateU2FDeviceHandlesTable: SQLCreateU2FDeviceHandlesTable, + sqlCreateAuthenticationLogsTable: fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100), successful BOOL, time INTEGER)", authenticationLogsTableName), + sqlCreateAuthenticationLogsUserTimeIndex: fmt.Sprintf("CREATE INDEX IF NOT EXISTS usr_time_idx ON %s (username, time)", authenticationLogsTableName), + sqlGetPreferencesByUsername: fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=$1", preferencesTableName), sqlUpsertSecondFactorPreference: fmt.Sprintf("INSERT INTO %s (username, second_factor_method) VALUES ($1, $2) ON CONFLICT (username) DO UPDATE SET second_factor_method=$2", preferencesTableName), diff --git a/internal/storage/sql_provider.go b/internal/storage/sql_provider.go index 5bf8d0c8f..3815fa57f 100644 --- a/internal/storage/sql_provider.go +++ b/internal/storage/sql_provider.go @@ -13,6 +13,13 @@ import ( type SQLProvider struct { db *sql.DB + sqlCreateUserPreferencesTable string + sqlCreateIdentityVerificationTokensTable string + sqlCreateTOTPSecretsTable string + sqlCreateU2FDeviceHandlesTable string + sqlCreateAuthenticationLogsTable string + sqlCreateAuthenticationLogsUserTimeIndex string + sqlGetPreferencesByUsername string sqlUpsertSecondFactorPreference string @@ -34,40 +41,39 @@ type SQLProvider struct { func (p *SQLProvider) initialize(db *sql.DB) error { p.db = db - _, err := db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100) PRIMARY KEY, second_factor_method VARCHAR(11))", preferencesTableName)) + _, err := db.Exec(p.sqlCreateUserPreferencesTable) if err != nil { - return err + return fmt.Errorf("Unable to create table %s: %v", preferencesTableName, err) } - _, err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (token VARCHAR(512))", identityVerificationTokensTableName)) + _, err = db.Exec(p.sqlCreateIdentityVerificationTokensTable) if err != nil { - return err + return fmt.Errorf("Unable to create table %s: %v", identityVerificationTokensTableName, err) } - _, err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100) PRIMARY KEY, secret VARCHAR(64))", totpSecretsTableName)) + _, err = db.Exec(p.sqlCreateTOTPSecretsTable) if err != nil { - return err + return fmt.Errorf("Unable to create table %s: %v", totpSecretsTableName, err) } // keyHandle and publicKey are stored in base64 format - _, err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100) PRIMARY KEY, keyHandle TEXT, publicKey TEXT)", u2fDeviceHandlesTableName)) + _, err = db.Exec(p.sqlCreateU2FDeviceHandlesTable) if err != nil { - return err + return fmt.Errorf("Unable to create table %s: %v", u2fDeviceHandlesTableName, err) } - _, err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100), successful BOOL, time INTEGER)", authenticationLogsTableName)) + _, err = db.Exec(p.sqlCreateAuthenticationLogsTable) if err != nil { - return err + return fmt.Errorf("Unable to create table %s: %v", authenticationLogsTableName, err) } - _, err = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS time ON %s (time);", authenticationLogsTableName)) - if err != nil { - return err - } - - _, err = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS username ON %s (username);", authenticationLogsTableName)) - if err != nil { - return err + // Create an index on (username, time) because this couple is highly used by the regulation module + // to check whether a user is banned. + if p.sqlCreateAuthenticationLogsUserTimeIndex != "" { + _, err = db.Exec(p.sqlCreateAuthenticationLogsUserTimeIndex) + if err != nil { + return fmt.Errorf("Unable to create table %s: %v", authenticationLogsTableName, err) + } } return nil } diff --git a/internal/storage/sqlite_provider.go b/internal/storage/sqlite_provider.go index 133eec12e..aa604d534 100644 --- a/internal/storage/sqlite_provider.go +++ b/internal/storage/sqlite_provider.go @@ -22,6 +22,13 @@ func NewSQLiteProvider(path string) *SQLiteProvider { provider := SQLiteProvider{ SQLProvider{ + sqlCreateUserPreferencesTable: SQLCreateUserPreferencesTable, + sqlCreateIdentityVerificationTokensTable: SQLCreateIdentityVerificationTokensTable, + sqlCreateTOTPSecretsTable: SQLCreateTOTPSecretsTable, + sqlCreateU2FDeviceHandlesTable: SQLCreateU2FDeviceHandlesTable, + sqlCreateAuthenticationLogsTable: fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (username VARCHAR(100), successful BOOL, time INTEGER)", authenticationLogsTableName), + sqlCreateAuthenticationLogsUserTimeIndex: fmt.Sprintf("CREATE INDEX IF NOT EXISTS usr_time_idx ON %s (username, time)", authenticationLogsTableName), + sqlGetPreferencesByUsername: fmt.Sprintf("SELECT second_factor_method FROM %s WHERE username=?", preferencesTableName), sqlUpsertSecondFactorPreference: fmt.Sprintf("REPLACE INTO %s (username, second_factor_method) VALUES (?, ?)", preferencesTableName), diff --git a/internal/suites/MySQL/configuration.yml b/internal/suites/MySQL/configuration.yml new file mode 100644 index 000000000..cdf215996 --- /dev/null +++ b/internal/suites/MySQL/configuration.yml @@ -0,0 +1,68 @@ +############################################################### +# Authelia minimal configuration # +############################################################### + +port: 9091 + +logs_level: debug + +default_redirection_url: https://home.example.com:8080/ + +jwt_secret: very_important_secret + +authentication_backend: + file: + path: /var/lib/authelia/users.yml + +session: + secret: unsecure_session_secret + domain: example.com + expiration: 3600 # 1 hour + inactivity: 300 # 5 minutes + +# Configuration of the storage backend used to store data and secrets. i.e. totp data +storage: + mysql: + host: mysql + port: 3306 + database: authelia + username: admin + password: password + +# TOTP Issuer Name +# +# This will be the issuer name displayed in Google Authenticator +# See: https://github.com/google/google-authenticator/wiki/Key-Uri-Format for more info on issuer names +totp: + issuer: example.com + +access_control: + default_policy: deny + rules: + - domain: "public.example.com" + policy: bypass + - domain: "admin.example.com" + policy: two_factor + - domain: "secure.example.com" + policy: two_factor + - domain: "singlefactor.example.com" + policy: one_factor + +# Configuration of the authentication regulation mechanism. +regulation: + # Set it to 0 to disable max_retries. + max_retries: 3 + + # The user is banned if the authentication failed `max_retries` times in a `find_time` seconds window. + find_time: 8 + + # The length of time before a banned user can login again. + ban_time: 10 + +notifier: + # Use a SMTP server for sending notifications + smtp: + host: smtp + port: 1025 + sender: admin@example.com + disable_require_tls: true \ No newline at end of file diff --git a/internal/suites/MySQL/docker-compose.yml b/internal/suites/MySQL/docker-compose.yml new file mode 100644 index 000000000..b3a10235b --- /dev/null +++ b/internal/suites/MySQL/docker-compose.yml @@ -0,0 +1,6 @@ +version: '3' +services: + authelia-backend: + volumes: + - './MySQL/configuration.yml:/etc/authelia/configuration.yml:ro' + - './MySQL/users.yml:/var/lib/authelia/users.yml' \ No newline at end of file diff --git a/internal/suites/MySQL/users.yml b/internal/suites/MySQL/users.yml new file mode 100644 index 000000000..4f1a7e4e8 --- /dev/null +++ b/internal/suites/MySQL/users.yml @@ -0,0 +1,29 @@ +############################################################### +# Users Database # +############################################################### + +# This file can be used if you do not have an LDAP set up. + +# List of users +users: + john: + password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/" + email: john.doe@authelia.com + groups: + - admins + - dev + + harry: + password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/" + email: harry.potter@authelia.com + groups: [] + + bob: + password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/" + email: bob.dylan@authelia.com + groups: + - dev + + james: + password: "$6$rounds=500000$jgiCMRyGXzoqpxS3$w2pJeZnnH8bwW3zzvoMWtTRfQYsHbWbD/hquuQ5vUeIyl9gdwBIt6RWk2S6afBA0DPakbeWgD/4SZPiS0hYtU/" + email: james.dean@authelia.com \ No newline at end of file diff --git a/internal/suites/example/compose/mysql/docker-compose.yml b/internal/suites/example/compose/mysql/docker-compose.yml new file mode 100644 index 000000000..208cc8b24 --- /dev/null +++ b/internal/suites/example/compose/mysql/docker-compose.yml @@ -0,0 +1,11 @@ +version: '3' +services: + mysql: + image: mysql:8.0 + environment: + - MYSQL_ROOT_PASSWORD=rootpassword + - MYSQL_USER=admin + - MYSQL_PASSWORD=password + - MYSQL_DATABASE=authelia + networks: + - authelianet \ No newline at end of file diff --git a/internal/suites/suite_mysql.go b/internal/suites/suite_mysql.go new file mode 100644 index 000000000..0ec5a2038 --- /dev/null +++ b/internal/suites/suite_mysql.go @@ -0,0 +1,58 @@ +package suites + +import ( + "fmt" + "time" +) + +var mysqlSuiteName = "MySQL" + +func init() { + dockerEnvironment := NewDockerEnvironment([]string{ + "internal/suites/docker-compose.yml", + "internal/suites/MySQL/docker-compose.yml", + "internal/suites/example/compose/authelia/docker-compose.backend.{}.yml", + "internal/suites/example/compose/authelia/docker-compose.frontend.{}.yml", + "internal/suites/example/compose/nginx/backend/docker-compose.yml", + "internal/suites/example/compose/nginx/portal/docker-compose.yml", + "internal/suites/example/compose/smtp/docker-compose.yml", + "internal/suites/example/compose/mysql/docker-compose.yml", + "internal/suites/example/compose/ldap/docker-compose.yml", + }) + + setup := func(suitePath string) error { + if err := dockerEnvironment.Up(); err != nil { + return err + } + + return waitUntilAutheliaBackendIsReady(dockerEnvironment) + } + + onSetupTimeout := func() error { + backendLogs, err := dockerEnvironment.Logs("authelia-backend", nil) + if err != nil { + return err + } + fmt.Println(backendLogs) + + frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil) + if err != nil { + return err + } + fmt.Println(frontendLogs) + return nil + } + + teardown := func(suitePath string) error { + err := dockerEnvironment.Down() + return err + } + + GlobalRegistry.Register(mysqlSuiteName, Suite{ + SetUp: setup, + SetUpTimeout: 5 * time.Minute, + OnSetupTimeout: onSetupTimeout, + TearDown: teardown, + TearDownTimeout: 2 * time.Minute, + }) +} diff --git a/internal/suites/suite_mysql_test.go b/internal/suites/suite_mysql_test.go new file mode 100644 index 000000000..b1813c6d8 --- /dev/null +++ b/internal/suites/suite_mysql_test.go @@ -0,0 +1,27 @@ +package suites + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type MySQLSuite struct { + *SeleniumSuite +} + +func NewMySQLSuite() *MySQLSuite { + return &MySQLSuite{SeleniumSuite: new(SeleniumSuite)} +} + +func (s *MySQLSuite) TestOneFactorScenario() { + suite.Run(s.T(), NewOneFactorScenario()) +} + +func (s *MySQLSuite) TestTwoFactorScenario() { + suite.Run(s.T(), NewTwoFactorScenario()) +} + +func TestMySQLSuite(t *testing.T) { + suite.Run(t, NewMySQLSuite()) +} diff --git a/internal/suites/webdriver.go b/internal/suites/webdriver.go index 0d7ed456b..31fd3cc54 100644 --- a/internal/suites/webdriver.go +++ b/internal/suites/webdriver.go @@ -27,8 +27,13 @@ func StartWebDriverWithProxy(proxy string, port int) (*WebDriverSession, error) return nil, err } + browserPath := os.Getenv("BROWSER_PATH") + if browserPath == "" { + browserPath = "/usr/bin/chromium-browser" + } + chromeCaps := chrome.Capabilities{ - Path: "/usr/bin/chromium-browser", + Path: browserPath, } chromeCaps.Args = append(chromeCaps.Args, "--ignore-certificate-errors")