From 91c0c81818d88707f32c4ab9b94c85603b672b9a Mon Sep 17 00:00:00 2001 From: Amir Zarrinkafsh Date: Mon, 2 May 2022 14:50:37 +1000 Subject: [PATCH] refactor(suites): stop integration tests on first failure (#3270) * refactor(suites): stop integration tests on first failure * refactor(suites): remove additional nginx instance * refactor(suites): log relevant containers * refactor(suites): add traefik2 logs to stdout * refactor(suites): explicitly enable traefik for tests * refactor(suites): remove redis restart and duplicate pathprefix tests * ci(buildkite): allow manual retry on integration tests --- .buildkite/steps/e2etests.sh | 4 ++- cmd/authelia-scripts/cmd_suites.go | 9 ++++- .../authelia/docker-compose.backend.dev.yml | 1 + .../authelia/docker-compose.backend.dist.yml | 8 +++-- .../authelia/docker-compose.frontend.dev.yml | 1 + .../authelia/docker-compose.frontend.dist.yml | 1 + .../example/compose/haproxy/haproxy.cfg | 3 +- .../compose/httpbin/docker-compose.yml | 1 + .../compose/nginx/backend/docker-compose.yml | 1 + .../compose/oidc-client/docker-compose.yml | 1 + .../example/compose/smtp/docker-compose.yml | 1 + .../compose/traefik2/docker-compose.yml | 4 ++- internal/suites/suite_haproxy.go | 25 +++++++++++--- internal/suites/suite_oidc_traefik.go | 26 +++++++++++--- internal/suites/suite_pathprefix.go | 25 +++++++++++--- internal/suites/suite_traefik.go | 25 +++++++++++--- internal/suites/suite_traefik2.go | 34 +++++++++++++++++-- internal/suites/suite_traefik2_test.go | 34 ++----------------- 18 files changed, 143 insertions(+), 61 deletions(-) diff --git a/.buildkite/steps/e2etests.sh b/.buildkite/steps/e2etests.sh index 286967fd6..df3d85205 100755 --- a/.buildkite/steps/e2etests.sh +++ b/.buildkite/steps/e2etests.sh @@ -4,9 +4,11 @@ set -eu for SUITE_NAME in $(authelia-scripts suites list); do cat << EOF - label: ":selenium: ${SUITE_NAME} Suite" - command: "authelia-scripts --log-level debug suites test ${SUITE_NAME} --headless" + command: "authelia-scripts --log-level debug suites test ${SUITE_NAME} --failfast --headless" retry: automatic: true + manual: + permit_on_passed: true EOF if [[ "${SUITE_NAME}" = "ActiveDirectory" ]]; then cat << EOF diff --git a/cmd/authelia-scripts/cmd_suites.go b/cmd/authelia-scripts/cmd_suites.go index e78c23cbd..07350fc82 100644 --- a/cmd/authelia-scripts/cmd_suites.go +++ b/cmd/authelia-scripts/cmd_suites.go @@ -26,10 +26,12 @@ var ErrNoRunningSuite = errors.New("no running suite") // runningSuiteFile name of the file containing the currently running suite. var runningSuiteFile = ".suite" +var failfast bool var headless bool var testPattern string func init() { + SuitesTestCmd.Flags().BoolVar(&failfast, "failfast", false, "Stops tests on first failure") SuitesTestCmd.Flags().BoolVar(&headless, "headless", false, "Run tests in headless mode") SuitesTestCmd.Flags().StringVar(&testPattern, "test", "", "The single test to run") } @@ -279,7 +281,12 @@ func runSuiteTests(suiteName string, withEnv bool) error { timeout = fmt.Sprintf("%ds", int64(suite.TestTimeout/time.Second)) } - testCmdLine := fmt.Sprintf("go test -count=1 -v ./internal/suites -timeout %s ", timeout) + fail := "" + if failfast { + fail = "-failfast" + } + + testCmdLine := fmt.Sprintf("go test -count=1 -v ./internal/suites -timeout %s %s ", timeout, fail) if testPattern != "" { testCmdLine += fmt.Sprintf("-run '%s'", testPattern) diff --git a/internal/suites/example/compose/authelia/docker-compose.backend.dev.yml b/internal/suites/example/compose/authelia/docker-compose.backend.dev.yml index bd031a864..70b015131 100644 --- a/internal/suites/example/compose/authelia/docker-compose.backend.dev.yml +++ b/internal/suites/example/compose/authelia/docker-compose.backend.dev.yml @@ -24,6 +24,7 @@ services: - 'traefik.frontend.rule=Host:login.example.com;PathPrefix:/api,/locales' - 'traefik.protocol=https' # Traefik 2.x + - 'traefik.enable=true' - 'traefik.http.routers.authelia_backend.rule=Host(`login.example.com`) && PathPrefix(`/.well-known`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/.well-known`) || Host(`login.example.com`) && PathPrefix(`/api`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/api`) || Host(`login.example.com`) && PathPrefix(`/locales`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/locales`) || Host(`login.example.com`) && Path(`/jwks.json`) || Host(`login.example.com`) && Path(`${PathPrefix}/jwks.json`)' # yamllint disable-line rule:line-length - 'traefik.http.routers.authelia_backend.entrypoints=https' - 'traefik.http.routers.authelia_backend.tls=true' diff --git a/internal/suites/example/compose/authelia/docker-compose.backend.dist.yml b/internal/suites/example/compose/authelia/docker-compose.backend.dist.yml index 5f6c62273..6a6d08b8f 100644 --- a/internal/suites/example/compose/authelia/docker-compose.backend.dist.yml +++ b/internal/suites/example/compose/authelia/docker-compose.backend.dist.yml @@ -5,14 +5,16 @@ services: image: authelia:dist labels: # Traefik 1.x - - 'traefik.frontend.rule=Host:login.example.com;PathPrefix:/api,/locales' + - 'traefik.frontend.rule=Host:login.example.com' - 'traefik.protocol=https' # Traefik 2.x - - 'traefik.http.routers.authelia_backend.rule=Host(`login.example.com`) && PathPrefix(`/.well-known`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/.well-known`) || Host(`login.example.com`) && PathPrefix(`/api`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/api`) || Host(`login.example.com`) && PathPrefix(`/locales`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/locales`) || Host(`login.example.com`) && Path(`/jwks.json`) || Host(`login.example.com`) && Path(`${PathPrefix}/jwks.json`)' # yamllint disable-line rule:line-length + - 'traefik.enable=true' + - 'traefik.http.routers.authelia_backend.rule=Host(`login.example.com`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/`)' # yamllint disable-line rule:line-length - 'traefik.http.routers.authelia_backend.entrypoints=https' - 'traefik.http.routers.authelia_backend.tls=true' - 'traefik.http.services.authelia_backend.loadbalancer.server.scheme=https' - - 'traefik.http.services.authelia_backend.passHostHeader=true' + healthcheck: + disable: true volumes: - '../..:/authelia' environment: diff --git a/internal/suites/example/compose/authelia/docker-compose.frontend.dev.yml b/internal/suites/example/compose/authelia/docker-compose.frontend.dev.yml index 0c6ed5d1e..176f1dc1d 100644 --- a/internal/suites/example/compose/authelia/docker-compose.frontend.dev.yml +++ b/internal/suites/example/compose/authelia/docker-compose.frontend.dev.yml @@ -19,6 +19,7 @@ services: # Traefik 1.x - 'traefik.frontend.rule=Host:login.example.com' # Traefik 2.x + - 'traefik.enable=true' - 'traefik.http.routers.authelia_frontend.rule=Host(`login.example.com`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/`)' # yamllint disable-line rule:line-length - 'traefik.http.routers.authelia_frontend.entrypoints=https' - 'traefik.http.routers.authelia_frontend.tls=true' diff --git a/internal/suites/example/compose/authelia/docker-compose.frontend.dist.yml b/internal/suites/example/compose/authelia/docker-compose.frontend.dist.yml index f4e85176f..1cfbd96db 100644 --- a/internal/suites/example/compose/authelia/docker-compose.frontend.dist.yml +++ b/internal/suites/example/compose/authelia/docker-compose.frontend.dist.yml @@ -9,6 +9,7 @@ services: # Traefik 1.x - 'traefik.frontend.rule=Host:login.example.com' # Traefik 2.x + - 'traefik.enable=true' - 'traefik.http.routers.authelia_frontend.rule=Host(`login.example.com`) || Host(`login.example.com`) && PathPrefix(`${PathPrefix}/`)' # yamllint disable-line rule:line-length - 'traefik.http.routers.authelia_frontend.entrypoints=https' - 'traefik.http.routers.authelia_frontend.tls=true' diff --git a/internal/suites/example/compose/haproxy/haproxy.cfg b/internal/suites/example/compose/haproxy/haproxy.cfg index ed06a4515..a73fbc48f 100644 --- a/internal/suites/example/compose/haproxy/haproxy.cfg +++ b/internal/suites/example/compose/haproxy/haproxy.cfg @@ -73,7 +73,8 @@ backend be_authelia server authelia-backend authelia-backend:9091 resolvers docker ssl verify none backend fe_authelia - server authelia-frontend authelia-frontend:3000 resolvers docker + server authelia-frontend authelia-frontend:3000 check resolvers docker + server authelia-backend authelia-backend:9091 check backup resolvers docker ssl verify none backend be_httpbin acl remote_user_exist var(req.auth_response_header.remote_user) -m found diff --git a/internal/suites/example/compose/httpbin/docker-compose.yml b/internal/suites/example/compose/httpbin/docker-compose.yml index 091bec1c0..fae6542b7 100644 --- a/internal/suites/example/compose/httpbin/docker-compose.yml +++ b/internal/suites/example/compose/httpbin/docker-compose.yml @@ -14,6 +14,7 @@ services: - 'traefik.frontend.auth.forward.trustForwardHeader=true' - 'traefik.frontend.auth.forward.authResponseHeaders=Remote-User, Remote-Groups, Remote-Name, Remote-Email' # Traefik 2.x + - 'traefik.enable=true' - 'traefik.http.routers.httpbin.rule=Host(`public.example.com`) && Path(`/headers`)' - 'traefik.http.routers.httpbin.priority=150' - 'traefik.http.routers.httpbin.tls=true' diff --git a/internal/suites/example/compose/nginx/backend/docker-compose.yml b/internal/suites/example/compose/nginx/backend/docker-compose.yml index 00c1f5633..89ed3adbe 100644 --- a/internal/suites/example/compose/nginx/backend/docker-compose.yml +++ b/internal/suites/example/compose/nginx/backend/docker-compose.yml @@ -11,6 +11,7 @@ services: - 'traefik.frontend.auth.forward.trustForwardHeader=true' - 'traefik.frontend.auth.forward.authResponseHeaders=Remote-User,Remote-Groups,Remote-Name,Remote-Email' # Traefik 2.x + - 'traefik.enable=true' - 'traefik.http.routers.protectedapps.rule=Host(`home.example.com`,`public.example.com`,`secure.example.com`,`admin.example.com`,`singlefactor.example.com`)' # yamllint disable-line rule:line-length - 'traefik.http.routers.protectedapps.entrypoints=https' - 'traefik.http.routers.protectedapps.tls=true' diff --git a/internal/suites/example/compose/oidc-client/docker-compose.yml b/internal/suites/example/compose/oidc-client/docker-compose.yml index 0a150c36c..3a9ecc808 100644 --- a/internal/suites/example/compose/oidc-client/docker-compose.yml +++ b/internal/suites/example/compose/oidc-client/docker-compose.yml @@ -11,6 +11,7 @@ services: expose: - 8080 labels: + - 'traefik.enable=true' - 'traefik.http.routers.oidc.rule=Host(`oidc.example.com`)' - 'traefik.http.routers.oidc.priority=150' - 'traefik.http.routers.oidc.tls=true' diff --git a/internal/suites/example/compose/smtp/docker-compose.yml b/internal/suites/example/compose/smtp/docker-compose.yml index 3305d0fa6..c5156110b 100644 --- a/internal/suites/example/compose/smtp/docker-compose.yml +++ b/internal/suites/example/compose/smtp/docker-compose.yml @@ -8,6 +8,7 @@ services: labels: - 'traefik.frontend.rule=Host:mail.example.com' - 'traefik.port=1080' + - 'traefik.enable=true' - 'traefik.http.routers.mail.rule=Host(`mail.example.com`)' - 'traefik.http.routers.mail.entrypoints=https' - 'traefik.http.routers.mail.tls=true' diff --git a/internal/suites/example/compose/traefik2/docker-compose.yml b/internal/suites/example/compose/traefik2/docker-compose.yml index 329fdbff7..fb43530fd 100644 --- a/internal/suites/example/compose/traefik2/docker-compose.yml +++ b/internal/suites/example/compose/traefik2/docker-compose.yml @@ -6,6 +6,7 @@ services: volumes: - '/var/run/docker.sock:/var/run/docker.sock' labels: + - 'traefik.enable=true' - 'traefik.http.routers.api.rule=Host(`traefik.example.com`)' - 'traefik.http.routers.api.entrypoints=https' - 'traefik.http.routers.api.service=api@internal' @@ -16,13 +17,14 @@ services: - 'traefik.http.middlewares.authelia.forwardauth.trustForwardHeader=true' - 'traefik.http.middlewares.authelia.forwardauth.authResponseHeaders=Remote-User, Remote-Groups, Remote-Name, Remote-Email' # yamllint disable-line rule:line-length command: + - '--accesslog=true' - '--api' - '--providers.docker=true' - '--entrypoints.https=true' - '--entrypoints.https.address=:8080' - '--log=true' - '--log.level=DEBUG' - - '--log.filepath=/var/log/traefik.log' + - '--providers.docker.exposedByDefault=false' - '--serversTransport.insecureSkipVerify=true' networks: authelianet: diff --git a/internal/suites/suite_haproxy.go b/internal/suites/suite_haproxy.go index 160d1b9ef..fff70feef 100644 --- a/internal/suites/suite_haproxy.go +++ b/internal/suites/suite_haproxy.go @@ -2,6 +2,7 @@ package suites import ( "fmt" + "os" "time" ) @@ -19,6 +20,18 @@ func init() { "internal/suites/example/compose/httpbin/docker-compose.yml", }) + if os.Getenv("CI") == t { + dockerEnvironment = NewDockerEnvironment([]string{ + "internal/suites/docker-compose.yml", + "internal/suites/HAProxy/docker-compose.yml", + "internal/suites/example/compose/authelia/docker-compose.backend.{}.yml", + "internal/suites/example/compose/nginx/backend/docker-compose.yml", + "internal/suites/example/compose/haproxy/docker-compose.yml", + "internal/suites/example/compose/smtp/docker-compose.yml", + "internal/suites/example/compose/httpbin/docker-compose.yml", + }) + } + setup := func(suitePath string) error { if err := dockerEnvironment.Up(); err != nil { return err @@ -35,12 +48,14 @@ func init() { fmt.Println(backendLogs) - frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil) - if err != nil { - return err - } + if os.Getenv("CI") != t { + frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil) + if err != nil { + return err + } - fmt.Println(frontendLogs) + fmt.Println(frontendLogs) + } haproxyLogs, err := dockerEnvironment.Logs("haproxy", nil) if err != nil { diff --git a/internal/suites/suite_oidc_traefik.go b/internal/suites/suite_oidc_traefik.go index 38f4a5b94..8e03b2aef 100644 --- a/internal/suites/suite_oidc_traefik.go +++ b/internal/suites/suite_oidc_traefik.go @@ -2,6 +2,7 @@ package suites import ( "fmt" + "os" "time" ) @@ -20,6 +21,19 @@ func init() { "internal/suites/example/compose/redis/docker-compose.yml", }) + if os.Getenv("CI") == t { + dockerEnvironment = NewDockerEnvironment([]string{ + "internal/suites/docker-compose.yml", + "internal/suites/OIDCTraefik/docker-compose.yml", + "internal/suites/example/compose/authelia/docker-compose.backend.{}.yml", + "internal/suites/example/compose/nginx/backend/docker-compose.yml", + "internal/suites/example/compose/traefik2/docker-compose.yml", + "internal/suites/example/compose/smtp/docker-compose.yml", + "internal/suites/example/compose/oidc-client/docker-compose.yml", + "internal/suites/example/compose/redis/docker-compose.yml", + }) + } + setup := func(suitePath string) error { // TODO(c.michaud): use version in tags for oidc-client but in the meantime we pull the image to make sure it's // up to date. @@ -46,12 +60,14 @@ func init() { fmt.Println(backendLogs) - frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil) - if err != nil { - return err - } + if os.Getenv("CI") != t { + frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil) + if err != nil { + return err + } - fmt.Println(frontendLogs) + fmt.Println(frontendLogs) + } oidcClientLogs, err := dockerEnvironment.Logs("oidc-client", nil) if err != nil { diff --git a/internal/suites/suite_pathprefix.go b/internal/suites/suite_pathprefix.go index 061660e03..d695f8c95 100644 --- a/internal/suites/suite_pathprefix.go +++ b/internal/suites/suite_pathprefix.go @@ -2,6 +2,7 @@ package suites import ( "fmt" + "os" "time" ) @@ -19,6 +20,18 @@ func init() { "internal/suites/example/compose/httpbin/docker-compose.yml", }) + if os.Getenv("CI") == t { + dockerEnvironment = NewDockerEnvironment([]string{ + "internal/suites/docker-compose.yml", + "internal/suites/PathPrefix/docker-compose.yml", + "internal/suites/example/compose/authelia/docker-compose.backend.{}.yml", + "internal/suites/example/compose/nginx/backend/docker-compose.yml", + "internal/suites/example/compose/traefik2/docker-compose.yml", + "internal/suites/example/compose/smtp/docker-compose.yml", + "internal/suites/example/compose/httpbin/docker-compose.yml", + }) + } + setup := func(suitePath string) error { if err := dockerEnvironment.Up(); err != nil { return err @@ -35,12 +48,14 @@ func init() { fmt.Println(backendLogs) - frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil) - if err != nil { - return err - } + if os.Getenv("CI") != t { + frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil) + if err != nil { + return err + } - fmt.Println(frontendLogs) + fmt.Println(frontendLogs) + } return nil } diff --git a/internal/suites/suite_traefik.go b/internal/suites/suite_traefik.go index de6d19a03..3d3712018 100644 --- a/internal/suites/suite_traefik.go +++ b/internal/suites/suite_traefik.go @@ -2,6 +2,7 @@ package suites import ( "fmt" + "os" "time" ) @@ -19,6 +20,18 @@ func init() { "internal/suites/example/compose/httpbin/docker-compose.yml", }) + if os.Getenv("CI") == t { + dockerEnvironment = NewDockerEnvironment([]string{ + "internal/suites/docker-compose.yml", + "internal/suites/Traefik/docker-compose.yml", + "internal/suites/example/compose/authelia/docker-compose.backend.{}.yml", + "internal/suites/example/compose/nginx/backend/docker-compose.yml", + "internal/suites/example/compose/traefik/docker-compose.yml", + "internal/suites/example/compose/smtp/docker-compose.yml", + "internal/suites/example/compose/httpbin/docker-compose.yml", + }) + } + setup := func(suitePath string) error { if err := dockerEnvironment.Up(); err != nil { return err @@ -35,12 +48,14 @@ func init() { fmt.Println(backendLogs) - frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil) - if err != nil { - return err - } + if os.Getenv("CI") != t { + frontendLogs, err := dockerEnvironment.Logs("authelia-frontend", nil) + if err != nil { + return err + } - fmt.Println(frontendLogs) + fmt.Println(frontendLogs) + } return nil } diff --git a/internal/suites/suite_traefik2.go b/internal/suites/suite_traefik2.go index cb5c8a0e3..c2f185fee 100644 --- a/internal/suites/suite_traefik2.go +++ b/internal/suites/suite_traefik2.go @@ -2,6 +2,7 @@ package suites import ( "fmt" + "os" "time" ) @@ -20,6 +21,19 @@ var traefik2DockerEnvironment = NewDockerEnvironment([]string{ }) func init() { + if os.Getenv("CI") == t { + traefik2DockerEnvironment = NewDockerEnvironment([]string{ + "internal/suites/docker-compose.yml", + "internal/suites/Traefik2/docker-compose.yml", + "internal/suites/example/compose/authelia/docker-compose.backend.{}.yml", + "internal/suites/example/compose/redis/docker-compose.yml", + "internal/suites/example/compose/nginx/backend/docker-compose.yml", + "internal/suites/example/compose/traefik2/docker-compose.yml", + "internal/suites/example/compose/smtp/docker-compose.yml", + "internal/suites/example/compose/httpbin/docker-compose.yml", + }) + } + setup := func(suitePath string) error { if err := traefik2DockerEnvironment.Up(); err != nil { return err @@ -36,12 +50,28 @@ func init() { fmt.Println(backendLogs) - frontendLogs, err := traefik2DockerEnvironment.Logs("authelia-frontend", nil) + if os.Getenv("CI") != t { + frontendLogs, err := traefik2DockerEnvironment.Logs("authelia-frontend", nil) + if err != nil { + return err + } + + fmt.Println(frontendLogs) + } + + redisLogs, err := traefik2DockerEnvironment.Logs("redis", nil) if err != nil { return err } - fmt.Println(frontendLogs) + fmt.Println(redisLogs) + + traefikLogs, err := traefik2DockerEnvironment.Logs("traefik", nil) + if err != nil { + return err + } + + fmt.Println(traefikLogs) return nil } diff --git a/internal/suites/suite_traefik2_test.go b/internal/suites/suite_traefik2_test.go index 0cbddf110..c9d08c5ed 100644 --- a/internal/suites/suite_traefik2_test.go +++ b/internal/suites/suite_traefik2_test.go @@ -1,10 +1,7 @@ package suites import ( - "context" - "fmt" "testing" - "time" "github.com/stretchr/testify/suite" ) @@ -29,35 +26,8 @@ func (s *Traefik2Suite) TestCustomHeaders() { suite.Run(s.T(), NewCustomHeadersScenario()) } -func (s *Traefik2Suite) TestShouldKeepSessionAfterRedisRestart() { - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) - defer func() { - cancel() - s.collectCoverage(s.Page) - s.collectScreenshot(ctx.Err(), s.Page) - s.MustClose() - err := s.RodSession.Stop() - s.Require().NoError(err) - }() - - browser, err := StartRod() - s.Require().NoError(err) - s.RodSession = browser - - s.Page = s.doCreateTab(s.T(), HomeBaseURL) - s.verifyIsHome(s.T(), s.Page) - secret := s.doRegisterThenLogout(s.T(), s.Context(ctx), "john", "password") - - s.doLoginTwoFactor(s.T(), s.Context(ctx), "john", "password", false, secret, "") - - s.doVisit(s.T(), s.Context(ctx), fmt.Sprintf("%s/secret.html", SecureBaseURL)) - s.verifySecretAuthorized(s.T(), s.Context(ctx)) - - err = traefik2DockerEnvironment.Restart("redis") - s.Require().NoError(err) - - s.doVisit(s.T(), s.Context(ctx), fmt.Sprintf("%s/secret.html", SecureBaseURL)) - s.verifySecretAuthorized(s.T(), s.Context(ctx)) +func (s *Traefik2Suite) TestResetPasswordScenario() { + suite.Run(s.T(), NewResetPasswordScenario()) } func TestTraefik2Suite(t *testing.T) {