ci: add integration containers for duo and haproxy (#3479)

* ci: add integration containers for duo and haproxy

This change utilises specific integration containers for the DuoPush and HAProxy suites.
In the case of DuoPush suite specifically in dev mode the container will be built on suite startup.

* ci: factorize pre-command hook and unset async on trigger steps

.buildkite/hooks/post-command

@@ -39,6 +39,10 @@ if [[ "${BUILDKITE_LABEL}" == ":debian: Package Builds" ]]; then
   cat .buildkite/annotations/artifacts | buildkite-agent annotate --style "success" --context "ctx-success"
 fi
 
+if [[ "${BUILDKITE_LABEL}" == ":docker: Build and Deploy Image" ]]; then
+  docker logout
+fi
+
 if [[ "${BUILDKITE_LABEL}" =~ ":docker: Deploy" ]]; then
   docker logout
   docker logout ghcr.io
@@ -78,4 +82,4 @@ if [[ "${BUILDKITE_LABEL}" == ":docker: Deploy Manifest" ]] && [[ "${BUILDKITE_B
       done
     done
   done
-fi
+fi

.buildkite/hooks/pre-command

@@ -2,6 +2,16 @@
 
 set +u
 
+INTEGRATION() {
+  if [[ "${BUILDKITE_BRANCH}" =~ ^renovate/ ]]; then
+    sed -i "s/${CONTAINER}/${CONTAINER}:renovate/" "${FILE}"
+  elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ ! "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+    sed -i "s/${CONTAINER}/${CONTAINER}:${BUILDKITE_BRANCH}/" "${FILE}"
+  elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+    sed -i "s/${CONTAINER}/${CONTAINER}:PR${BUILDKITE_PULL_REQUEST}/" "${FILE}"
+  fi
+}
+
 if [[ "${BUILDKITE_LABEL}" == ":service_dog: Linting" ]]; then
   cd web && pnpm install && cd ../
 fi
@@ -24,6 +34,20 @@ if [[ "${BUILDKITE_LABEL}" =~ ":selenium:" ]]; then
   buildkite-agent artifact download "authelia-image-${DEFAULT_ARCH}*" .
   zstdcat "authelia-image-${DEFAULT_ARCH}.tar.zst" | docker load
   docker tag authelia/authelia authelia:dist
+
+  if [[ "${BUILD_DUO}" == "true" ]] && [[ "${SUITE}" == "DuoPush" ]]; then
+    CONTAINER="integration-duo"
+    FILE="internal/suites/example/compose/duo-api/docker-compose.dist.yml"
+    INTEGRATION
+  elif [[ "${BUILD_HAPROXY}" == "true" ]] && [[ "${SUITE}" == "HAProxy" ]]; then
+    CONTAINER="integration-haproxy"
+    FILE="internal/suites/example/compose/haproxy/docker-compose.yml"
+    INTEGRATION
+  fi
+fi
+
+if [[ "${BUILDKITE_LABEL}" == ":docker: Build and Deploy" ]]; then
+  echo ${DOCKER_PASSWORD} | docker login -u ${DOCKER_USERNAME} --password-stdin
 fi
 
 if [[ "${BUILDKITE_LABEL}" == ":docker: Deploy Manifest" ]]; then
@@ -39,4 +63,4 @@ fi
 
 if [[ "${BUILDKITE_LABEL}" == ":debian: :fedora: :ubuntu: Deploy APT" ]]; then
   buildkite-agent artifact download "authelia_*.deb" .
-fi
+fi

.buildkite/integration.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -u
+
+DIRECTORY="unset"
+GROUP="unset"
+PREFIX="authelia/"
+TAG="unset"
+
+if [[ "${BUILDKITE_BRANCH}" =~ ^renovate/ ]]; then
+  TAG="renovate"
+elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ ! "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+  TAG="${BUILDKITE_BRANCH}"
+elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+  TAG="PR${BUILDKITE_PULL_REQUEST}"
+elif [[ "${BUILDKITE_BRANCH}" == "master" ]] && [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
+  TAG="latest"
+fi
+
+if [[ "${BUILDKITE_PIPELINE_NAME}" == "integration-duo" ]]; then
+  DIRECTORY="internal/suites/example/compose/duo-api"
+  GROUP="duo-deployments"
+elif [[ "${BUILDKITE_PIPELINE_NAME}" == "integration-haproxy" ]]; then
+  DIRECTORY="internal/suites/example/compose/haproxy"
+  GROUP="haproxy-deployments"
+fi
+
+cat << EOF
+steps:
+  - label: ":docker: Build and Deploy"
+    commands:
+      - "cd ${DIRECTORY}"
+      - "docker build --tag ${PREFIX}${BUILDKITE_PIPELINE_NAME}:${TAG} --platform linux/amd64 --builder buildx --pull --push ."
+    concurrency: 1
+    concurrency_group: "${GROUP}"
+    agents:
+      upload: "fast"
+EOF

.buildkite/pipeline.sh

@@ -6,23 +6,33 @@ DIVERGED=$(git merge-base --fork-point origin/master > /dev/null; echo $?)
 if [[ "${DIVERGED}" == 0 ]]; then
   if [[ "${BUILDKITE_TAG}" == "" ]]; then
     if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
+      BUILD_DUO=$(git diff --name-only HEAD~1 | grep -q ^internal/suites/example/compose/duo-api/.* && echo true || echo false)
+      BUILD_HAPROXY=$(git diff --name-only HEAD~1 | grep -q ^internal/suites/example/compose/haproxy/Dockerfile && echo true || echo false)
       CI_BYPASS=$(git diff --name-only HEAD~1 | sed -rn '/^(CODE_OF_CONDUCT\.md|CONTRIBUTING\.md|README\.md|SECURITY\.md|crowdin\.yml|\.all-contributorsrc|\.editorconfig|\.github\/.*|docs\/.*|examples\/.*)/!{q1}' && echo true || echo false)
     else
+      BUILD_DUO=$(git diff --name-only `git merge-base --fork-point origin/master` | grep -q ^internal/suites/example/compose/duo-api/.* && echo true || echo false)
+      BUILD_HAPROXY=$(git diff --name-only `git merge-base --fork-point origin/master` | grep -q ^internal/suites/example/compose/haproxy/Dockerfile && echo true || echo false)
       CI_BYPASS=$(git diff --name-only `git merge-base --fork-point origin/master` | sed -rn '/^(CODE_OF_CONDUCT\.md|CONTRIBUTING\.md|README\.md|SECURITY\.md|crowdin\.yml|\.all-contributorsrc|\.editorconfig|\.github\/.*|docs\/.*|examples\/.*)/!{q1}' && echo true || echo false)
     fi
 
-    if [[ $CI_BYPASS == "true" ]]; then
+    if [[ "${CI_BYPASS}" == "true" ]]; then
       cat .buildkite/annotations/bypass | buildkite-agent annotate --style "info" --context "ctx-info"
     fi
   else
+    BUILD_DUO="false"
+    BUILD_HAPROXY="false"
     CI_BYPASS="false"
   fi
 else
+  BUILD_DUO="false"
+  BUILD_HAPROXY="false"
   CI_BYPASS="false"
 fi
 
 cat << EOF
 env:
+  BUILD_DUO: ${BUILD_DUO}
+  BUILD_HAPROXY: ${BUILD_HAPROXY}
   CI_BYPASS: ${CI_BYPASS}
 
 steps:
@@ -42,7 +52,38 @@ steps:
 
   - wait:
     if: build.env("CI_BYPASS") != "true"
-
+EOF
+if [[ "${BUILD_DUO}" == "true" ]]; then
+cat << EOF
+  - label: ":rocket: Trigger Pipeline [integration-duo]"
+    trigger: "integration-duo"
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}"
+        BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
+    depends_on: ~
+EOF
+fi
+if [[ "${BUILD_HAPROXY}" == "true" ]]; then
+cat << EOF
+  - label: ":rocket: Trigger Pipeline [integration-haproxy]"
+    trigger: "integration-haproxy"
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}"
+        BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
+    depends_on: ~
+EOF
+fi
+cat << EOF
   - label: ":docker: Build Image [coverage]"
     command: "authelia-scripts docker build --container=coverage"
     retry:
@@ -69,4 +110,4 @@ steps:
     depends_on:
       - "build-docker-linux-coverage"
     if: build.branch !~ /^(v[0-9]+\.[0-9]+\.[0-9]+)$\$/ && build.env("CI_BYPASS") != "true" && build.message !~ /\[(skip test|test skip)\]/
-EOF
+EOF

.editorconfig

@@ -7,7 +7,11 @@ trim_trailing_whitespace = true
 end_of_line = lf
 insert_final_newline = true
 
-[*.{yml,yaml}]
+[.buildkite/hooks/**]
+indent_style = space
+indent_size = 2
+
+[*.{sh,yml,yaml}]
 indent_style = space
 indent_size = 2
 

internal/suites/example/compose/duo-api/docker-compose.dist.yml

@@ -0,0 +1,8 @@
+---
+version: '3'
+services:
+  duo-api:
+    image: authelia/integration-duo
+    networks:
+      - authelianet
+...

internal/suites/example/compose/haproxy/docker-compose.yml

@@ -2,7 +2,7 @@
 version: '3'
 services:
   haproxy:
-    build: ./example/compose/haproxy/
+    image: authelia/integration-haproxy
     volumes:
       - ./example/compose/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
       - ./example/compose/haproxy/http.lua:/usr/local/etc/haproxy/haproxy-lua-http/http.lua

internal/suites/suite_bypass_all.go

@@ -17,7 +17,6 @@ func init() {
 		"internal/suites/example/compose/nginx/portal/docker-compose.yml",
 		"internal/suites/example/compose/httpbin/docker-compose.yml",
 		"internal/suites/example/compose/smtp/docker-compose.yml",
-		"internal/suites/example/compose/duo-api/docker-compose.yml",
 	})
 
 	setup := func(suitePath string) error {

internal/suites/suite_duo_push.go

@@ -16,7 +16,7 @@ func init() {
 		"internal/suites/example/compose/authelia/docker-compose.frontend.{}.yml",
 		"internal/suites/example/compose/nginx/backend/docker-compose.yml",
 		"internal/suites/example/compose/nginx/portal/docker-compose.yml",
-		"internal/suites/example/compose/duo-api/docker-compose.yml",
+		"internal/suites/example/compose/duo-api/docker-compose.{}.yml",
 	})
 
 	setup := func(suitePath string) error {