ci: add integration containers for duo and haproxy (#3479)

* ci: add integration containers for duo and haproxy

This change utilises specific integration containers for the DuoPush and HAProxy suites.
In the case of DuoPush suite specifically in dev mode the container will be built on suite startup.

* ci: factorize pre-command hook and unset async on trigger steps

.buildkite/hooks/post-command

@@ -39,6 +39,10 @@ if [[ "${BUILDKITE_LABEL}" == ":debian: Package Builds" ]]; then
   cat .buildkite/annotations/artifacts | buildkite-agent annotate --style "success" --context "ctx-success"
 fi
 
+if [[ "${BUILDKITE_LABEL}" == ":docker: Build and Deploy Image" ]]; then
+  docker logout
+fi
+
 if [[ "${BUILDKITE_LABEL}" =~ ":docker: Deploy" ]]; then
   docker logout
   docker logout ghcr.io

.buildkite/hooks/pre-command

@@ -2,6 +2,16 @@
 
 set +u
 
+INTEGRATION() {
+  if [[ "${BUILDKITE_BRANCH}" =~ ^renovate/ ]]; then
+    sed -i "s/${CONTAINER}/${CONTAINER}:renovate/" "${FILE}"
+  elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ ! "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+    sed -i "s/${CONTAINER}/${CONTAINER}:${BUILDKITE_BRANCH}/" "${FILE}"
+  elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+    sed -i "s/${CONTAINER}/${CONTAINER}:PR${BUILDKITE_PULL_REQUEST}/" "${FILE}"
+  fi
+}
+
 if [[ "${BUILDKITE_LABEL}" == ":service_dog: Linting" ]]; then
   cd web && pnpm install && cd ../
 fi
@@ -24,6 +34,20 @@ if [[ "${BUILDKITE_LABEL}" =~ ":selenium:" ]]; then
   buildkite-agent artifact download "authelia-image-${DEFAULT_ARCH}*" .
   zstdcat "authelia-image-${DEFAULT_ARCH}.tar.zst" | docker load
   docker tag authelia/authelia authelia:dist
+
+  if [[ "${BUILD_DUO}" == "true" ]] && [[ "${SUITE}" == "DuoPush" ]]; then
+    CONTAINER="integration-duo"
+    FILE="internal/suites/example/compose/duo-api/docker-compose.dist.yml"
+    INTEGRATION
+  elif [[ "${BUILD_HAPROXY}" == "true" ]] && [[ "${SUITE}" == "HAProxy" ]]; then
+    CONTAINER="integration-haproxy"
+    FILE="internal/suites/example/compose/haproxy/docker-compose.yml"
+    INTEGRATION
+  fi
+fi
+
+if [[ "${BUILDKITE_LABEL}" == ":docker: Build and Deploy" ]]; then
+  echo ${DOCKER_PASSWORD} | docker login -u ${DOCKER_USERNAME} --password-stdin
 fi
 
 if [[ "${BUILDKITE_LABEL}" == ":docker: Deploy Manifest" ]]; then

.buildkite/integration.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -u
+
+DIRECTORY="unset"
+GROUP="unset"
+PREFIX="authelia/"
+TAG="unset"
+
+if [[ "${BUILDKITE_BRANCH}" =~ ^renovate/ ]]; then
+  TAG="renovate"
+elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ ! "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+  TAG="${BUILDKITE_BRANCH}"
+elif [[ "${BUILDKITE_BRANCH}" != "master" ]] && [[ "${BUILDKITE_BRANCH}" =~ .*:.* ]]; then
+  TAG="PR${BUILDKITE_PULL_REQUEST}"
+elif [[ "${BUILDKITE_BRANCH}" == "master" ]] && [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
+  TAG="latest"
+fi
+
+if [[ "${BUILDKITE_PIPELINE_NAME}" == "integration-duo" ]]; then
+  DIRECTORY="internal/suites/example/compose/duo-api"
+  GROUP="duo-deployments"
+elif [[ "${BUILDKITE_PIPELINE_NAME}" == "integration-haproxy" ]]; then
+  DIRECTORY="internal/suites/example/compose/haproxy"
+  GROUP="haproxy-deployments"
+fi
+
+cat << EOF
+steps:
+  - label: ":docker: Build and Deploy"
+    commands:
+      - "cd ${DIRECTORY}"
+      - "docker build --tag ${PREFIX}${BUILDKITE_PIPELINE_NAME}:${TAG} --platform linux/amd64 --builder buildx --pull --push ."
+    concurrency: 1
+    concurrency_group: "${GROUP}"
+    agents:
+      upload: "fast"
+EOF

.buildkite/pipeline.sh

@@ -6,23 +6,33 @@ DIVERGED=$(git merge-base --fork-point origin/master > /dev/null; echo $?)
 if [[ "${DIVERGED}" == 0 ]]; then
   if [[ "${BUILDKITE_TAG}" == "" ]]; then
     if [[ "${BUILDKITE_BRANCH}" == "master" ]]; then
+      BUILD_DUO=$(git diff --name-only HEAD~1 | grep -q ^internal/suites/example/compose/duo-api/.* && echo true || echo false)
+      BUILD_HAPROXY=$(git diff --name-only HEAD~1 | grep -q ^internal/suites/example/compose/haproxy/Dockerfile && echo true || echo false)
       CI_BYPASS=$(git diff --name-only HEAD~1 | sed -rn '/^(CODE_OF_CONDUCT\.md|CONTRIBUTING\.md|README\.md|SECURITY\.md|crowdin\.yml|\.all-contributorsrc|\.editorconfig|\.github\/.*|docs\/.*|examples\/.*)/!{q1}' && echo true || echo false)
     else
+      BUILD_DUO=$(git diff --name-only `git merge-base --fork-point origin/master` | grep -q ^internal/suites/example/compose/duo-api/.* && echo true || echo false)
+      BUILD_HAPROXY=$(git diff --name-only `git merge-base --fork-point origin/master` | grep -q ^internal/suites/example/compose/haproxy/Dockerfile && echo true || echo false)
       CI_BYPASS=$(git diff --name-only `git merge-base --fork-point origin/master` | sed -rn '/^(CODE_OF_CONDUCT\.md|CONTRIBUTING\.md|README\.md|SECURITY\.md|crowdin\.yml|\.all-contributorsrc|\.editorconfig|\.github\/.*|docs\/.*|examples\/.*)/!{q1}' && echo true || echo false)
     fi
 
-    if [[ $CI_BYPASS == "true" ]]; then
+    if [[ "${CI_BYPASS}" == "true" ]]; then
       cat .buildkite/annotations/bypass | buildkite-agent annotate --style "info" --context "ctx-info"
     fi
   else
+    BUILD_DUO="false"
+    BUILD_HAPROXY="false"
     CI_BYPASS="false"
   fi
 else
+  BUILD_DUO="false"
+  BUILD_HAPROXY="false"
   CI_BYPASS="false"
 fi
 
 cat << EOF
 env:
+  BUILD_DUO: ${BUILD_DUO}
+  BUILD_HAPROXY: ${BUILD_HAPROXY}
   CI_BYPASS: ${CI_BYPASS}
 
 steps:
@@ -42,7 +52,38 @@ steps:
 
   - wait:
     if: build.env("CI_BYPASS") != "true"
-
+EOF
+if [[ "${BUILD_DUO}" == "true" ]]; then
+cat << EOF
+  - label: ":rocket: Trigger Pipeline [integration-duo]"
+    trigger: "integration-duo"
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}"
+        BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
+    depends_on: ~
+EOF
+fi
+if [[ "${BUILD_HAPROXY}" == "true" ]]; then
+cat << EOF
+  - label: ":rocket: Trigger Pipeline [integration-haproxy]"
+    trigger: "integration-haproxy"
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST}"
+        BUILDKITE_PULL_REQUEST_BASE_BRANCH: "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        BUILDKITE_PULL_REQUEST_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
+    depends_on: ~
+EOF
+fi
+cat << EOF
   - label: ":docker: Build Image [coverage]"
     command: "authelia-scripts docker build --container=coverage"
     retry:

.editorconfig

@@ -7,7 +7,11 @@ trim_trailing_whitespace = true
 end_of_line = lf
 insert_final_newline = true
 
-[*.{yml,yaml}]
+[.buildkite/hooks/**]
+indent_style = space
+indent_size = 2
+
+[*.{sh,yml,yaml}]
 indent_style = space
 indent_size = 2
 

internal/suites/example/compose/duo-api/docker-compose.dist.yml

@@ -0,0 +1,8 @@
+---
+version: '3'
+services:
+  duo-api:
+    image: authelia/integration-duo
+    networks:
+      - authelianet
+...

internal/suites/example/compose/haproxy/docker-compose.yml

@@ -2,7 +2,7 @@
 version: '3'
 services:
   haproxy:
-    build: ./example/compose/haproxy/
+    image: authelia/integration-haproxy
     volumes:
       - ./example/compose/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
       - ./example/compose/haproxy/http.lua:/usr/local/etc/haproxy/haproxy-lua-http/http.lua

internal/suites/suite_bypass_all.go

@@ -17,7 +17,6 @@ func init() {
 		"internal/suites/example/compose/nginx/portal/docker-compose.yml",
 		"internal/suites/example/compose/httpbin/docker-compose.yml",
 		"internal/suites/example/compose/smtp/docker-compose.yml",
-		"internal/suites/example/compose/duo-api/docker-compose.yml",
 	})
 
 	setup := func(suitePath string) error {

internal/suites/suite_duo_push.go

@@ -16,7 +16,7 @@ func init() {
 		"internal/suites/example/compose/authelia/docker-compose.frontend.{}.yml",
 		"internal/suites/example/compose/nginx/backend/docker-compose.yml",
 		"internal/suites/example/compose/nginx/portal/docker-compose.yml",
-		"internal/suites/example/compose/duo-api/docker-compose.yml",
+		"internal/suites/example/compose/duo-api/docker-compose.{}.yml",
 	})
 
 	setup := func(suitePath string) error {