From 49e739d0091d3d6e733ee1a1b13e5eac581db486 Mon Sep 17 00:00:00 2001 From: Amir Zarrinkafsh Date: Thu, 30 Jan 2020 18:37:11 +1100 Subject: [PATCH] [Buildkite] Add automatic deployment and removal of Docker images for Branches and PRs (#592) --- .buildkite/hooks/post-checkout | 9 +++------ .buildkite/hooks/post-command | 28 ++++++++++++++++++++++++---- .buildkite/hooks/pre-artifact | 7 +++---- .buildkite/hooks/pre-command | 9 +++------ .buildkite/pipeline.yml | 8 ++------ .buildkite/steps/buildimages.sh | 4 ---- cmd/authelia-scripts/cmd_docker.go | 17 +++++++++++++++-- 7 files changed, 50 insertions(+), 32 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index d5f9a7792..fb44f1fc1 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -3,16 +3,13 @@ set +u if [[ ! $BUILDKITE_COMMAND =~ "buildkite-agent pipeline upload" ]] || \ -[[ $BUILDKITE_COMMAND == ".buildkite/steps/e2etests.sh | buildkite-agent pipeline upload" ]]; -then +[[ $BUILDKITE_COMMAND == ".buildkite/steps/e2etests.sh | buildkite-agent pipeline upload" ]]; then echo "--- :buildkite: Setting up Build environment" source bootstrap.sh - if [[ $BUILDKITE_COMMAND == "authelia-scripts --log-level debug ci" ]]; - then + if [[ $BUILDKITE_COMMAND == "authelia-scripts --log-level debug ci" ]]; then go mod download fi - if [[ $BUILDKITE_LABEL =~ ":selenium:" ]]; - then + if [[ $BUILDKITE_LABEL =~ ":selenium:" ]]; then go mod download fi fi \ No newline at end of file diff --git a/.buildkite/hooks/post-command b/.buildkite/hooks/post-command index d37657e6d..3a0d43448 100755 --- a/.buildkite/hooks/post-command +++ b/.buildkite/hooks/post-command @@ -2,12 +2,32 @@ set +u -if [[ $BUILDKITE_LABEL =~ ":selenium:" ]] || [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]]; -then +if [[ $BUILDKITE_LABEL =~ ":selenium:" ]] || [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]]; then CONTAINERS=$(docker ps -a -q) - if [[ ${CONTAINERS} != "" ]]; - then + if [[ ${CONTAINERS} != "" ]]; then echo "--- :docker: Clean environment" docker rm -f ${CONTAINERS} fi +fi + +if [[ $BUILDKITE_BRANCH == "master" ]] || [[ $BUILDKITE_BRANCH =~ ^v.* ]] && [[ $BUILDKITE_PULL_REQUEST == "false" ]]; then + if [[ $BUILDKITE_LABEL == ":docker: Deploy Manifests" ]]; then + echo "--- :docker: Removing tags for deleted branches" + for BRANCH_TAG in $(dockerbranchtags=$(anontoken=$(curl -fsL --retry 3 'https://auth.docker.io/token?service=registry.docker.io&scope=repository:authelia/authelia:pull' | jq -r .token) && \ + curl -fsL --retry 3 -H "Authorization: Bearer ${anontoken}" https://registry-1.docker.io/v2/authelia/authelia/tags/list | jq -r '.tags[] | select(startswith("PR") | not)' | \ + sed -r '/^(latest|develop|v.*|([[:digit:]]+)\.?([[:digit:]]+)?\.?([[:digit:]]+)?)/d' | sort) && \ + githubbranches=$(curl -fs --retry 3 https://api.github.com/repos/authelia/authelia/branches | jq -r '.[].name' | sort) && \ + comm -23 <(echo "${dockerbranchtags}") <(echo "${githubbranches}")); do + echo "Removing tag ${BRANCH_TAG}" + authtoken=$(curl -fs --retry 3 -H "Content-Type: application/json" -X "POST" -d '{"username": "'${DOCKER_USERNAME}'", "password": "'${DOCKER_PASSWORD}'"}' https://hub.docker.com/v2/users/login/ | jq -r .token) && \ + curl -fsL --retry 3 -o /dev/null -X "DELETE" -H "Authorization: JWT ${authtoken}" https://hub.docker.com/v2/repositories/authelia/authelia/tags/${BRANCH_TAG}/ + done + echo "--- :docker: Removing tags for merged or closed pull requests" + for PR_TAG in $(dockerprtags=$(curl -fsL --retry 3 -H "Authorization: Bearer ${anontoken}" https://registry-1.docker.io/v2/authelia/authelia/tags/list | jq -r '.tags[] | select(startswith("PR"))' | sort) && \ + githubprs=$(curl -fs --retry 3 https://api.github.com/repos/authelia/authelia/pulls | jq -r '.[].number' | sed -e 's/^/PR/' | sort) && \ + comm -23 <(echo "${dockerprtags}") <(echo "${githubprs}")); do + echo "Removing tag ${PR_TAG}" + curl -fsL --retry 3 -o /dev/null -X "DELETE" -H "Authorization: JWT ${authtoken}" https://hub.docker.com/v2/repositories/authelia/authelia/tags/${PR_TAG}/ + done + fi fi \ No newline at end of file diff --git a/.buildkite/hooks/pre-artifact b/.buildkite/hooks/pre-artifact index 9275390cb..ab4b9ade5 100755 --- a/.buildkite/hooks/pre-artifact +++ b/.buildkite/hooks/pre-artifact @@ -4,16 +4,15 @@ set +u DOCKER_IMAGE=authelia/authelia -if [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]]; -then +if [[ $BUILDKITE_LABEL =~ ":docker: Build Image" ]]; then echo "--- :docker: Saving artifacts for :buildkite: :docker: :github: releases" # Save binary for buildkite and github artifacts - docker create --name authelia-binary $DOCKER_IMAGE:latest + docker create --name authelia-binary ${DOCKER_IMAGE}:latest docker cp authelia-binary:/usr/app/authelia ./authelia-linux-"${ARCH}" docker cp authelia-binary:/usr/app/public_html ./ docker rm -f authelia-binary tar -czf authelia-linux-"${ARCH}".tar.gz authelia-linux-"${ARCH}" authelia.service config.template.yml public_html sha256sum authelia-linux-"${ARCH}".tar.gz > authelia-linux-"${ARCH}".tar.gz.sha256 # Saving image for push to docker hub - docker save $DOCKER_IMAGE | zstdmt -T0 -12 > authelia-image-"${ARCH}".tar.zst + docker save ${DOCKER_IMAGE} | zstdmt -T0 -12 > authelia-image-"${ARCH}".tar.zst fi \ No newline at end of file diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 5f4670f59..6235700b5 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,8 +2,7 @@ set +u -if [[ $BUILDKITE_LABEL =~ ":selenium:" ]]; -then +if [[ $BUILDKITE_LABEL =~ ":selenium:" ]]; then DEFAULT_ARCH=amd64 echo "--- :docker: Extract, load and tag build container" buildkite-agent artifact download "authelia-image-${DEFAULT_ARCH}*" . @@ -11,13 +10,11 @@ then docker tag authelia/authelia authelia:dist fi -if [[ $BUILDKITE_LABEL =~ ":docker: Deploy Image" ]]; -then +if [[ $BUILDKITE_LABEL =~ ":docker: Deploy Image" ]]; then buildkite-agent artifact download "authelia-image-${ARCH}*" . zstdcat authelia-image-"${ARCH}".tar.zst | docker load fi -if [[ $BUILDKITE_LABEL == ":github: Deploy Artifacts" ]]; -then +if [[ $BUILDKITE_LABEL == ":github: Deploy Artifacts" ]]; then buildkite-agent artifact download "authelia-linux-*" . fi \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 02300e659..58dd9e493 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -17,21 +17,17 @@ steps: depends_on: - "build-docker-amd64" - - wait: - if: build.branch == "master" || build.branch =~ /^v/ + - wait - label: ":docker: Image Deployments" command: ".buildkite/steps/deployimages.sh | buildkite-agent pipeline upload" - branches: "master v*" concurrency: 1 concurrency_group: "deployments" - - wait: - if: build.branch == "master" || build.branch =~ /^v/ + - wait - label: ":docker: Deploy Manifests" command: "authelia-scripts docker push-manifest" - branches: "master v*" concurrency: 1 concurrency_group: "deployments" env: diff --git a/.buildkite/steps/buildimages.sh b/.buildkite/steps/buildimages.sh index f6536e2aa..2c857e29a 100755 --- a/.buildkite/steps/buildimages.sh +++ b/.buildkite/steps/buildimages.sh @@ -10,10 +10,6 @@ do echo " - \"authelia-image-${BUILD_ARCH}.tar.zst\"" echo " - \"authelia-linux-${BUILD_ARCH}.tar.gz\"" echo " - \"authelia-linux-${BUILD_ARCH}.tar.gz.sha256\"" - if [[ "${BUILD_ARCH}" != "amd64" ]]; - then - echo " branches: \"master v*\"" - fi echo " env:" echo " "ARCH: ${BUILD_ARCH}"" echo " key: \"build-docker-${BUILD_ARCH}\"" diff --git a/cmd/authelia-scripts/cmd_docker.go b/cmd/authelia-scripts/cmd_docker.go index f8d15fd78..c9033739e 100644 --- a/cmd/authelia-scripts/cmd_docker.go +++ b/cmd/authelia-scripts/cmd_docker.go @@ -23,6 +23,7 @@ var ciPullRequest = os.Getenv("BUILDKITE_PULL_REQUEST") var ciTag = os.Getenv("BUILDKITE_TAG") var dockerTags = regexp.MustCompile(`v(?P(?P(?P\d+)\.\d+)\.\d+.*)`) var ignoredSuffixes = regexp.MustCompile("alpha|beta") +var publicRepo = regexp.MustCompile(`.*\:.*`) var tags = dockerTags.FindStringSubmatch(ciTag) func init() { @@ -202,7 +203,13 @@ func deployManifest(docker *Docker, tag string, amd64tag string, arm32v7tag stri func publishDockerImage(arch string) { docker := &Docker{} - if ciBranch == "master" && ciPullRequest == "false" { + if ciBranch != "master" && !publicRepo.MatchString(ciBranch) { + login(docker) + deploy(docker, ciBranch+"-"+arch) + } else if ciBranch != "master" && publicRepo.MatchString(ciBranch) { + login(docker) + deploy(docker, "PR"+ciPullRequest+"-"+arch) + } else if ciBranch == "master" && ciPullRequest == "false" { login(docker) deploy(docker, "master-"+arch) } else if ciTag != "" { @@ -227,7 +234,13 @@ func publishDockerImage(arch string) { func publishDockerManifest() { docker := &Docker{} - if ciBranch == "master" && ciPullRequest == "false" { + if ciBranch != "master" && !publicRepo.MatchString(ciBranch) { + login(docker) + deployManifest(docker, ciBranch, ciBranch+"-amd64", ciBranch+"-arm32v7", ciBranch+"-arm64v8") + } else if ciBranch != "master" && publicRepo.MatchString(ciBranch) { + login(docker) + deployManifest(docker, "PR"+ciPullRequest, "PR"+ciPullRequest+"-amd64", "PR"+ciPullRequest+"-arm32v7", "PR"+ciPullRequest+"-arm64v8") + } else if ciBranch == "master" && ciPullRequest == "false" { login(docker) deployManifest(docker, "master", "master-amd64", "master-arm32v7", "master-arm64v8") publishDockerReadme(docker)