build: reproducible

Signed-off-by: James Elliott <james-d-elliott@users.noreply.github.com>

cmd/authelia-scripts/cmd/build.go

@@ -1,7 +1,10 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -24,154 +27,185 @@ func newBuildCmd() (cmd *cobra.Command) {
 		DisableAutoGenTag: true,
 	}
 
+	cmd.Flags().Bool("print", false, "Prints the command instead of running it, useful for reproducible builds")
+	cmd.Flags().Int("build-number", 0, "Forcefully sets the build number, useful for reproducible builds")
+
 	return cmd
 }
 
-func cmdBuildRun(cobraCmd *cobra.Command, args []string) {
+func cmdBuildRun(cmd *cobra.Command, args []string) {
 	branch := os.Getenv("BUILDKITE_BRANCH")
 
+	var (
+		buildPrint bool
+		err        error
+	)
+
+	if buildPrint, err = cmd.Flags().GetBool("print"); err != nil {
+		log.Fatal(err)
+	}
+
 	if strings.HasPrefix(branch, "renovate/") {
-		buildFrontend(branch)
+		buildFrontend(false, branch)
 		log.Info("Skip building Authelia for deps...")
 		os.Exit(0)
 	}
 
-	log.Info("Building Authelia...")
+	switch {
+	case buildPrint:
+		log.Info("Printing Build Authelia Commands...")
+	default:
+		log.Info("Building Authelia...")
 
-	cmdCleanRun(cobraCmd, args)
+		cmdCleanRun(cmd, args)
+
+		log.Debug("Creating `" + OutputDir + "` directory")
+
+		if err = os.MkdirAll(OutputDir, os.ModePerm); err != nil {
+			log.Fatal(err)
+		}
+	}
 
 	buildMetaData, err := getBuild(branch, os.Getenv("BUILDKITE_BUILD_NUMBER"), "")
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	log.Debug("Creating `" + OutputDir + "` directory")
+	if cmd.Flags().Changed("build-number") {
-
+		buildMetaData.Number, _ = cmd.Flags().GetInt("build-number")
-	if err = os.MkdirAll(OutputDir, os.ModePerm); err != nil {
-		log.Fatal(err)
 	}
 
 	log.Debug("Building Authelia frontend...")
-	buildFrontend(branch)
+	buildFrontend(buildPrint, branch)
 
 	log.Debug("Building swagger-ui frontend...")
-	buildSwagger()
+	buildSwagger(buildPrint)
 
-	buildkite, _ := cobraCmd.Flags().GetBool("buildkite")
+	buildkite, _ := cmd.Flags().GetBool("buildkite")
 
 	if buildkite {
-		log.Info("Building Authelia Go binaries with gox...")
+		buildAutheliaBinaryGOX(buildPrint, buildMetaData)
-
-		buildAutheliaBinaryGOX(buildMetaData.XFlags())
 	} else {
-		log.Info("Building Authelia Go binary...")
+		buildAutheliaBinaryGO(buildPrint, buildMetaData)
-
-		buildAutheliaBinaryGO(buildMetaData.XFlags())
 	}
 
-	cleanAssets()
+	if !buildPrint {
+		cleanAssets()
+	}
 }
 
-func buildAutheliaBinaryGOX(xflags []string) {
+func buildAutheliaBinaryGOX(buildPrint bool, buildMetaData *Build) {
 	var wg sync.WaitGroup
 
-	s := time.Now()
+	started := time.Now()
 
-	wg.Add(2)
+	xflags := buildMetaData.XFlags()
+
+	cmds := make([]*exec.Cmd, 2)
+
+	cmds[0] = utils.CommandWithStdout("gox", "-output={{.Dir}}-{{.OS}}-{{.Arch}}-musl", "-buildmode=pie", "-trimpath", "-cgo", "-ldflags=-linkmode=external -s -w "+strings.Join(xflags, " "), "-osarch=linux/amd64 linux/arm linux/arm64", "./cmd/authelia/")
+
+	cmds[0].Env = append(cmds[0].Env,
+		"CGO_CPPFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong", "CGO_LDFLAGS=-Wl,-z,relro,-z,now",
+		"GOX_LINUX_ARM_CC=arm-linux-musleabihf-gcc", "GOX_LINUX_ARM64_CC=aarch64-linux-musl-gcc")
+
+	cmds[1] = utils.CommandWithStdout("bash", "-c", "docker run --rm -e GOX_LINUX_ARM_CC=arm-linux-gnueabihf-gcc -e GOX_LINUX_ARM64_CC=aarch64-linux-gnu-gcc -e GOX_FREEBSD_AMD64_CC=x86_64-pc-freebsd13-gcc -v ${PWD}:/workdir -v /buildkite/.go:/root/go authelia/crossbuild "+
+		"gox -output={{.Dir}}-{{.OS}}-{{.Arch}} -buildmode=pie -trimpath -cgo -ldflags=\"-linkmode=external -s -w "+strings.Join(xflags, " ")+"\" -osarch=\"linux/amd64 linux/arm linux/arm64 freebsd/amd64\" ./cmd/authelia/")
+
+	if buildPrint {
+		for _, cmd := range cmds {
+			buildCmdPrint(cmd)
+		}
+
+		return
+	}
+
+	log.Info("Building Authelia Go binaries with gox...")
+
+	wg.Add(len(cmds))
 
 	go func() {
 		defer wg.Done()
 
-		cmd := utils.CommandWithStdout("gox", "-output={{.Dir}}-{{.OS}}-{{.Arch}}-musl", "-buildmode=pie", "-trimpath", "-cgo", "-ldflags=-linkmode=external -s -w "+strings.Join(xflags, " "), "-osarch=linux/amd64 linux/arm linux/arm64", "./cmd/authelia/")
+		buildCmdRun(cmds[0])
-
-		cmd.Env = append(os.Environ(),
-			"CGO_CPPFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong", "CGO_LDFLAGS=-Wl,-z,relro,-z,now",
-			"GOX_LINUX_ARM_CC=arm-linux-musleabihf-gcc", "GOX_LINUX_ARM64_CC=aarch64-linux-musl-gcc")
-
-		err := cmd.Run()
-		if err != nil {
-			log.Fatal(err)
-		}
 	}()
 
 	go func() {
 		defer wg.Done()
 
-		cmd := utils.CommandWithStdout("bash", "-c", "docker run --rm -e GOX_LINUX_ARM_CC=arm-linux-gnueabihf-gcc -e GOX_LINUX_ARM64_CC=aarch64-linux-gnu-gcc -e GOX_FREEBSD_AMD64_CC=x86_64-pc-freebsd13-gcc -v ${PWD}:/workdir -v /buildkite/.go:/root/go authelia/crossbuild "+
+		buildCmdRun(cmds[1])
-			"gox -output={{.Dir}}-{{.OS}}-{{.Arch}} -buildmode=pie -trimpath -cgo -ldflags=\"-linkmode=external -s -w "+strings.Join(xflags, " ")+"\" -osarch=\"linux/amd64 linux/arm linux/arm64 freebsd/amd64\" ./cmd/authelia/")
-
-		err := cmd.Run()
-		if err != nil {
-			log.Fatal(err)
-		}
 	}()
 
 	wg.Wait()
 
-	e := time.Since(s)
+	log.Debugf("Binary compilation completed in %s.", time.Since(started))
-
-	log.Debugf("Binary compilation completed in %s.", e)
 }
 
-func buildAutheliaBinaryGO(xflags []string) {
+func buildAutheliaBinaryGO(buildPrint bool, buildMetaData *Build) {
-	cmd := utils.CommandWithStdout("go", "build", "-buildmode=pie", "-trimpath", "-o", OutputDir+"/authelia", "-ldflags", "-linkmode=external -s -w "+strings.Join(xflags, " "), "./cmd/authelia/")
+	cmd := utils.CommandWithStdout("go", "build", "-buildmode=pie", "-trimpath", "-o", OutputDir+"/authelia", "-ldflags", "-linkmode=external -s -w "+strings.Join(buildMetaData.XFlags(), " "), "./cmd/authelia/")
 
-	cmd.Env = append(os.Environ(),
+	cmd.Env = append(cmd.Env,
 		"CGO_CPPFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong", "CGO_LDFLAGS=-Wl,-z,relro,-z,now")
 
-	err := cmd.Run()
+	if buildPrint {
-	if err != nil {
+		buildCmdPrint(cmd)
-		log.Fatal(err)
+
+		return
 	}
+
+	log.Info("Building Authelia Go binary...")
+
+	buildCmdRun(cmd)
 }
 
-func buildFrontend(branch string) {
+func buildFrontend(buildPrint bool, branch string) {
-	cmd := utils.CommandWithStdout("pnpm", "install")
+	var (
-	cmd.Dir = webDirectory
+		cmds []*exec.Cmd
+		cmd  *exec.Cmd
+	)
 
-	err := cmd.Run()
+	cmd = utils.CommandWithStdout("pnpm", "install")
-	if err != nil {
+	cmd.Dir = filepath.Join(cmd.Dir, webDirectory)
-		log.Fatal(err)
+
-	}
+	cmds = append(cmds, cmd)
 
 	if !strings.HasPrefix(branch, "renovate/") {
 		cmd = utils.CommandWithStdout("pnpm", "build")
-		cmd.Dir = webDirectory
+		cmd.Dir = filepath.Join(cmd.Dir, webDirectory)
 
-		err = cmd.Run()
+		cmds = append(cmds, cmd)
-		if err != nil {
+	}
-			log.Fatal(err)
+
+	for _, cmd = range cmds {
+		if buildPrint {
+			buildCmdPrint(cmd)
+
+			continue
 		}
+
+		buildCmdRun(cmd)
 	}
 }
 
-func buildSwagger() {
+func buildSwagger(buildPrint bool) {
-	cmd := utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+versionSwaggerUI+".tar.gz -O ./v"+versionSwaggerUI+".tar.gz")
+	var (
+		cmds []*exec.Cmd
+		cmd  *exec.Cmd
+	)
 
-	err := cmd.Run()
+	cmds = append(cmds, utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+versionSwaggerUI+".tar.gz -O ./v"+versionSwaggerUI+".tar.gz"))
-	if err != nil {
+	cmds = append(cmds, utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html"))
-		log.Fatal(err)
+	cmds = append(cmds, utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+versionSwaggerUI+".tar.gz", "swagger-ui-"+versionSwaggerUI+"/dist"))
-	}
+	cmds = append(cmds, utils.CommandWithStdout("rm", "./v"+versionSwaggerUI+".tar.gz"))
 
-	cmd = utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html")
+	for _, cmd = range cmds {
+		if buildPrint {
+			buildCmdPrint(cmd)
 
-	err = cmd.Run()
+			continue
-	if err != nil {
+		}
-		log.Fatal(err)
-	}
 
-	cmd = utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+versionSwaggerUI+".tar.gz", "swagger-ui-"+versionSwaggerUI+"/dist")
+		buildCmdRun(cmd)
-
-	err = cmd.Run()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	cmd = utils.CommandWithStdout("rm", "./v"+versionSwaggerUI+".tar.gz")
-
-	err = cmd.Run()
-	if err != nil {
-		log.Fatal(err)
 	}
 }
 
@@ -182,13 +216,36 @@ func cleanAssets() {
 
 	cmd := utils.CommandWithStdout("mkdir", "-p", "internal/server/public_html/api")
 
-	if err := cmd.Run(); err != nil {
+	buildCmdRun(cmd)
-		log.Fatal(err)
-	}
 
 	cmd = utils.CommandWithStdout("bash", "-c", "touch internal/server/public_html/{index.html,api/index.html,api/openapi.yml}")
 
+	buildCmdRun(cmd)
+}
+
+func buildCmdRun(cmd *exec.Cmd) {
+	if len(cmd.Env) != 0 {
+		cmd.Env = append(os.Environ(), cmd.Env...)
+	}
+
 	if err := cmd.Run(); err != nil {
 		log.Fatal(err)
 	}
 }
+
+func buildCmdPrint(cmd *exec.Cmd) {
+	b := &strings.Builder{}
+
+	if cmd.Dir != "" {
+		b.WriteString(fmt.Sprintf("cd %s\n", cmd.Dir))
+	}
+
+	if len(cmd.Env) != 0 {
+		b.WriteString(strings.Join(cmd.Env, " "))
+		b.WriteString(" ")
+	}
+
+	b.WriteString(cmd.String())
+
+	fmt.Println(b.String())
+}

cmd/authelia-scripts/cmd/helpers.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/authelia/authelia/v4/internal/utils"
@@ -50,6 +51,21 @@ func getBuild(branch, buildNumber, extra string) (b *Build, err error) {
 		return nil, fmt.Errorf("error getting commit with git rev-parse: %w", err)
 	}
 
+	var (
+		gitCommitTS  string
+		gitCommitTSI int
+	)
+
+	if gitCommitTS, _, err = utils.RunCommandAndReturnOutput(fmt.Sprintf("git show -s --format=%%ct %s", b.Commit)); err != nil {
+		return nil, fmt.Errorf("error getting commit date with git show -s --format=%%ct %s: %w", b.Commit, err)
+	}
+
+	if gitCommitTSI, err = strconv.Atoi(strings.TrimSpace(gitCommitTS)); err != nil {
+		return nil, fmt.Errorf("error getting commit date with git show -s --format=%%ct %s: %w", b.Commit, err)
+	}
+
+	b.Date = time.Unix(int64(gitCommitTSI), 0).UTC()
+
 	if gitTagCommit == b.Commit {
 		b.Tagged = true
 	}
@@ -58,7 +74,5 @@ func getBuild(branch, buildNumber, extra string) (b *Build, err error) {
 		b.Clean = true
 	}
 
-	b.Date = time.Now()
-
 	return b, nil
 }

cmd/authelia-scripts/cmd/types.go

@@ -85,7 +85,7 @@ func (b Build) XFlags() []string {
 		fmt.Sprintf(fmtLDFLAGSX, "BuildBranch", b.Branch),
 		fmt.Sprintf(fmtLDFLAGSX, "BuildTag", b.Tag),
 		fmt.Sprintf(fmtLDFLAGSX, "BuildCommit", b.Commit),
-		fmt.Sprintf(fmtLDFLAGSX, "BuildDate", b.Date.Format("Mon, 02 Jan 2006 15:04:05 -0700")),
+		fmt.Sprintf(fmtLDFLAGSX, "BuildDate", b.Date.Format(time.RFC1123)),
 		fmt.Sprintf(fmtLDFLAGSX, "BuildState", b.State()),
 		fmt.Sprintf(fmtLDFLAGSX, "BuildExtra", b.Extra),
 		fmt.Sprintf(fmtLDFLAGSX, "BuildNumber", strconv.Itoa(b.Number)),
@@ -112,7 +112,7 @@ func (b Build) ContainerLabels() (labels map[string]string) {
 	}
 
 	labels = map[string]string{
-		"org.opencontainers.image.created":       b.Date.Format(time.RFC3339),
+		"org.opencontainers.image.created":       b.Date.UTC().Format(time.RFC3339),
 		"org.opencontainers.image.authors":       "",
 		"org.opencontainers.image.url":           "https://github.com/authelia/authelia/pkgs/container/authelia",
 		"org.opencontainers.image.documentation": "https://www.authelia.com",

docs/content/en/contributing/development/build-and-test.md

@@ -102,6 +102,8 @@ If you want to manually build the binary from source you will require the open s
 [Development Environment](./environment.md#setup) documentation. Then you can follow the below steps on Linux (you may
 have to adapt them on other systems).
 
+#### Basic Steps
+
 Clone the Repository:
 
 ```bash
@@ -137,6 +139,48 @@ CGO_ENABLED=1 CGO_CPPFLAGS="-D_FORTIFY_SOURCE=2 -fstack-protector-strong" CGO_LD
 go build -ldflags "-linkmode=external -s -w" -trimpath -buildmode=pie -o authelia ./cmd/authelia
 ```
 
+#### Reproducible Builds
+
+Authelia allows production of reproducible builds that were built using our pipeline. The only variables injected into
+a build are from commit information other than the exceptions listed in this section. This means that we can provide the
+exact build commands for any given build with very limited input from users. The elements injected into the binary as
+part of the build process (using linker flags) are:
+
+- Commit SHA1
+- Commit Date (using the RFC1123 layout strictly using the UTC timezone)
+- Latest Tag
+- Tag State (i.e. if the HEAD commit has the latest tag)
+- Working Tree State (dirty, clean, etc)
+- Branch Name
+- Build Number
+
+The exceptions of this list which cannot be obtained from commit information (but can be supplied by an environment
+variable or CLI argument):
+
+- Build Number
+
+##### Instructions
+
+To perform a reproducible build users should follow these steps:
+
+1. Run the `authelia build-info` command which contains useful information for reproducing the build including:
+   1. The `Build Number` field.
+   2. The `Build Go Version` field.
+2. Install all of the required dependencies. It's recommended if you're looking for a reproducible build that you use
+   the same Go version from step 1.
+3. Run the following command from the root of the repository to output the build commands (where 100 is the number from
+   step 1):
+
+```bash
+go run ./cmd/authelia-scripts build --print --build-number 100
+```
+
+The output of the above command may be ran to perform all of the build steps manually.
+
+*__Important Note:__ If you wish to use [gox](https://gitihub.com/authelia/gox) to build Authelia please run the
+`go run ./cmd/authelia-scripts build --print --buildkite --build-number 100` command instead of the above command (i.e.
+adding the `--buildkite` flag).*
+
 [suites]: ./integration-suites.md
 [React]: https://reactjs.org/
 [go]: https://go.dev/dl/

internal/commands/build-info.go

@@ -27,7 +27,7 @@ func newBuildInfoCmd(ctx *CmdCtx) (cmd *cobra.Command) {
 // BuildInfoRunE is the RunE for the authelia build-info command.
 func (ctx *CmdCtx) BuildInfoRunE(_ *cobra.Command, _ []string) (err error) {
 	_, err = fmt.Printf(fmtAutheliaBuild, utils.BuildTag, utils.BuildState, utils.BuildBranch, utils.BuildCommit,
-		utils.BuildNumber, runtime.GOOS, runtime.GOARCH, utils.BuildDate, utils.BuildExtra)
+		utils.BuildDate, utils.BuildNumber, runtime.GOOS, runtime.GOARCH, runtime.Version(), utils.BuildExtra)
 
 	return err
 }

internal/commands/const.go

@@ -24,10 +24,11 @@ authelia --config /etc/authelia/config/`
 State: %s
 Branch: %s
 Commit: %s
+Commit Date: %s
 Build Number: %s
 Build OS: %s
 Build Arch: %s
-Build Date: %s
+Build Go Version: %s
 Extra: %s
 `
 

internal/suites/suite_cli_test.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/suite"
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/authelia/authelia/v4/internal/model"
 	"github.com/authelia/authelia/v4/internal/storage"
@@ -50,15 +50,8 @@ func (s *CLISuite) TestShouldPrintBuildInformation() {
 
 	output, err := s.Exec("authelia-backend", []string{"authelia", "build-info"})
 	s.Assert().NoError(err)
-	s.Assert().Contains(output, "Last Tag: ")
-	s.Assert().Contains(output, "State: ")
-	s.Assert().Contains(output, "Branch: ")
-	s.Assert().Contains(output, "Build Number: ")
-	s.Assert().Contains(output, "Build OS: ")
-	s.Assert().Contains(output, "Build Arch: ")
-	s.Assert().Contains(output, "Build Date: ")
 
-	r := regexp.MustCompile(`^Last Tag: v\d+\.\d+\.\d+\nState: (tagged|untagged) (clean|dirty)\nBranch: [^\s\n]+\nCommit: [0-9a-f]{40}\nBuild Number: \d+\nBuild OS: (linux|darwin|windows|freebsd)\nBuild Arch: (amd64|arm|arm64)\nBuild Date: (Sun|Mon|Tue|Wed|Thu|Fri|Sat), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} [+-]\d{4}\nExtra: \n`)
+	r := regexp.MustCompile(`^Last Tag: v\d+\.\d+\.\d+\nState: (tagged|untagged) (clean|dirty)\nBranch: [^\s\n]+\nCommit: [0-9a-f]{40}\nCommit Date: (Sun|Mon|Tue|Wed|Thu|Fri|Sat), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} \+0000\nBuild Number: \d+\nBuild OS: (linux|darwin|windows|freebsd)\nBuild Arch: (amd64|arm|arm64)\nBuild Go Version: go\d+\.\d+(\.\d+)?\nExtra:`)
 	s.Assert().Regexp(r, output)
 }