From 5c84a17e194c37fb035b97ed201f39ce21b575ac Mon Sep 17 00:00:00 2001 From: James Elliott Date: Thu, 13 Apr 2023 22:44:10 +1000 Subject: [PATCH] build: reproducible Signed-off-by: James Elliott --- cmd/authelia-scripts/cmd/build.go | 225 +++++++++++------- cmd/authelia-scripts/cmd/helpers.go | 18 +- cmd/authelia-scripts/cmd/types.go | 4 +- .../development/build-and-test.md | 44 ++++ internal/commands/build-info.go | 2 +- internal/commands/const.go | 3 +- internal/suites/suite_cli_test.go | 11 +- 7 files changed, 208 insertions(+), 99 deletions(-) diff --git a/cmd/authelia-scripts/cmd/build.go b/cmd/authelia-scripts/cmd/build.go index fa72b83f6..bddf4801b 100644 --- a/cmd/authelia-scripts/cmd/build.go +++ b/cmd/authelia-scripts/cmd/build.go @@ -1,7 +1,10 @@ package cmd import ( + "fmt" "os" + "os/exec" + "path/filepath" "strings" "sync" "time" @@ -24,154 +27,185 @@ func newBuildCmd() (cmd *cobra.Command) { DisableAutoGenTag: true, } + cmd.Flags().Bool("print", false, "Prints the command instead of running it, useful for reproducible builds") + cmd.Flags().Int("build-number", 0, "Forcefully sets the build number, useful for reproducible builds") + return cmd } -func cmdBuildRun(cobraCmd *cobra.Command, args []string) { +func cmdBuildRun(cmd *cobra.Command, args []string) { branch := os.Getenv("BUILDKITE_BRANCH") + var ( + buildPrint bool + err error + ) + + if buildPrint, err = cmd.Flags().GetBool("print"); err != nil { + log.Fatal(err) + } + if strings.HasPrefix(branch, "renovate/") { - buildFrontend(branch) + buildFrontend(false, branch) log.Info("Skip building Authelia for deps...") os.Exit(0) } - log.Info("Building Authelia...") + switch { + case buildPrint: + log.Info("Printing Build Authelia Commands...") + default: + log.Info("Building Authelia...") - cmdCleanRun(cobraCmd, args) + cmdCleanRun(cmd, args) + + log.Debug("Creating `" + OutputDir + "` directory") + + if err = os.MkdirAll(OutputDir, os.ModePerm); err != nil { + log.Fatal(err) + } + } buildMetaData, err := getBuild(branch, os.Getenv("BUILDKITE_BUILD_NUMBER"), "") if err != nil { log.Fatal(err) } - log.Debug("Creating `" + OutputDir + "` directory") - - if err = os.MkdirAll(OutputDir, os.ModePerm); err != nil { - log.Fatal(err) + if cmd.Flags().Changed("build-number") { + buildMetaData.Number, _ = cmd.Flags().GetInt("build-number") } log.Debug("Building Authelia frontend...") - buildFrontend(branch) + buildFrontend(buildPrint, branch) log.Debug("Building swagger-ui frontend...") - buildSwagger() + buildSwagger(buildPrint) - buildkite, _ := cobraCmd.Flags().GetBool("buildkite") + buildkite, _ := cmd.Flags().GetBool("buildkite") if buildkite { - log.Info("Building Authelia Go binaries with gox...") - - buildAutheliaBinaryGOX(buildMetaData.XFlags()) + buildAutheliaBinaryGOX(buildPrint, buildMetaData) } else { - log.Info("Building Authelia Go binary...") - - buildAutheliaBinaryGO(buildMetaData.XFlags()) + buildAutheliaBinaryGO(buildPrint, buildMetaData) } - cleanAssets() + if !buildPrint { + cleanAssets() + } } -func buildAutheliaBinaryGOX(xflags []string) { +func buildAutheliaBinaryGOX(buildPrint bool, buildMetaData *Build) { var wg sync.WaitGroup - s := time.Now() + started := time.Now() - wg.Add(2) + xflags := buildMetaData.XFlags() + + cmds := make([]*exec.Cmd, 2) + + cmds[0] = utils.CommandWithStdout("gox", "-output={{.Dir}}-{{.OS}}-{{.Arch}}-musl", "-buildmode=pie", "-trimpath", "-cgo", "-ldflags=-linkmode=external -s -w "+strings.Join(xflags, " "), "-osarch=linux/amd64 linux/arm linux/arm64", "./cmd/authelia/") + + cmds[0].Env = append(cmds[0].Env, + "CGO_CPPFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong", "CGO_LDFLAGS=-Wl,-z,relro,-z,now", + "GOX_LINUX_ARM_CC=arm-linux-musleabihf-gcc", "GOX_LINUX_ARM64_CC=aarch64-linux-musl-gcc") + + cmds[1] = utils.CommandWithStdout("bash", "-c", "docker run --rm -e GOX_LINUX_ARM_CC=arm-linux-gnueabihf-gcc -e GOX_LINUX_ARM64_CC=aarch64-linux-gnu-gcc -e GOX_FREEBSD_AMD64_CC=x86_64-pc-freebsd13-gcc -v ${PWD}:/workdir -v /buildkite/.go:/root/go authelia/crossbuild "+ + "gox -output={{.Dir}}-{{.OS}}-{{.Arch}} -buildmode=pie -trimpath -cgo -ldflags=\"-linkmode=external -s -w "+strings.Join(xflags, " ")+"\" -osarch=\"linux/amd64 linux/arm linux/arm64 freebsd/amd64\" ./cmd/authelia/") + + if buildPrint { + for _, cmd := range cmds { + buildCmdPrint(cmd) + } + + return + } + + log.Info("Building Authelia Go binaries with gox...") + + wg.Add(len(cmds)) go func() { defer wg.Done() - cmd := utils.CommandWithStdout("gox", "-output={{.Dir}}-{{.OS}}-{{.Arch}}-musl", "-buildmode=pie", "-trimpath", "-cgo", "-ldflags=-linkmode=external -s -w "+strings.Join(xflags, " "), "-osarch=linux/amd64 linux/arm linux/arm64", "./cmd/authelia/") - - cmd.Env = append(os.Environ(), - "CGO_CPPFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong", "CGO_LDFLAGS=-Wl,-z,relro,-z,now", - "GOX_LINUX_ARM_CC=arm-linux-musleabihf-gcc", "GOX_LINUX_ARM64_CC=aarch64-linux-musl-gcc") - - err := cmd.Run() - if err != nil { - log.Fatal(err) - } + buildCmdRun(cmds[0]) }() go func() { defer wg.Done() - cmd := utils.CommandWithStdout("bash", "-c", "docker run --rm -e GOX_LINUX_ARM_CC=arm-linux-gnueabihf-gcc -e GOX_LINUX_ARM64_CC=aarch64-linux-gnu-gcc -e GOX_FREEBSD_AMD64_CC=x86_64-pc-freebsd13-gcc -v ${PWD}:/workdir -v /buildkite/.go:/root/go authelia/crossbuild "+ - "gox -output={{.Dir}}-{{.OS}}-{{.Arch}} -buildmode=pie -trimpath -cgo -ldflags=\"-linkmode=external -s -w "+strings.Join(xflags, " ")+"\" -osarch=\"linux/amd64 linux/arm linux/arm64 freebsd/amd64\" ./cmd/authelia/") - - err := cmd.Run() - if err != nil { - log.Fatal(err) - } + buildCmdRun(cmds[1]) }() wg.Wait() - e := time.Since(s) - - log.Debugf("Binary compilation completed in %s.", e) + log.Debugf("Binary compilation completed in %s.", time.Since(started)) } -func buildAutheliaBinaryGO(xflags []string) { - cmd := utils.CommandWithStdout("go", "build", "-buildmode=pie", "-trimpath", "-o", OutputDir+"/authelia", "-ldflags", "-linkmode=external -s -w "+strings.Join(xflags, " "), "./cmd/authelia/") +func buildAutheliaBinaryGO(buildPrint bool, buildMetaData *Build) { + cmd := utils.CommandWithStdout("go", "build", "-buildmode=pie", "-trimpath", "-o", OutputDir+"/authelia", "-ldflags", "-linkmode=external -s -w "+strings.Join(buildMetaData.XFlags(), " "), "./cmd/authelia/") - cmd.Env = append(os.Environ(), + cmd.Env = append(cmd.Env, "CGO_CPPFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong", "CGO_LDFLAGS=-Wl,-z,relro,-z,now") - err := cmd.Run() - if err != nil { - log.Fatal(err) + if buildPrint { + buildCmdPrint(cmd) + + return } + + log.Info("Building Authelia Go binary...") + + buildCmdRun(cmd) } -func buildFrontend(branch string) { - cmd := utils.CommandWithStdout("pnpm", "install") - cmd.Dir = webDirectory +func buildFrontend(buildPrint bool, branch string) { + var ( + cmds []*exec.Cmd + cmd *exec.Cmd + ) - err := cmd.Run() - if err != nil { - log.Fatal(err) - } + cmd = utils.CommandWithStdout("pnpm", "install") + cmd.Dir = filepath.Join(cmd.Dir, webDirectory) + + cmds = append(cmds, cmd) if !strings.HasPrefix(branch, "renovate/") { cmd = utils.CommandWithStdout("pnpm", "build") - cmd.Dir = webDirectory + cmd.Dir = filepath.Join(cmd.Dir, webDirectory) - err = cmd.Run() - if err != nil { - log.Fatal(err) + cmds = append(cmds, cmd) + } + + for _, cmd = range cmds { + if buildPrint { + buildCmdPrint(cmd) + + continue } + + buildCmdRun(cmd) } } -func buildSwagger() { - cmd := utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+versionSwaggerUI+".tar.gz -O ./v"+versionSwaggerUI+".tar.gz") +func buildSwagger(buildPrint bool) { + var ( + cmds []*exec.Cmd + cmd *exec.Cmd + ) - err := cmd.Run() - if err != nil { - log.Fatal(err) - } + cmds = append(cmds, utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+versionSwaggerUI+".tar.gz -O ./v"+versionSwaggerUI+".tar.gz")) + cmds = append(cmds, utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html")) + cmds = append(cmds, utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+versionSwaggerUI+".tar.gz", "swagger-ui-"+versionSwaggerUI+"/dist")) + cmds = append(cmds, utils.CommandWithStdout("rm", "./v"+versionSwaggerUI+".tar.gz")) - cmd = utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html") + for _, cmd = range cmds { + if buildPrint { + buildCmdPrint(cmd) - err = cmd.Run() - if err != nil { - log.Fatal(err) - } + continue + } - cmd = utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+versionSwaggerUI+".tar.gz", "swagger-ui-"+versionSwaggerUI+"/dist") - - err = cmd.Run() - if err != nil { - log.Fatal(err) - } - - cmd = utils.CommandWithStdout("rm", "./v"+versionSwaggerUI+".tar.gz") - - err = cmd.Run() - if err != nil { - log.Fatal(err) + buildCmdRun(cmd) } } @@ -182,13 +216,36 @@ func cleanAssets() { cmd := utils.CommandWithStdout("mkdir", "-p", "internal/server/public_html/api") - if err := cmd.Run(); err != nil { - log.Fatal(err) - } + buildCmdRun(cmd) cmd = utils.CommandWithStdout("bash", "-c", "touch internal/server/public_html/{index.html,api/index.html,api/openapi.yml}") + buildCmdRun(cmd) +} + +func buildCmdRun(cmd *exec.Cmd) { + if len(cmd.Env) != 0 { + cmd.Env = append(os.Environ(), cmd.Env...) + } + if err := cmd.Run(); err != nil { log.Fatal(err) } } + +func buildCmdPrint(cmd *exec.Cmd) { + b := &strings.Builder{} + + if cmd.Dir != "" { + b.WriteString(fmt.Sprintf("cd %s\n", cmd.Dir)) + } + + if len(cmd.Env) != 0 { + b.WriteString(strings.Join(cmd.Env, " ")) + b.WriteString(" ") + } + + b.WriteString(cmd.String()) + + fmt.Println(b.String()) +} diff --git a/cmd/authelia-scripts/cmd/helpers.go b/cmd/authelia-scripts/cmd/helpers.go index 12f7e7dc8..6afb97f20 100644 --- a/cmd/authelia-scripts/cmd/helpers.go +++ b/cmd/authelia-scripts/cmd/helpers.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "strconv" + "strings" "time" "github.com/authelia/authelia/v4/internal/utils" @@ -50,6 +51,21 @@ func getBuild(branch, buildNumber, extra string) (b *Build, err error) { return nil, fmt.Errorf("error getting commit with git rev-parse: %w", err) } + var ( + gitCommitTS string + gitCommitTSI int + ) + + if gitCommitTS, _, err = utils.RunCommandAndReturnOutput(fmt.Sprintf("git show -s --format=%%ct %s", b.Commit)); err != nil { + return nil, fmt.Errorf("error getting commit date with git show -s --format=%%ct %s: %w", b.Commit, err) + } + + if gitCommitTSI, err = strconv.Atoi(strings.TrimSpace(gitCommitTS)); err != nil { + return nil, fmt.Errorf("error getting commit date with git show -s --format=%%ct %s: %w", b.Commit, err) + } + + b.Date = time.Unix(int64(gitCommitTSI), 0).UTC() + if gitTagCommit == b.Commit { b.Tagged = true } @@ -58,7 +74,5 @@ func getBuild(branch, buildNumber, extra string) (b *Build, err error) { b.Clean = true } - b.Date = time.Now() - return b, nil } diff --git a/cmd/authelia-scripts/cmd/types.go b/cmd/authelia-scripts/cmd/types.go index e18f95391..8191db932 100644 --- a/cmd/authelia-scripts/cmd/types.go +++ b/cmd/authelia-scripts/cmd/types.go @@ -85,7 +85,7 @@ func (b Build) XFlags() []string { fmt.Sprintf(fmtLDFLAGSX, "BuildBranch", b.Branch), fmt.Sprintf(fmtLDFLAGSX, "BuildTag", b.Tag), fmt.Sprintf(fmtLDFLAGSX, "BuildCommit", b.Commit), - fmt.Sprintf(fmtLDFLAGSX, "BuildDate", b.Date.Format("Mon, 02 Jan 2006 15:04:05 -0700")), + fmt.Sprintf(fmtLDFLAGSX, "BuildDate", b.Date.Format(time.RFC1123)), fmt.Sprintf(fmtLDFLAGSX, "BuildState", b.State()), fmt.Sprintf(fmtLDFLAGSX, "BuildExtra", b.Extra), fmt.Sprintf(fmtLDFLAGSX, "BuildNumber", strconv.Itoa(b.Number)), @@ -112,7 +112,7 @@ func (b Build) ContainerLabels() (labels map[string]string) { } labels = map[string]string{ - "org.opencontainers.image.created": b.Date.Format(time.RFC3339), + "org.opencontainers.image.created": b.Date.UTC().Format(time.RFC3339), "org.opencontainers.image.authors": "", "org.opencontainers.image.url": "https://github.com/authelia/authelia/pkgs/container/authelia", "org.opencontainers.image.documentation": "https://www.authelia.com", diff --git a/docs/content/en/contributing/development/build-and-test.md b/docs/content/en/contributing/development/build-and-test.md index ec153ea46..7aefe7b1d 100644 --- a/docs/content/en/contributing/development/build-and-test.md +++ b/docs/content/en/contributing/development/build-and-test.md @@ -102,6 +102,8 @@ If you want to manually build the binary from source you will require the open s [Development Environment](./environment.md#setup) documentation. Then you can follow the below steps on Linux (you may have to adapt them on other systems). +#### Basic Steps + Clone the Repository: ```bash @@ -137,6 +139,48 @@ CGO_ENABLED=1 CGO_CPPFLAGS="-D_FORTIFY_SOURCE=2 -fstack-protector-strong" CGO_LD go build -ldflags "-linkmode=external -s -w" -trimpath -buildmode=pie -o authelia ./cmd/authelia ``` +#### Reproducible Builds + +Authelia allows production of reproducible builds that were built using our pipeline. The only variables injected into +a build are from commit information other than the exceptions listed in this section. This means that we can provide the +exact build commands for any given build with very limited input from users. The elements injected into the binary as +part of the build process (using linker flags) are: + +- Commit SHA1 +- Commit Date (using the RFC1123 layout strictly using the UTC timezone) +- Latest Tag +- Tag State (i.e. if the HEAD commit has the latest tag) +- Working Tree State (dirty, clean, etc) +- Branch Name +- Build Number + +The exceptions of this list which cannot be obtained from commit information (but can be supplied by an environment +variable or CLI argument): + +- Build Number + +##### Instructions + +To perform a reproducible build users should follow these steps: + +1. Run the `authelia build-info` command which contains useful information for reproducing the build including: + 1. The `Build Number` field. + 2. The `Build Go Version` field. +2. Install all of the required dependencies. It's recommended if you're looking for a reproducible build that you use + the same Go version from step 1. +3. Run the following command from the root of the repository to output the build commands (where 100 is the number from + step 1): + +```bash +go run ./cmd/authelia-scripts build --print --build-number 100 +``` + +The output of the above command may be ran to perform all of the build steps manually. + +*__Important Note:__ If you wish to use [gox](https://gitihub.com/authelia/gox) to build Authelia please run the +`go run ./cmd/authelia-scripts build --print --buildkite --build-number 100` command instead of the above command (i.e. +adding the `--buildkite` flag).* + [suites]: ./integration-suites.md [React]: https://reactjs.org/ [go]: https://go.dev/dl/ diff --git a/internal/commands/build-info.go b/internal/commands/build-info.go index fd321c8e7..a38c4f34e 100644 --- a/internal/commands/build-info.go +++ b/internal/commands/build-info.go @@ -27,7 +27,7 @@ func newBuildInfoCmd(ctx *CmdCtx) (cmd *cobra.Command) { // BuildInfoRunE is the RunE for the authelia build-info command. func (ctx *CmdCtx) BuildInfoRunE(_ *cobra.Command, _ []string) (err error) { _, err = fmt.Printf(fmtAutheliaBuild, utils.BuildTag, utils.BuildState, utils.BuildBranch, utils.BuildCommit, - utils.BuildNumber, runtime.GOOS, runtime.GOARCH, utils.BuildDate, utils.BuildExtra) + utils.BuildDate, utils.BuildNumber, runtime.GOOS, runtime.GOARCH, runtime.Version(), utils.BuildExtra) return err } diff --git a/internal/commands/const.go b/internal/commands/const.go index b26c1e94b..fea997dc6 100644 --- a/internal/commands/const.go +++ b/internal/commands/const.go @@ -24,10 +24,11 @@ authelia --config /etc/authelia/config/` State: %s Branch: %s Commit: %s +Commit Date: %s Build Number: %s Build OS: %s Build Arch: %s -Build Date: %s +Build Go Version: %s Extra: %s ` diff --git a/internal/suites/suite_cli_test.go b/internal/suites/suite_cli_test.go index a77507efe..1efcb88af 100644 --- a/internal/suites/suite_cli_test.go +++ b/internal/suites/suite_cli_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/stretchr/testify/suite" - "gopkg.in/yaml.v3" + yaml "gopkg.in/yaml.v3" "github.com/authelia/authelia/v4/internal/model" "github.com/authelia/authelia/v4/internal/storage" @@ -50,15 +50,8 @@ func (s *CLISuite) TestShouldPrintBuildInformation() { output, err := s.Exec("authelia-backend", []string{"authelia", "build-info"}) s.Assert().NoError(err) - s.Assert().Contains(output, "Last Tag: ") - s.Assert().Contains(output, "State: ") - s.Assert().Contains(output, "Branch: ") - s.Assert().Contains(output, "Build Number: ") - s.Assert().Contains(output, "Build OS: ") - s.Assert().Contains(output, "Build Arch: ") - s.Assert().Contains(output, "Build Date: ") - r := regexp.MustCompile(`^Last Tag: v\d+\.\d+\.\d+\nState: (tagged|untagged) (clean|dirty)\nBranch: [^\s\n]+\nCommit: [0-9a-f]{40}\nBuild Number: \d+\nBuild OS: (linux|darwin|windows|freebsd)\nBuild Arch: (amd64|arm|arm64)\nBuild Date: (Sun|Mon|Tue|Wed|Thu|Fri|Sat), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} [+-]\d{4}\nExtra: \n`) + r := regexp.MustCompile(`^Last Tag: v\d+\.\d+\.\d+\nState: (tagged|untagged) (clean|dirty)\nBranch: [^\s\n]+\nCommit: [0-9a-f]{40}\nCommit Date: (Sun|Mon|Tue|Wed|Thu|Fri|Sat), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} \+0000\nBuild Number: \d+\nBuild OS: (linux|darwin|windows|freebsd)\nBuild Arch: (amd64|arm|arm64)\nBuild Go Version: go\d+\.\d+(\.\d+)?\nExtra:`) s.Assert().Regexp(r, output) }