RPJosh
/
authelia
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: reproducible 

Signed-off-by: James Elliott <james-d-elliott@users.noreply.github.com>
pull/5232/head
James Elliott 2023-04-13 22:44:10 +10:00
parent 54bee80baf
commit 5c84a17e19
No known key found for this signature in database
GPG Key ID: 0F1C4A096E857E49
7 changed files with 208 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
cmd/authelia-scripts/cmd/build.go
View File

@ -1,7 +1,10 @@
package cmd
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
@ -24,154 +27,185 @@ func newBuildCmd() (cmd *cobra.Command) {
DisableAutoGenTag: true,
}
cmd.Flags().Bool("print", false, "Prints the command instead of running it, useful for reproducible builds")
cmd.Flags().Int("build-number", 0, "Forcefully sets the build number, useful for reproducible builds")
return cmd
}
func cmdBuildRun(cobraCmd *cobra.Command, args []string) {
func cmdBuildRun(cmd *cobra.Command, args []string) {
branch := os.Getenv("BUILDKITE_BRANCH")
var (
buildPrint bool
err error
)
if buildPrint, err = cmd.Flags().GetBool("print"); err != nil {
log.Fatal(err)
}
if strings.HasPrefix(branch, "renovate/") {
buildFrontend(branch)
buildFrontend(false, branch)
log.Info("Skip building Authelia for deps...")
os.Exit(0)
}
switch {
case buildPrint:
log.Info("Printing Build Authelia Commands...")
default:
log.Info("Building Authelia...")
cmdCleanRun(cobraCmd, args)
buildMetaData, err := getBuild(branch, os.Getenv("BUILDKITE_BUILD_NUMBER"), "")
if err != nil {
log.Fatal(err)
}
cmdCleanRun(cmd, args)
log.Debug("Creating `" + OutputDir + "` directory")
if err = os.MkdirAll(OutputDir, os.ModePerm); err != nil {
log.Fatal(err)
}
log.Debug("Building Authelia frontend...")
buildFrontend(branch)
log.Debug("Building swagger-ui frontend...")
buildSwagger()
buildkite, _ := cobraCmd.Flags().GetBool("buildkite")
if buildkite {
log.Info("Building Authelia Go binaries with gox...")
buildAutheliaBinaryGOX(buildMetaData.XFlags())
} else {
log.Info("Building Authelia Go binary...")
buildAutheliaBinaryGO(buildMetaData.XFlags())
}
buildMetaData, err := getBuild(branch, os.Getenv("BUILDKITE_BUILD_NUMBER"), "")
if err != nil {
log.Fatal(err)
}
if cmd.Flags().Changed("build-number") {
buildMetaData.Number, _ = cmd.Flags().GetInt("build-number")
}
log.Debug("Building Authelia frontend...")
buildFrontend(buildPrint, branch)
log.Debug("Building swagger-ui frontend...")
buildSwagger(buildPrint)
buildkite, _ := cmd.Flags().GetBool("buildkite")
if buildkite {
buildAutheliaBinaryGOX(buildPrint, buildMetaData)
} else {
buildAutheliaBinaryGO(buildPrint, buildMetaData)
}
if !buildPrint {
cleanAssets()
}
}
func buildAutheliaBinaryGOX(xflags []string) {
func buildAutheliaBinaryGOX(buildPrint bool, buildMetaData *Build) {
var wg sync.WaitGroup
s := time.Now()
started := time.Now()
wg.Add(2)
xflags := buildMetaData.XFlags()
cmds := make([]*exec.Cmd, 2)
cmds[0] = utils.CommandWithStdout("gox", "-output={{.Dir}}-{{.OS}}-{{.Arch}}-musl", "-buildmode=pie", "-trimpath", "-cgo", "-ldflags=-linkmode=external -s -w "+strings.Join(xflags, " "), "-osarch=linux/amd64 linux/arm linux/arm64", "./cmd/authelia/")
cmds[0].Env = append(cmds[0].Env,
"CGO_CPPFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong", "CGO_LDFLAGS=-Wl,-z,relro,-z,now",
"GOX_LINUX_ARM_CC=arm-linux-musleabihf-gcc", "GOX_LINUX_ARM64_CC=aarch64-linux-musl-gcc")
cmds[1] = utils.CommandWithStdout("bash", "-c", "docker run --rm -e GOX_LINUX_ARM_CC=arm-linux-gnueabihf-gcc -e GOX_LINUX_ARM64_CC=aarch64-linux-gnu-gcc -e GOX_FREEBSD_AMD64_CC=x86_64-pc-freebsd13-gcc -v ${PWD}:/workdir -v /buildkite/.go:/root/go authelia/crossbuild "+
"gox -output={{.Dir}}-{{.OS}}-{{.Arch}} -buildmode=pie -trimpath -cgo -ldflags=\"-linkmode=external -s -w "+strings.Join(xflags, " ")+"\" -osarch=\"linux/amd64 linux/arm linux/arm64 freebsd/amd64\" ./cmd/authelia/")
if buildPrint {
for _, cmd := range cmds {
buildCmdPrint(cmd)
}
return
}
log.Info("Building Authelia Go binaries with gox...")
wg.Add(len(cmds))
go func() {
defer wg.Done()
cmd := utils.CommandWithStdout("gox", "-output={{.Dir}}-{{.OS}}-{{.Arch}}-musl", "-buildmode=pie", "-trimpath", "-cgo", "-ldflags=-linkmode=external -s -w "+strings.Join(xflags, " "), "-osarch=linux/amd64 linux/arm linux/arm64", "./cmd/authelia/")
cmd.Env = append(os.Environ(),
"CGO_CPPFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong", "CGO_LDFLAGS=-Wl,-z,relro,-z,now",
"GOX_LINUX_ARM_CC=arm-linux-musleabihf-gcc", "GOX_LINUX_ARM64_CC=aarch64-linux-musl-gcc")
err := cmd.Run()
if err != nil {
log.Fatal(err)
}
buildCmdRun(cmds[0])
}()
go func() {
defer wg.Done()
cmd := utils.CommandWithStdout("bash", "-c", "docker run --rm -e GOX_LINUX_ARM_CC=arm-linux-gnueabihf-gcc -e GOX_LINUX_ARM64_CC=aarch64-linux-gnu-gcc -e GOX_FREEBSD_AMD64_CC=x86_64-pc-freebsd13-gcc -v ${PWD}:/workdir -v /buildkite/.go:/root/go authelia/crossbuild "+
"gox -output={{.Dir}}-{{.OS}}-{{.Arch}} -buildmode=pie -trimpath -cgo -ldflags=\"-linkmode=external -s -w "+strings.Join(xflags, " ")+"\" -osarch=\"linux/amd64 linux/arm linux/arm64 freebsd/amd64\" ./cmd/authelia/")
err := cmd.Run()
if err != nil {
log.Fatal(err)
}
buildCmdRun(cmds[1])
}()
wg.Wait()
e := time.Since(s)
log.Debugf("Binary compilation completed in %s.", e)
log.Debugf("Binary compilation completed in %s.", time.Since(started))
}
func buildAutheliaBinaryGO(xflags []string) {
cmd := utils.CommandWithStdout("go", "build", "-buildmode=pie", "-trimpath", "-o", OutputDir+"/authelia", "-ldflags", "-linkmode=external -s -w "+strings.Join(xflags, " "), "./cmd/authelia/")
func buildAutheliaBinaryGO(buildPrint bool, buildMetaData *Build) {
cmd := utils.CommandWithStdout("go", "build", "-buildmode=pie", "-trimpath", "-o", OutputDir+"/authelia", "-ldflags", "-linkmode=external -s -w "+strings.Join(buildMetaData.XFlags(), " "), "./cmd/authelia/")
cmd.Env = append(os.Environ(),
cmd.Env = append(cmd.Env,
"CGO_CPPFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong", "CGO_LDFLAGS=-Wl,-z,relro,-z,now")
err := cmd.Run()
if err != nil {
log.Fatal(err)
if buildPrint {
buildCmdPrint(cmd)
return
}
log.Info("Building Authelia Go binary...")
buildCmdRun(cmd)
}
func buildFrontend(branch string) {
cmd := utils.CommandWithStdout("pnpm", "install")
cmd.Dir = webDirectory
func buildFrontend(buildPrint bool, branch string) {
var (
cmds []*exec.Cmd
cmd *exec.Cmd
)
err := cmd.Run()
if err != nil {
log.Fatal(err)
}
cmd = utils.CommandWithStdout("pnpm", "install")
cmd.Dir = filepath.Join(cmd.Dir, webDirectory)
cmds = append(cmds, cmd)
if !strings.HasPrefix(branch, "renovate/") {
cmd = utils.CommandWithStdout("pnpm", "build")
cmd.Dir = webDirectory
cmd.Dir = filepath.Join(cmd.Dir, webDirectory)
err = cmd.Run()
if err != nil {
log.Fatal(err)
cmds = append(cmds, cmd)
}
for _, cmd = range cmds {
if buildPrint {
buildCmdPrint(cmd)
continue
}
buildCmdRun(cmd)
}
}
func buildSwagger() {
cmd := utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+versionSwaggerUI+".tar.gz -O ./v"+versionSwaggerUI+".tar.gz")
func buildSwagger(buildPrint bool) {
var (
cmds []*exec.Cmd
cmd *exec.Cmd
)
err := cmd.Run()
if err != nil {
log.Fatal(err)
cmds = append(cmds, utils.CommandWithStdout("bash", "-c", "wget -q https://github.com/swagger-api/swagger-ui/archive/v"+versionSwaggerUI+".tar.gz -O ./v"+versionSwaggerUI+".tar.gz"))
cmds = append(cmds, utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html"))
cmds = append(cmds, utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+versionSwaggerUI+".tar.gz", "swagger-ui-"+versionSwaggerUI+"/dist"))
cmds = append(cmds, utils.CommandWithStdout("rm", "./v"+versionSwaggerUI+".tar.gz"))
for _, cmd = range cmds {
if buildPrint {
buildCmdPrint(cmd)
continue
}
cmd = utils.CommandWithStdout("cp", "-r", "api", "internal/server/public_html")
err = cmd.Run()
if err != nil {
log.Fatal(err)
}
cmd = utils.CommandWithStdout("tar", "-C", "internal/server/public_html/api", "--exclude=index.html", "--strip-components=2", "-xf", "v"+versionSwaggerUI+".tar.gz", "swagger-ui-"+versionSwaggerUI+"/dist")
err = cmd.Run()
if err != nil {
log.Fatal(err)
}
cmd = utils.CommandWithStdout("rm", "./v"+versionSwaggerUI+".tar.gz")
err = cmd.Run()
if err != nil {
log.Fatal(err)
buildCmdRun(cmd)
}
}
@ -182,13 +216,36 @@ func cleanAssets() {
cmd := utils.CommandWithStdout("mkdir", "-p", "internal/server/public_html/api")
if err := cmd.Run(); err != nil {
log.Fatal(err)
}
buildCmdRun(cmd)
cmd = utils.CommandWithStdout("bash", "-c", "touch internal/server/public_html/{index.html,api/index.html,api/openapi.yml}")
buildCmdRun(cmd)
}
func buildCmdRun(cmd *exec.Cmd) {
if len(cmd.Env) != 0 {
cmd.Env = append(os.Environ(), cmd.Env...)
}
if err := cmd.Run(); err != nil {
log.Fatal(err)
}
}
func buildCmdPrint(cmd *exec.Cmd) {
b := &strings.Builder{}
if cmd.Dir != "" {
b.WriteString(fmt.Sprintf("cd %s\n", cmd.Dir))
}
if len(cmd.Env) != 0 {
b.WriteString(strings.Join(cmd.Env, " "))
b.WriteString(" ")
}
b.WriteString(cmd.String())
fmt.Println(b.String())
}

18
cmd/authelia-scripts/cmd/helpers.go
View File

@ -3,6 +3,7 @@ package cmd
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/authelia/authelia/v4/internal/utils"
@ -50,6 +51,21 @@ func getBuild(branch, buildNumber, extra string) (b *Build, err error) {
return nil, fmt.Errorf("error getting commit with git rev-parse: %w", err)
}
var (
gitCommitTS string
gitCommitTSI int
)
if gitCommitTS, _, err = utils.RunCommandAndReturnOutput(fmt.Sprintf("git show -s --format=%%ct %s", b.Commit)); err != nil {
return nil, fmt.Errorf("error getting commit date with git show -s --format=%%ct %s: %w", b.Commit, err)
}
if gitCommitTSI, err = strconv.Atoi(strings.TrimSpace(gitCommitTS)); err != nil {
return nil, fmt.Errorf("error getting commit date with git show -s --format=%%ct %s: %w", b.Commit, err)
}
b.Date = time.Unix(int64(gitCommitTSI), 0).UTC()
if gitTagCommit == b.Commit {
b.Tagged = true
}
@ -58,7 +74,5 @@ func getBuild(branch, buildNumber, extra string) (b *Build, err error) {
b.Clean = true
}
b.Date = time.Now()
return b, nil
}

4
cmd/authelia-scripts/cmd/types.go
View File

@ -85,7 +85,7 @@ func (b Build) XFlags() []string {
fmt.Sprintf(fmtLDFLAGSX, "BuildBranch", b.Branch),
fmt.Sprintf(fmtLDFLAGSX, "BuildTag", b.Tag),
fmt.Sprintf(fmtLDFLAGSX, "BuildCommit", b.Commit),
fmt.Sprintf(fmtLDFLAGSX, "BuildDate", b.Date.Format("Mon, 02 Jan 2006 15:04:05 -0700")),
fmt.Sprintf(fmtLDFLAGSX, "BuildDate", b.Date.Format(time.RFC1123)),
fmt.Sprintf(fmtLDFLAGSX, "BuildState", b.State()),
fmt.Sprintf(fmtLDFLAGSX, "BuildExtra", b.Extra),
fmt.Sprintf(fmtLDFLAGSX, "BuildNumber", strconv.Itoa(b.Number)),
@ -112,7 +112,7 @@ func (b Build) ContainerLabels() (labels map[string]string) {
}
labels = map[string]string{
"org.opencontainers.image.created": b.Date.Format(time.RFC3339),
"org.opencontainers.image.created": b.Date.UTC().Format(time.RFC3339),
"org.opencontainers.image.authors": "",
"org.opencontainers.image.url": "https://github.com/authelia/authelia/pkgs/container/authelia",
"org.opencontainers.image.documentation": "https://www.authelia.com",

44
docs/content/en/contributing/development/build-and-test.md
View File

@ -102,6 +102,8 @@ If you want to manually build the binary from source you will require the open s
[Development Environment](./environment.md#setup) documentation. Then you can follow the below steps on Linux (you may
have to adapt them on other systems).
#### Basic Steps
Clone the Repository:
```bash
@ -137,6 +139,48 @@ CGO_ENABLED=1 CGO_CPPFLAGS="-D_FORTIFY_SOURCE=2 -fstack-protector-strong" CGO_LD
go build -ldflags "-linkmode=external -s -w" -trimpath -buildmode=pie -o authelia ./cmd/authelia
```
#### Reproducible Builds
Authelia allows production of reproducible builds that were built using our pipeline. The only variables injected into
a build are from commit information other than the exceptions listed in this section. This means that we can provide the
exact build commands for any given build with very limited input from users. The elements injected into the binary as
part of the build process (using linker flags) are:
- Commit SHA1
- Commit Date (using the RFC1123 layout strictly using the UTC timezone)
- Latest Tag
- Tag State (i.e. if the HEAD commit has the latest tag)
- Working Tree State (dirty, clean, etc)
- Branch Name
- Build Number
The exceptions of this list which cannot be obtained from commit information (but can be supplied by an environment
variable or CLI argument):
- Build Number
##### Instructions
To perform a reproducible build users should follow these steps:
1. Run the `authelia build-info` command which contains useful information for reproducing the build including:
1. The `Build Number` field.
2. The `Build Go Version` field.
2. Install all of the required dependencies. It's recommended if you're looking for a reproducible build that you use
the same Go version from step 1.
3. Run the following command from the root of the repository to output the build commands (where 100 is the number from
step 1):
```bash
go run ./cmd/authelia-scripts build --print --build-number 100
```
The output of the above command may be ran to perform all of the build steps manually.
*__Important Note:__ If you wish to use [gox](https://gitihub.com/authelia/gox) to build Authelia please run the
`go run ./cmd/authelia-scripts build --print --buildkite --build-number 100` command instead of the above command (i.e.
adding the `--buildkite` flag).*
[suites]: ./integration-suites.md
[React]: https://reactjs.org/
[go]: https://go.dev/dl/

2
internal/commands/build-info.go
View File

@ -27,7 +27,7 @@ func newBuildInfoCmd(ctx *CmdCtx) (cmd *cobra.Command) {
// BuildInfoRunE is the RunE for the authelia build-info command.
func (ctx *CmdCtx) BuildInfoRunE(_ *cobra.Command, _ []string) (err error) {
_, err = fmt.Printf(fmtAutheliaBuild, utils.BuildTag, utils.BuildState, utils.BuildBranch, utils.BuildCommit,
utils.BuildNumber, runtime.GOOS, runtime.GOARCH, utils.BuildDate, utils.BuildExtra)
utils.BuildDate, utils.BuildNumber, runtime.GOOS, runtime.GOARCH, runtime.Version(), utils.BuildExtra)
return err
}

3
internal/commands/const.go
View File

@ -24,10 +24,11 @@ authelia --config /etc/authelia/config/`
State: %s
Branch: %s
Commit: %s
Commit Date: %s
Build Number: %s
Build OS: %s
Build Arch: %s
Build Date: %s
Build Go Version: %s
Extra: %s
`

11
internal/suites/suite_cli_test.go
View File

@ -11,7 +11,7 @@ import (
"testing"
"github.com/stretchr/testify/suite"
"gopkg.in/yaml.v3"
yaml "gopkg.in/yaml.v3"
"github.com/authelia/authelia/v4/internal/model"
"github.com/authelia/authelia/v4/internal/storage"
@ -50,15 +50,8 @@ func (s *CLISuite) TestShouldPrintBuildInformation() {
output, err := s.Exec("authelia-backend", []string{"authelia", "build-info"})
s.Assert().NoError(err)
s.Assert().Contains(output, "Last Tag: ")
s.Assert().Contains(output, "State: ")
s.Assert().Contains(output, "Branch: ")
s.Assert().Contains(output, "Build Number: ")
s.Assert().Contains(output, "Build OS: ")
s.Assert().Contains(output, "Build Arch: ")
s.Assert().Contains(output, "Build Date: ")
r := regexp.MustCompile(`^Last Tag: v\d+\.\d+\.\d+\nState: (tagged|untagged) (clean|dirty)\nBranch: [^\s\n]+\nCommit: [0-9a-f]{40}\nBuild Number: \d+\nBuild OS: (linux|darwin|windows|freebsd)\nBuild Arch: (amd64|arm|arm64)\nBuild Date: (Sun|Mon|Tue|Wed|Thu|Fri|Sat), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} [+-]\d{4}\nExtra: \n`)
r := regexp.MustCompile(`^Last Tag: v\d+\.\d+\.\d+\nState: (tagged|untagged) (clean|dirty)\nBranch: [^\s\n]+\nCommit: [0-9a-f]{40}\nCommit Date: (Sun|Mon|Tue|Wed|Thu|Fri|Sat), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} \+0000\nBuild Number: \d+\nBuild OS: (linux|darwin|windows|freebsd)\nBuild Arch: (amd64|arm|arm64)\nBuild Go Version: go\d+\.\d+(\.\d+)?\nExtra:`)
s.Assert().Regexp(r, output)
}