From d3ae996af6e6c84ace4c13426e75060eddc17549 Mon Sep 17 00:00:00 2001 From: Jeff Wong Date: Tue, 27 Aug 2024 22:13:39 -0700 Subject: [PATCH] FEATURE: add configure command (#841) Add 'configure' command - If run after the "build" command, this is equivalent to today's 'bootstrap' command. Note that unlike build command, a docker run+commit pattern needs to be used here as this requires a running database + mounted volumes. --- launcher_go/v2/cli_build.go | 25 ++++ launcher_go/v2/cli_build_test.go | 39 +++++ launcher_go/v2/config/config.go | 26 +++- launcher_go/v2/config/config_test.go | 15 ++ launcher_go/v2/docker/commands.go | 199 +++++++++++++++++++++++++ launcher_go/v2/docker/commands_test.go | 43 ++++++ launcher_go/v2/go_suite_test.go | 5 + launcher_go/v2/main.go | 11 +- launcher_go/v2/test_utils/utils.go | 6 + launcher_go/v2/utils/consts.go | 3 + 10 files changed, 365 insertions(+), 7 deletions(-) create mode 100644 launcher_go/v2/docker/commands_test.go diff --git a/launcher_go/v2/cli_build.go b/launcher_go/v2/cli_build.go index 0a55757..d35f8d2 100644 --- a/launcher_go/v2/cli_build.go +++ b/launcher_go/v2/cli_build.go @@ -5,6 +5,8 @@ import ( "errors" "github.com/discourse/discourse_docker/launcher_go/v2/config" "github.com/discourse/discourse_docker/launcher_go/v2/docker" + "github.com/discourse/discourse_docker/launcher_go/v2/utils" + "github.com/google/uuid" "os" "strings" ) @@ -53,6 +55,29 @@ func (r *DockerBuildCmd) Run(cli *Cli, ctx *context.Context) error { return nil } +type DockerConfigureCmd struct { + Tag string `default:"latest" help:"Resulting image tag."` + Config string `arg:"" name:"config" help:"config" predictor:"config"` +} + +func (r *DockerConfigureCmd) Run(cli *Cli, ctx *context.Context) error { + config, err := config.LoadConfig(cli.ConfDir, r.Config, true, cli.TemplatesDir) + if err != nil { + return errors.New("YAML syntax error. Please check your containers/*.yml config files.") + } + + containerId := "discourse-build-" + uuid.NewString() + pups := docker.DockerPupsRunner{ + Config: config, + PupsArgs: "--tags=db,precompile", + SavedImageName: utils.BaseImageName + r.Config + ":" + r.Tag, + ExtraEnv: []string{"SKIP_EMBER_CLI_COMPILE=1"}, + Ctx: ctx, + ContainerId: containerId, + } + return pups.Run() +} + type CleanCmd struct { Config string `arg:"" name:"config" help:"config to clean" predictor:"config"` } diff --git a/launcher_go/v2/cli_build_test.go b/launcher_go/v2/cli_build_test.go index 3aa8f79..98cc299 100644 --- a/launcher_go/v2/cli_build_test.go +++ b/launcher_go/v2/cli_build_test.go @@ -57,11 +57,50 @@ var _ = Describe("Build", func() { Expect(buf.String()).ToNot(ContainSubstring("SKIP_EMBER_CLI_COMPILE=1")) } + var checkConfigureCmd = func(cmd exec.Cmd) { + Expect(cmd.String()).To(ContainSubstring("docker run")) + Expect(cmd.String()).To(ContainSubstring("--env DISCOURSE_DEVELOPER_EMAILS")) + Expect(cmd.String()).To(ContainSubstring("--env SKIP_EMBER_CLI_COMPILE=1")) + // we commit, we need the container to stick around after it is stopped. + Expect(cmd.String()).ToNot(ContainSubstring("--rm")) + + // we don't expose ports on configure command + Expect(cmd.String()).ToNot(ContainSubstring("-p 80")) + Expect(cmd.Env).To(ContainElement("DISCOURSE_DB_PASSWORD=SOME_SECRET")) + buf := new(strings.Builder) + io.Copy(buf, cmd.Stdin) + // docker run's stdin is a pups config + Expect(buf.String()).To(ContainSubstring("path: /etc/service/nginx/run")) + } + + // commit on configure + var checkConfigureCommit = func(cmd exec.Cmd) { + Expect(cmd.String()).To(ContainSubstring("docker commit")) + Expect(cmd.String()).To(ContainSubstring("--change CMD [\"/sbin/boot\"]")) + Expect(cmd.String()).To(ContainSubstring("discourse-build")) + Expect(cmd.String()).To(ContainSubstring("local_discourse/test")) + Expect(cmd.Env).ToNot(ContainElement("DISCOURSE_DB_PASSWORD=SOME_SECRET")) + } + + // configure also cleans up + var checkConfigureClean = func(cmd exec.Cmd) { + Expect(cmd.String()).To(ContainSubstring("docker rm -f discourse-build-")) + } + It("Should run docker build with correct arguments", func() { runner := ddocker.DockerBuildCmd{Config: "test"} runner.Run(cli, &ctx) Expect(len(RanCmds)).To(Equal(1)) checkBuildCmd(RanCmds[0]) }) + + It("Should run docker run followed by docker commit and rm container when configuring", func() { + runner := ddocker.DockerConfigureCmd{Config: "test"} + runner.Run(cli, &ctx) + Expect(len(RanCmds)).To(Equal(3)) + checkConfigureCmd(RanCmds[0]) + checkConfigureCommit(RanCmds[1]) + checkConfigureClean(RanCmds[2]) + }) }) }) diff --git a/launcher_go/v2/config/config.go b/launcher_go/v2/config/config.go index 4c9ab22..c527b4b 100644 --- a/launcher_go/v2/config/config.go +++ b/launcher_go/v2/config/config.go @@ -143,7 +143,7 @@ func (config *Config) Dockerfile(pupsArgs string, bakeEnv bool) string { builder.WriteString("RUN " + "cat /temp-config.yaml | /usr/local/bin/pups " + pupsArgs + " --stdin " + "&& rm /temp-config.yaml\n") - builder.WriteString("CMD [\"" + config.bootCommand() + "\"]") + builder.WriteString("CMD [\"" + config.BootCommand() + "\"]") return builder.String() } @@ -155,7 +155,7 @@ func (config *Config) WriteYamlConfig(dir string) error { return nil } -func (config *Config) bootCommand() string { +func (config *Config) BootCommand() string { if len(config.Boot_Command) > 0 { return config.Boot_Command } else if config.No_Boot_Command { @@ -177,6 +177,10 @@ func (config *Config) EnvArray(includeKnownSecrets bool) []string { return envs } +func (config *Config) DockerArgs() []string { + return strings.Fields(config.Docker_Args) +} + func (config *Config) dockerfileEnvs() string { builder := []string{} for k, _ := range config.Env { @@ -207,3 +211,21 @@ func (config *Config) dockerfileExpose() string { slices.Sort(builder) return strings.Join(builder, "\n") } + +func (config *Config) RunImage() string { + if len(config.Run_Image) > 0 { + return config.Run_Image + } + return utils.BaseImageName + config.Name +} + +func (config *Config) DockerHostname(defaultHostname string) string { + _, exists := config.Env["DOCKER_USE_HOSTNAME"] + re := regexp.MustCompile(`[^a-zA-Z-]`) + hostname := defaultHostname + if exists { + hostname = config.Env["DISCOURSE_HOSTNAME"] + } + hostname = string(re.ReplaceAll([]byte(hostname), []byte("-"))[:]) + return hostname +} diff --git a/launcher_go/v2/config/config_test.go b/launcher_go/v2/config/config_test.go index 24bd4b8..85ce46e 100644 --- a/launcher_go/v2/config/config_test.go +++ b/launcher_go/v2/config/config_test.go @@ -43,4 +43,19 @@ var _ = Describe("Config", func() { Expect(dockerfile).To(ContainSubstring("RUN cat /temp-config.yaml")) Expect(dockerfile).To(ContainSubstring("EXPOSE 80")) }) + + Context("hostname tests", func() { + It("replaces hostname", func() { + config := config.Config{Env: map[string]string{"DOCKER_USE_HOSTNAME": "true", "DISCOURSE_HOSTNAME": "asdfASDF"}} + Expect(config.DockerHostname("")).To(Equal("asdfASDF")) + }) + It("replaces hostname", func() { + config := config.Config{Env: map[string]string{"DOCKER_USE_HOSTNAME": "true", "DISCOURSE_HOSTNAME": "asdf!@#$%^&*()ASDF"}} + Expect(config.DockerHostname("")).To(Equal("asdf----------ASDF")) + }) + It("replaces a default hostnamehostname", func() { + config := config.Config{} + Expect(config.DockerHostname("asdf!@#")).To(Equal("asdf---")) + }) + }) }) diff --git a/launcher_go/v2/docker/commands.go b/launcher_go/v2/docker/commands.go index 04c4879..d6fd21f 100644 --- a/launcher_go/v2/docker/commands.go +++ b/launcher_go/v2/docker/commands.go @@ -2,13 +2,18 @@ package docker import ( "context" + "fmt" + "github.com/Wing924/shellwords" "github.com/discourse/discourse_docker/launcher_go/v2/config" "github.com/discourse/discourse_docker/launcher_go/v2/utils" "golang.org/x/sys/unix" "io" "os" "os/exec" + "runtime" + "strings" "syscall" + "time" ) type DockerBuilder struct { @@ -52,3 +57,197 @@ func (r *DockerBuilder) Run() error { } return nil } + +type DockerRunner struct { + Config *config.Config + Ctx *context.Context + ExtraEnv []string + ExtraFlags []string + Rm bool + ContainerId string + CustomImage string + Cmd []string + Stdin io.Reader + SkipPorts bool + DryRun bool + Restart bool + Detatch bool + Hostname string +} + +func (r *DockerRunner) Run() error { + cmd := exec.CommandContext(*r.Ctx, utils.DockerPath, "run") + + // Detatch signifies we do not want to supervise + if !r.Detatch { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if runtime.GOOS == "darwin" { + runCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + stopCmd := exec.CommandContext(runCtx, utils.DockerPath, "stop", r.ContainerId) + utils.CmdRunner(stopCmd).Run() + cancel() + } + return unix.Kill(-cmd.Process.Pid, unix.SIGINT) + } + } + cmd.Env = r.Config.EnvArray(true) + + if r.DryRun { + // multi-line env doesn't work super great from CLI, but we can print out the rest. + for k, v := range r.Config.Env { + if !strings.Contains(v, "\n") { + cmd.Args = append(cmd.Args, "--env") + cmd.Args = append(cmd.Args, k+"="+shellwords.Escape(v)) + } + } + } else { + for k, _ := range r.Config.Env { + cmd.Args = append(cmd.Args, "--env") + cmd.Args = append(cmd.Args, k) + } + } + + // Order is important here, we add extra env after config's env to override anything set in env. + for _, e := range r.ExtraEnv { + cmd.Args = append(cmd.Args, "--env") + cmd.Args = append(cmd.Args, e) + } + for k, v := range r.Config.Labels { + cmd.Args = append(cmd.Args, "--label") + cmd.Args = append(cmd.Args, k+"="+v) + } + if !r.SkipPorts { + for _, v := range r.Config.Expose { + if strings.Contains(v, ":") { + cmd.Args = append(cmd.Args, "-p") + cmd.Args = append(cmd.Args, v) + } else { + cmd.Args = append(cmd.Args, "--expose") + cmd.Args = append(cmd.Args, v) + } + } + } + for _, v := range r.Config.Volumes { + cmd.Args = append(cmd.Args, "-v") + cmd.Args = append(cmd.Args, v.Volume.Host+":"+v.Volume.Guest) + } + for _, v := range r.Config.Links { + cmd.Args = append(cmd.Args, "--link") + cmd.Args = append(cmd.Args, v.Link.Name+":"+v.Link.Alias) + } + cmd.Args = append(cmd.Args, "--shm-size=512m") + if r.Rm { + cmd.Args = append(cmd.Args, "--rm") + } + if r.Restart { + cmd.Args = append(cmd.Args, "--restart=always") + } else { + cmd.Args = append(cmd.Args, "--restart=no") + } + if r.Detatch { + cmd.Args = append(cmd.Args, "-d") + } + cmd.Args = append(cmd.Args, "-i") + + // Docker args override settings above + for _, f := range r.Config.DockerArgs() { + cmd.Args = append(cmd.Args, f) + } + for _, f := range r.ExtraFlags { + cmd.Args = append(cmd.Args, f) + } + cmd.Args = append(cmd.Args, "-h") + cmd.Args = append(cmd.Args, r.Hostname) + cmd.Args = append(cmd.Args, "--name") + cmd.Args = append(cmd.Args, r.ContainerId) + if len(r.CustomImage) > 0 { + cmd.Args = append(cmd.Args, r.CustomImage) + } else { + cmd.Args = append(cmd.Args, r.Config.RunImage()) + } + + for _, c := range r.Cmd { + cmd.Args = append(cmd.Args, c) + } + + if !r.Detatch { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = r.Stdin + } + runner := utils.CmdRunner(cmd) + if r.DryRun { + fmt.Println(cmd) + } else { + if err := runner.Run(); err != nil { + return err + } + } + return nil +} + +type DockerPupsRunner struct { + Config *config.Config + PupsArgs string + SavedImageName string + ExtraEnv []string + Ctx *context.Context + ContainerId string +} + +func (r *DockerPupsRunner) Run() error { + rm := false + // remove : in case docker tag is blank, and use default latest tag + r.SavedImageName = strings.TrimRight(r.SavedImageName, ":") + if r.SavedImageName == "" { + rm = true + } + defer func(rm bool) { + if !rm { + time.Sleep(utils.CommitWait) + runCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + cmd := exec.CommandContext(runCtx, utils.DockerPath, "rm", "-f", r.ContainerId) + utils.CmdRunner(cmd).Run() + cancel() + } + }(rm) + commands := []string{"/bin/bash", + "-c", + "/usr/local/bin/pups --stdin " + r.PupsArgs} + + runner := DockerRunner{Config: r.Config, + Ctx: r.Ctx, + ExtraEnv: r.ExtraEnv, + Rm: rm, + ContainerId: r.ContainerId, + Cmd: commands, + Stdin: strings.NewReader(r.Config.Yaml()), + SkipPorts: true, //pups runs don't need to expose ports + } + + if err := runner.Run(); err != nil { + return err + } + + if len(r.SavedImageName) > 0 { + time.Sleep(utils.CommitWait) + cmd := exec.Command("docker", + "commit", + "--change", + "LABEL org.opencontainers.image.created=\""+time.Now().Format(time.RFC3339)+"\"", + "--change", + "CMD [\""+r.Config.BootCommand()+"\"]", + r.ContainerId, + r.SavedImageName, + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + fmt.Fprintln(utils.Out, cmd) + if err := utils.CmdRunner(cmd).Run(); err != nil { + return err + } + } + return nil +} diff --git a/launcher_go/v2/docker/commands_test.go b/launcher_go/v2/docker/commands_test.go new file mode 100644 index 0000000..aa7078c --- /dev/null +++ b/launcher_go/v2/docker/commands_test.go @@ -0,0 +1,43 @@ +package docker_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "bytes" + "context" + "github.com/discourse/discourse_docker/launcher_go/v2/config" + "github.com/discourse/discourse_docker/launcher_go/v2/docker" + . "github.com/discourse/discourse_docker/launcher_go/v2/test_utils" + "github.com/discourse/discourse_docker/launcher_go/v2/utils" + "strings" +) + +var _ = Describe("Commands", func() { + Context("under normal conditions", func() { + var conf *config.Config + var out *bytes.Buffer + var ctx context.Context + + BeforeEach(func() { + utils.DockerPath = "docker" + out = &bytes.Buffer{} + utils.Out = out + utils.CommitWait = 0 + conf = &config.Config{Name: "test"} + ctx = context.Background() + utils.CmdRunner = CreateNewFakeCmdRunner() + }) + It("Removes unspecified image tags on commit", func() { + runner := docker.DockerPupsRunner{Config: conf, ContainerId: "123", Ctx: &ctx, SavedImageName: "local_discourse/test:"} + runner.Run() + cmd := GetLastCommand() + Expect(cmd.String()).To(ContainSubstring("docker run")) + cmd = GetLastCommand() + Expect(cmd.String()).To(ContainSubstring("docker commit")) + Expect(strings.HasSuffix(cmd.String(), ":")).To(BeFalse()) + cmd = GetLastCommand() + Expect(cmd.String()).To(ContainSubstring("docker rm")) + }) + }) +}) diff --git a/launcher_go/v2/go_suite_test.go b/launcher_go/v2/go_suite_test.go index 7549994..8ca1647 100644 --- a/launcher_go/v2/go_suite_test.go +++ b/launcher_go/v2/go_suite_test.go @@ -3,6 +3,7 @@ package main_test import ( "testing" + "github.com/discourse/discourse_docker/launcher_go/v2/utils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -11,3 +12,7 @@ func TestMain(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "Main Suite") } + +var _ = BeforeSuite(func() { + utils.CommitWait = 0 +}) diff --git a/launcher_go/v2/main.go b/launcher_go/v2/main.go index 0c7258f..165c8e8 100644 --- a/launcher_go/v2/main.go +++ b/launcher_go/v2/main.go @@ -12,11 +12,12 @@ import ( ) type Cli struct { - Version kong.VersionFlag `help:"Show version."` - ConfDir string `default:"./containers" hidden:"" help:"Discourse pups config directory." predictor:"dir"` - TemplatesDir string `default:"." hidden:"" help:"Home project directory containing a templates/ directory which in turn contains pups yaml templates." predictor:"dir"` - BuildDir string `default:"./tmp" hidden:"" help:"Temporary build folder for building images." predictor:"dir"` - BuildCmd DockerBuildCmd `cmd:"" name:"build" help:"Build a base image. This command does not need a running database. Saves resulting container."` + Version kong.VersionFlag `help:"Show version."` + ConfDir string `default:"./containers" hidden:"" help:"Discourse pups config directory." predictor:"dir"` + TemplatesDir string `default:"." hidden:"" help:"Home project directory containing a templates/ directory which in turn contains pups yaml templates." predictor:"dir"` + BuildDir string `default:"./tmp" hidden:"" help:"Temporary build folder for building images." predictor:"dir"` + BuildCmd DockerBuildCmd `cmd:"" name:"build" help:"Build a base image. This command does not need a running database. Saves resulting container."` + ConfigureCmd DockerConfigureCmd `cmd:"" name:"configure" help:"Configure and save an image with all dependencies and environment baked in. Updates themes and precompiles all assets. Saves resulting container."` } func main() { diff --git a/launcher_go/v2/test_utils/utils.go b/launcher_go/v2/test_utils/utils.go index 1c716e6..c1c1ee8 100644 --- a/launcher_go/v2/test_utils/utils.go +++ b/launcher_go/v2/test_utils/utils.go @@ -34,3 +34,9 @@ func CreateNewFakeCmdRunner() func(cmd *exec.Cmd) utils.ICmdRunner { return cmdRunner } } + +func GetLastCommand() exec.Cmd { + cmd := RanCmds[0] + RanCmds = RanCmds[1:] + return cmd +} diff --git a/launcher_go/v2/utils/consts.go b/launcher_go/v2/utils/consts.go index 21cc3a6..5fd4e64 100644 --- a/launcher_go/v2/utils/consts.go +++ b/launcher_go/v2/utils/consts.go @@ -4,6 +4,7 @@ import ( "io" "os" "os/exec" + "time" ) const Version = "v2.0.0" @@ -44,3 +45,5 @@ func findDockerPath() string { var DockerPath = findDockerPath() var Out io.Writer = os.Stdout + +var CommitWait = 2 * time.Second -- 2.25.1