diff --git a/go.mod b/go.mod index 202b675..803341b 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/nalgeon/codapi -go 1.20 +go 1.21 diff --git a/internal/config/config.go b/internal/config/config.go index 3fc4036..5323886 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -59,7 +59,8 @@ type Box struct { Runtime string `json:"runtime"` Host - Files []string `json:"files"` + Versions []string `json:"versions"` + Files []string `json:"files"` } // A Host describes container Host attributes. @@ -96,6 +97,7 @@ type Command struct { // A Step describes a single step of a command. type Step struct { Box string `json:"box"` + Version string `json:"version"` User string `json:"user"` Action string `json:"action"` Stdin bool `json:"stdin"` diff --git a/internal/engine/docker.go b/internal/engine/docker.go index 66ef02d..5214067 100644 --- a/internal/engine/docker.go +++ b/internal/engine/docker.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" "time" @@ -62,7 +63,7 @@ func (e *Docker) Exec(req Request) Execution { // initialization step if e.cmd.Before != nil { - out := e.execStep(e.cmd.Before, req.ID, dir, nil) + out := e.execStep(e.cmd.Before, req, dir, nil) if !out.OK { return out } @@ -70,14 +71,14 @@ func (e *Docker) Exec(req Request) Execution { // the first step is required first, rest := e.cmd.Steps[0], e.cmd.Steps[1:] - out := e.execStep(first, req.ID, dir, req.Files) + out := e.execStep(first, req, dir, req.Files) // the rest are optional if out.OK && len(rest) > 0 { // each step operates on the results of the previous one, // without using the source files - hence `nil` instead of `files` for _, step := range rest { - out = e.execStep(step, req.ID, dir, nil) + out = e.execStep(step, req, dir, nil) if !out.OK { break } @@ -86,7 +87,7 @@ func (e *Docker) Exec(req Request) Execution { // cleanup step if e.cmd.After != nil { - afterOut := e.execStep(e.cmd.After, req.ID, dir, nil) + afterOut := e.execStep(e.cmd.After, req, dir, nil) if out.OK && !afterOut.OK { return afterOut } @@ -96,27 +97,44 @@ func (e *Docker) Exec(req Request) Execution { } // execStep executes a step using the docker container. -func (e *Docker) execStep(step *config.Step, reqID, dir string, files Files) Execution { +func (e *Docker) execStep(step *config.Step, req Request, dir string, files Files) Execution { box := e.cfg.Boxes[step.Box] - err := e.copyFiles(box, dir) + err := e.validateVersion(box, step, req) if err != nil { - err = NewExecutionError("copy files to temp dir", err) - return Fail(reqID, err) + return Fail(req.ID, err) } - stdout, stderr, err := e.exec(box, step, reqID, dir, files) + err = e.copyFiles(box, dir) if err != nil { - return Fail(reqID, err) + err = NewExecutionError("copy files to temp dir", err) + return Fail(req.ID, err) + } + + stdout, stderr, err := e.exec(box, step, req, dir, files) + if err != nil { + return Fail(req.ID, err) } return Execution{ - ID: reqID, + ID: req.ID, OK: true, Stdout: stdout, Stderr: stderr, } } +func (e *Docker) validateVersion(box *config.Box, step *config.Step, req Request) error { + // If the version is set in the step config, use it. + // If the version isn't set in the request, use the latest one. + // Otherwise, check that the version in the request is supported + // according to the box config. + if step.Version == "" && req.Version != "" && !slices.Contains(box.Versions, req.Version) { + err := fmt.Errorf("box %s does not support version %s", step.Box, req.Version) + return err + } + return nil +} + // copyFiles copies box files to the temporary directory. func (e *Docker) copyFiles(box *config.Box, dir string) error { if box == nil || len(box.Files) == 0 { @@ -147,18 +165,18 @@ func (e *Docker) writeFiles(dir string, files Files) error { // exec executes the step in the docker container // using the files from in the temporary directory. -func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, files Files) (stdout string, stderr string, err error) { +func (e *Docker) exec(box *config.Box, step *config.Step, req Request, dir string, files Files) (stdout string, stderr string, err error) { // limit the stdout/stderr size prog := NewProgram(step.Timeout, int64(step.NOutput)) - args := e.buildArgs(box, step, reqID, dir) + args := e.buildArgs(box, step, req, dir) if step.Stdin { // pass files to container from stdin stdin := filesReader(files) - stdout, stderr, err = prog.RunStdin(stdin, reqID, "docker", args...) + stdout, stderr, err = prog.RunStdin(stdin, req.ID, "docker", args...) } else { // pass files to container from temp directory - stdout, stderr, err = prog.Run(reqID, "docker", args...) + stdout, stderr, err = prog.Run(req.ID, "docker", args...) } if err == nil { @@ -172,11 +190,11 @@ func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, fil // inside the container is not related to the "docker run" process, // and will hang forever after the "docker run" process is killed go func() { - err = dockerKill(reqID) + err = dockerKill(req.ID) if err == nil { - logx.Debug("%s: docker kill ok", reqID) + logx.Debug("%s: docker kill ok", req.ID) } else { - logx.Log("%s: docker kill failed: %v", reqID, err) + logx.Log("%s: docker kill failed: %v", req.ID, err) } }() } @@ -202,10 +220,10 @@ func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, fil } // buildArgs prepares the arguments for the `docker` command. -func (e *Docker) buildArgs(box *config.Box, step *config.Step, name, dir string) []string { +func (e *Docker) buildArgs(box *config.Box, step *config.Step, req Request, dir string) []string { var args []string if step.Action == actionRun { - args = dockerRunArgs(box, step, name, dir) + args = dockerRunArgs(box, step, req, dir) } else if step.Action == actionExec { args = dockerExecArgs(step) } else { @@ -213,17 +231,17 @@ func (e *Docker) buildArgs(box *config.Box, step *config.Step, name, dir string) args = []string{"version"} } - command := expandVars(step.Command, name) + command := expandVars(step.Command, req.ID) args = append(args, command...) logx.Debug("%v", args) return args } // buildArgs prepares the arguments for the `docker run` command. -func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []string { +func dockerRunArgs(box *config.Box, step *config.Step, req Request, dir string) []string { args := []string{ actionRun, "--rm", - "--name", name, + "--name", req.ID, "--runtime", box.Runtime, "--cpus", strconv.Itoa(box.CPU), "--memory", fmt.Sprintf("%dm", box.Memory), @@ -255,7 +273,17 @@ func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []strin for _, lim := range box.Ulimit { args = append(args, "--ulimit", lim) } - args = append(args, box.Image) + + if step.Version != "" { + // if the version is set in the step config, use it + args = append(args, box.Image+":"+step.Version) + } else if req.Version != "" { + // if the version is set in the request, use it + args = append(args, box.Image+":"+req.Version) + } else { + // otherwise, use the latest version + args = append(args, box.Image) + } return args } diff --git a/internal/engine/docker_test.go b/internal/engine/docker_test.go index 0322ee9..cb69ff9 100644 --- a/internal/engine/docker_test.go +++ b/internal/engine/docker_test.go @@ -11,6 +11,25 @@ import ( var dockerCfg = &config.Config{ Boxes: map[string]*config.Box{ + "alpine": { + Image: "codapi/alpine", + Runtime: "runc", + Host: config.Host{ + CPU: 1, Memory: 64, Network: "none", + Volume: "%s:/sandbox:ro", + NProc: 64, + }, + }, + "go": { + Image: "codapi/go", + Runtime: "runc", + Host: config.Host{ + CPU: 1, Memory: 64, Network: "none", + Volume: "%s:/sandbox:ro", + NProc: 64, + }, + Versions: []string{"dev"}, + }, "postgresql": { Image: "codapi/postgresql", Runtime: "runc", @@ -28,9 +47,28 @@ var dockerCfg = &config.Config{ Volume: "%s:/sandbox:ro", NProc: 64, }, + Versions: []string{"dev"}, }, }, Commands: map[string]config.SandboxCommands{ + "go": map[string]*config.Command{ + "run": { + Engine: "docker", + Steps: []*config.Step{ + { + Box: "go", User: "sandbox", Action: "run", + Command: []string{"go", "build"}, + NOutput: 4096, + }, + { + Box: "alpine", Version: "latest", + User: "sandbox", Action: "run", + Command: []string{"./main"}, + NOutput: 4096, + }, + }, + }, + }, "postgresql": map[string]*config.Command{ "run": { Engine: "docker", @@ -75,9 +113,10 @@ func TestDockerRun(t *testing.T) { "docker run": {Stdout: "hello world", Stderr: "", Err: nil}, } mem := execy.Mock(commands) - engine := NewDocker(dockerCfg, "python", "run") t.Run("success", func(t *testing.T) { + mem.Clear() + engine := NewDocker(dockerCfg, "python", "run") req := Request{ ID: "http_42", Sandbox: "python", @@ -103,8 +142,88 @@ func TestDockerRun(t *testing.T) { if out.Err != nil { t.Errorf("Err: expected nil, got %v", out.Err) } + mem.MustHave(t, "codapi/python") mem.MustHave(t, "python main.py") }) + + t.Run("latest version", func(t *testing.T) { + mem.Clear() + engine := NewDocker(dockerCfg, "python", "run") + req := Request{ + ID: "http_42", + Sandbox: "python", + Command: "run", + Files: map[string]string{ + "": "print('hello world')", + }, + } + out := engine.Exec(req) + if !out.OK { + t.Error("OK: expected true") + } + mem.MustHave(t, "codapi/python") + }) + + t.Run("custom version", func(t *testing.T) { + mem.Clear() + engine := NewDocker(dockerCfg, "python", "run") + req := Request{ + ID: "http_42", + Sandbox: "python", + Version: "dev", + Command: "run", + Files: map[string]string{ + "": "print('hello world')", + }, + } + out := engine.Exec(req) + if !out.OK { + t.Error("OK: expected true") + } + mem.MustHave(t, "codapi/python:dev") + }) + + t.Run("step version", func(t *testing.T) { + mem.Clear() + engine := NewDocker(dockerCfg, "go", "run") + req := Request{ + ID: "http_42", + Sandbox: "go", + Version: "dev", + Command: "run", + Files: map[string]string{ + "": "var n = 42", + }, + } + out := engine.Exec(req) + if !out.OK { + t.Error("OK: expected true") + } + mem.MustHave(t, "codapi/go:dev") + mem.MustHave(t, "codapi/alpine:latest") + }) + + t.Run("unsupported version", func(t *testing.T) { + mem.Clear() + engine := NewDocker(dockerCfg, "python", "run") + req := Request{ + ID: "http_42", + Sandbox: "python", + Version: "42", + Command: "run", + Files: map[string]string{ + "": "print('hello world')", + }, + } + out := engine.Exec(req) + if out.OK { + t.Error("OK: expected false") + } + want := "box python does not support version 42" + if out.Stderr != want { + t.Errorf("Stderr: unexpected value: %s", out.Stderr) + } + }) } func TestDockerExec(t *testing.T) { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index d3d6584..e40eabb 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -12,13 +12,18 @@ import ( type Request struct { ID string `json:"id"` Sandbox string `json:"sandbox"` + Version string `json:"version,omitempty"` Command string `json:"command"` Files Files `json:"files"` } // GenerateID() sets a unique ID for the request. func (r *Request) GenerateID() { - r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8)) + if r.Version != "" { + r.ID = fmt.Sprintf("%s.%s_%s_%s", r.Sandbox, r.Version, r.Command, stringx.RandString(8)) + } else { + r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8)) + } } // An Execution is an output from the code execution engine. diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index e14afb2..8d54b03 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -4,9 +4,34 @@ import ( "errors" "reflect" "sort" + "strings" "testing" ) +func TestGenerateID(t *testing.T) { + t.Run("with version", func(t *testing.T) { + req := Request{ + Sandbox: "python", + Version: "dev", + Command: "run", + } + req.GenerateID() + if !strings.HasPrefix(req.ID, "python.dev_run_") { + t.Errorf("ID: unexpected prefix %s", req.ID) + } + }) + t.Run("without version", func(t *testing.T) { + req := Request{ + Sandbox: "python", + Command: "run", + } + req.GenerateID() + if !strings.HasPrefix(req.ID, "python_run_") { + t.Errorf("ID: unexpected prefix %s", req.ID) + } + }) +} + func TestExecutionError(t *testing.T) { inner := errors.New("inner error") err := NewExecutionError("failed", inner) diff --git a/internal/logx/memory.go b/internal/logx/memory.go index 7d78ace..e13eeae 100644 --- a/internal/logx/memory.go +++ b/internal/logx/memory.go @@ -53,9 +53,14 @@ func (m *Memory) MustNotHave(t *testing.T, msg string) { } } +// Clear cleares the memory. +func (m *Memory) Clear() { + m.Lines = []string{} +} + // Print prints memory lines to stdout. func (m *Memory) Print() { for _, line := range m.Lines { - fmt.Print(line) + fmt.Println(line) } }