JoBoEVD/codapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support different versions of the same box

Browse Source
This commit is contained in:
Anton
2023-12-20 22:43:07 +05:00
parent 162ca55092
commit ade821ff61
7 changed files with 213 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -1,3 +1,3 @@
module github.com/nalgeon/codapi
go 1.20
go 1.21

2
internal/config/config.go
View File

@@ -59,6 +59,7 @@ type Box struct {
Runtime string `json:"runtime"`
Host
Versions []string `json:"versions"`
Files []string `json:"files"`
}
@@ -96,6 +97,7 @@ type Command struct {
// A Step describes a single step of a command.
type Step struct {
Box string `json:"box"`
Version string `json:"version"`
User string `json:"user"`
Action string `json:"action"`
Stdin bool `json:"stdin"`

74
internal/engine/docker.go
View File

@@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path/filepath"
"slices"
"strconv"
"strings"
"time"
@@ -62,7 +63,7 @@ func (e *Docker) Exec(req Request) Execution {
// initialization step
if e.cmd.Before != nil {
out := e.execStep(e.cmd.Before, req.ID, dir, nil)
out := e.execStep(e.cmd.Before, req, dir, nil)
if !out.OK {
return out
}
@@ -70,14 +71,14 @@ func (e *Docker) Exec(req Request) Execution {
// the first step is required
first, rest := e.cmd.Steps[0], e.cmd.Steps[1:]
out := e.execStep(first, req.ID, dir, req.Files)
out := e.execStep(first, req, dir, req.Files)
// the rest are optional
if out.OK && len(rest) > 0 {
// each step operates on the results of the previous one,
// without using the source files - hence `nil` instead of `files`
for _, step := range rest {
out = e.execStep(step, req.ID, dir, nil)
out = e.execStep(step, req, dir, nil)
if !out.OK {
break
}
@@ -86,7 +87,7 @@ func (e *Docker) Exec(req Request) Execution {
// cleanup step
if e.cmd.After != nil {
afterOut := e.execStep(e.cmd.After, req.ID, dir, nil)
afterOut := e.execStep(e.cmd.After, req, dir, nil)
if out.OK && !afterOut.OK {
return afterOut
}
@@ -96,27 +97,44 @@ func (e *Docker) Exec(req Request) Execution {
}
// execStep executes a step using the docker container.
func (e *Docker) execStep(step *config.Step, reqID, dir string, files Files) Execution {
func (e *Docker) execStep(step *config.Step, req Request, dir string, files Files) Execution {
box := e.cfg.Boxes[step.Box]
err := e.copyFiles(box, dir)
err := e.validateVersion(box, step, req)
if err != nil {
err = NewExecutionError("copy files to temp dir", err)
return Fail(reqID, err)
return Fail(req.ID, err)
}
stdout, stderr, err := e.exec(box, step, reqID, dir, files)
err = e.copyFiles(box, dir)
if err != nil {
return Fail(reqID, err)
err = NewExecutionError("copy files to temp dir", err)
return Fail(req.ID, err)
}
stdout, stderr, err := e.exec(box, step, req, dir, files)
if err != nil {
return Fail(req.ID, err)
}
return Execution{
ID: reqID,
ID: req.ID,
OK: true,
Stdout: stdout,
Stderr: stderr,
}
}
func (e *Docker) validateVersion(box *config.Box, step *config.Step, req Request) error {
// If the version is set in the step config, use it.
// If the version isn't set in the request, use the latest one.
// Otherwise, check that the version in the request is supported
// according to the box config.
if step.Version == "" && req.Version != "" && !slices.Contains(box.Versions, req.Version) {
err := fmt.Errorf("box %s does not support version %s", step.Box, req.Version)
return err
}
return nil
}
// copyFiles copies box files to the temporary directory.
func (e *Docker) copyFiles(box *config.Box, dir string) error {
if box == nil || len(box.Files) == 0 {
@@ -147,18 +165,18 @@ func (e *Docker) writeFiles(dir string, files Files) error {
// exec executes the step in the docker container
// using the files from in the temporary directory.
func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, files Files) (stdout string, stderr string, err error) {
func (e *Docker) exec(box *config.Box, step *config.Step, req Request, dir string, files Files) (stdout string, stderr string, err error) {
// limit the stdout/stderr size
prog := NewProgram(step.Timeout, int64(step.NOutput))
args := e.buildArgs(box, step, reqID, dir)
args := e.buildArgs(box, step, req, dir)
if step.Stdin {
// pass files to container from stdin
stdin := filesReader(files)
stdout, stderr, err = prog.RunStdin(stdin, reqID, "docker", args...)
stdout, stderr, err = prog.RunStdin(stdin, req.ID, "docker", args...)
} else {
// pass files to container from temp directory
stdout, stderr, err = prog.Run(reqID, "docker", args...)
stdout, stderr, err = prog.Run(req.ID, "docker", args...)
}
if err == nil {
@@ -172,11 +190,11 @@ func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, fil
// inside the container is not related to the "docker run" process,
// and will hang forever after the "docker run" process is killed
go func() {
err = dockerKill(reqID)
err = dockerKill(req.ID)
if err == nil {
logx.Debug("%s: docker kill ok", reqID)
logx.Debug("%s: docker kill ok", req.ID)
} else {
logx.Log("%s: docker kill failed: %v", reqID, err)
logx.Log("%s: docker kill failed: %v", req.ID, err)
}
}()
}
@@ -202,10 +220,10 @@ func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, fil
}
// buildArgs prepares the arguments for the `docker` command.
func (e *Docker) buildArgs(box *config.Box, step *config.Step, name, dir string) []string {
func (e *Docker) buildArgs(box *config.Box, step *config.Step, req Request, dir string) []string {
var args []string
if step.Action == actionRun {
args = dockerRunArgs(box, step, name, dir)
args = dockerRunArgs(box, step, req, dir)
} else if step.Action == actionExec {
args = dockerExecArgs(step)
} else {
@@ -213,17 +231,17 @@ func (e *Docker) buildArgs(box *config.Box, step *config.Step, name, dir string)
args = []string{"version"}
}
command := expandVars(step.Command, name)
command := expandVars(step.Command, req.ID)
args = append(args, command...)
logx.Debug("%v", args)
return args
}
// buildArgs prepares the arguments for the `docker run` command.
func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []string {
func dockerRunArgs(box *config.Box, step *config.Step, req Request, dir string) []string {
args := []string{
actionRun, "--rm",
"--name", name,
"--name", req.ID,
"--runtime", box.Runtime,
"--cpus", strconv.Itoa(box.CPU),
"--memory", fmt.Sprintf("%dm", box.Memory),
@@ -255,7 +273,17 @@ func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []strin
for _, lim := range box.Ulimit {
args = append(args, "--ulimit", lim)
}
if step.Version != "" {
// if the version is set in the step config, use it
args = append(args, box.Image+":"+step.Version)
} else if req.Version != "" {
// if the version is set in the request, use it
args = append(args, box.Image+":"+req.Version)
} else {
// otherwise, use the latest version
args = append(args, box.Image)
}
return args
}

121
internal/engine/docker_test.go
View File

@@ -11,6 +11,25 @@ import (
var dockerCfg = &config.Config{
Boxes: map[string]*config.Box{
"alpine": {
Image: "codapi/alpine",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
"go": {
Image: "codapi/go",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
Versions: []string{"dev"},
},
"postgresql": {
Image: "codapi/postgresql",
Runtime: "runc",
@@ -28,9 +47,28 @@ var dockerCfg = &config.Config{
Volume: "%s:/sandbox:ro",
NProc: 64,
},
Versions: []string{"dev"},
},
},
Commands: map[string]config.SandboxCommands{
"go": map[string]*config.Command{
"run": {
Engine: "docker",
Steps: []*config.Step{
{
Box: "go", User: "sandbox", Action: "run",
Command: []string{"go", "build"},
NOutput: 4096,
},
{
Box: "alpine", Version: "latest",
User: "sandbox", Action: "run",
Command: []string{"./main"},
NOutput: 4096,
},
},
},
},
"postgresql": map[string]*config.Command{
"run": {
Engine: "docker",
@@ -75,9 +113,10 @@ func TestDockerRun(t *testing.T) {
"docker run": {Stdout: "hello world", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
engine := NewDocker(dockerCfg, "python", "run")
t.Run("success", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
@@ -103,8 +142,88 @@ func TestDockerRun(t *testing.T) {
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "codapi/python")
mem.MustHave(t, "python main.py")
})
t.Run("latest version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/python")
})
t.Run("custom version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Version: "dev",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/python:dev")
})
t.Run("step version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "go", "run")
req := Request{
ID: "http_42",
Sandbox: "go",
Version: "dev",
Command: "run",
Files: map[string]string{
"": "var n = 42",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/go:dev")
mem.MustHave(t, "codapi/alpine:latest")
})
t.Run("unsupported version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Version: "42",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if out.OK {
t.Error("OK: expected false")
}
want := "box python does not support version 42"
if out.Stderr != want {
t.Errorf("Stderr: unexpected value: %s", out.Stderr)
}
})
}
func TestDockerExec(t *testing.T) {

5
internal/engine/engine.go
View File

@@ -12,14 +12,19 @@ import (
type Request struct {
ID string `json:"id"`
Sandbox string `json:"sandbox"`
Version string `json:"version,omitempty"`
Command string `json:"command"`
Files Files `json:"files"`
}
// GenerateID() sets a unique ID for the request.
func (r *Request) GenerateID() {
if r.Version != "" {
r.ID = fmt.Sprintf("%s.%s_%s_%s", r.Sandbox, r.Version, r.Command, stringx.RandString(8))
} else {
r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8))
}
}
// An Execution is an output from the code execution engine.
type Execution struct {

25
internal/engine/engine_test.go
View File

@@ -4,9 +4,34 @@ import (
"errors"
"reflect"
"sort"
"strings"
"testing"
)
func TestGenerateID(t *testing.T) {
t.Run("with version", func(t *testing.T) {
req := Request{
Sandbox: "python",
Version: "dev",
Command: "run",
}
req.GenerateID()
if !strings.HasPrefix(req.ID, "python.dev_run_") {
t.Errorf("ID: unexpected prefix %s", req.ID)
}
})
t.Run("without version", func(t *testing.T) {
req := Request{
Sandbox: "python",
Command: "run",
}
req.GenerateID()
if !strings.HasPrefix(req.ID, "python_run_") {
t.Errorf("ID: unexpected prefix %s", req.ID)
}
})
}
func TestExecutionError(t *testing.T) {
inner := errors.New("inner error")
err := NewExecutionError("failed", inner)

7
internal/logx/memory.go
View File

@@ -53,9 +53,14 @@ func (m *Memory) MustNotHave(t *testing.T, msg string) {
}
}
// Clear cleares the memory.
func (m *Memory) Clear() {
m.Lines = []string{}
}
// Print prints memory lines to stdout.
func (m *Memory) Print() {
for _, line := range m.Lines {
fmt.Print(line)
fmt.Println(line)
}
}