9 Commits
0.7.0 ... 0.8.0

Author SHA1 Message Date
Anton
4468c6193c refactor: trim space in stdout and stderr 2024-03-03 20:13:26 +05:00
Anton
49dffc8f1d feat: docker stop action 2024-03-03 16:53:29 +05:00
Anton
d6945f0048 impr: backward-compatible compliance with rfc 2397 2024-02-20 12:10:21 +05:00
Anton
460493eeaa impr: comply with rfc 2397 for binary files in request 2024-02-20 11:46:06 +05:00
Stanislav Fesenko
cc3567f26e impr: unlimited replacements when expanding command vars (#8) 2024-02-20 11:30:22 +05:00
Anton
4218065e0e build: build on pull request 2024-02-20 11:22:19 +05:00
Anton
12f7e25a85 impr: request files have priority over box files 2024-02-11 18:23:37 +05:00
Anton
b312100bbc doc: codapi 0.7.1 2024-01-19 21:01:51 +05:00
Anton
bca91d71e5 fix: prevent directory traversal attack when writing request files 2024-01-19 20:47:09 +05:00
11 changed files with 479 additions and 74 deletions

View File

@@ -7,6 +7,12 @@ on:
- "docs/**"
- Makefile
- README.md
pull_request:
branches: [main]
paths-ignore:
- "docs/**"
- Makefile
- README.md
workflow_dispatch:
defaults:

View File

@@ -28,10 +28,10 @@ docker run hello-world
```sh
cd /opt/codapi
curl -L -O "https://github.com/nalgeon/codapi/releases/download/0.6.0/codapi_0.6.0_linux_amd64.tar.gz"
tar xvzf codapi_0.6.0_linux_amd64.tar.gz
curl -L -O "https://github.com/nalgeon/codapi/releases/download/0.7.1/codapi_0.7.1_linux_amd64.tar.gz"
tar xvzf codapi_0.7.1_linux_amd64.tar.gz
chmod +x codapi
rm -f codapi_0.6.0_linux_amd64.tar.gz
rm -f codapi_0.7.1_linux_amd64.tar.gz
```
5. Build Docker images (as codapi):

View File

@@ -99,6 +99,7 @@ type Step struct {
Version string `json:"version"`
User string `json:"user"`
Action string `json:"action"`
Detach bool `json:"detach"`
Stdin bool `json:"stdin"`
Command []string `json:"command"`
Timeout int `json:"timeout"`

View File

@@ -8,7 +8,6 @@ import (
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
@@ -24,6 +23,7 @@ var killTimeout = 5 * time.Second
const (
actionRun = "run"
actionExec = "exec"
actionStop = "stop"
)
// A Docker engine executes a specific sandbox command
@@ -54,7 +54,10 @@ func (e *Docker) Exec(req Request) Execution {
if e.cmd.Entry != "" {
// write request files to the temp directory
err = e.writeFiles(dir, req.Files)
if err != nil {
var argErr ArgumentError
if errors.As(err, &argErr) {
return Fail(req.ID, err)
} else if err != nil {
err = NewExecutionError("write files to temp dir", err)
return Fail(req.ID, err)
}
@@ -123,9 +126,9 @@ func (e *Docker) execStep(step *config.Step, req Request, dir string, files File
// getBox selects an appropriate box for the step (if any).
func (e *Docker) getBox(step *config.Step, req Request) (*config.Box, error) {
if step.Action == actionExec {
// exec steps use existing instances
// and do not spin up new boxes
if step.Action != actionRun {
// steps other than "run" use existing containers
// and do not spin up new ones
return nil, nil
}
var boxName string
@@ -171,7 +174,12 @@ func (e *Docker) writeFiles(dir string, files Files) error {
if name == "" {
name = e.cmd.Entry
}
path := filepath.Join(dir, name)
var path string
path, err = fileio.JoinDir(dir, name)
if err != nil {
err = NewArgumentError(fmt.Sprintf("files[%s]", name), err)
return false
}
err = fileio.WriteFile(path, content, 0444)
return err == nil
})
@@ -237,11 +245,14 @@ func (e *Docker) exec(box *config.Box, step *config.Step, req Request, dir strin
// buildArgs prepares the arguments for the `docker` command.
func (e *Docker) buildArgs(box *config.Box, step *config.Step, req Request, dir string) []string {
var args []string
if step.Action == actionRun {
switch step.Action {
case actionRun:
args = dockerRunArgs(box, step, req, dir)
} else if step.Action == actionExec {
args = dockerExecArgs(step)
} else {
case actionExec:
args = dockerExecArgs(step, req)
case actionStop:
args = dockerStopArgs(step, req)
default:
// should never happen if the config is valid
args = []string{"version"}
}
@@ -264,12 +275,15 @@ func dockerRunArgs(box *config.Box, step *config.Step, req Request, dir string)
"--pids-limit", strconv.Itoa(box.NProc),
"--user", step.User,
}
if !box.Writable {
args = append(args, "--read-only")
if step.Detach {
args = append(args, "--detach")
}
if step.Stdin {
args = append(args, "--interactive")
}
if !box.Writable {
args = append(args, "--read-only")
}
if box.Storage != "" {
args = append(args, "--storage-opt", fmt.Sprintf("size=%s", box.Storage))
}
@@ -293,14 +307,23 @@ func dockerRunArgs(box *config.Box, step *config.Step, req Request, dir string)
}
// dockerExecArgs prepares the arguments for the `docker exec` command.
func dockerExecArgs(step *config.Step) []string {
func dockerExecArgs(step *config.Step, req Request) []string {
// :name means executing in the container passed in the request
box := strings.Replace(step.Box, ":name", req.ID, 1)
return []string{
actionExec, "--interactive",
"--user", step.User,
step.Box,
box,
}
}
// dockerStopArgs prepares the arguments for the `docker stop` command.
func dockerStopArgs(step *config.Step, req Request) []string {
// :name means executing in the container passed in the request
box := strings.Replace(step.Box, ":name", req.ID, 1)
return []string{actionStop, box}
}
// filesReader creates a reader over an in-memory collection of files.
func filesReader(files Files) io.Reader {
var input strings.Builder
@@ -316,7 +339,7 @@ func expandVars(command []string, name string) []string {
expanded := make([]string, len(command))
copy(expanded, command)
for i, cmd := range expanded {
expanded[i] = strings.Replace(cmd, ":name", name, 1)
expanded[i] = strings.Replace(cmd, ":name", name, -1)
}
return expanded
}

View File

@@ -1,6 +1,7 @@
package engine
import (
"fmt"
"strings"
"testing"
@@ -58,6 +59,27 @@ var dockerCfg = &config.Config{
},
},
Commands: map[string]config.SandboxCommands{
"alpine": map[string]*config.Command{
"echo": {
Engine: "docker",
Before: &config.Step{
Box: "alpine", User: "sandbox", Action: "run", Detach: true,
Command: []string{"echo", "before"},
NOutput: 4096,
},
Steps: []*config.Step{
{
Box: ":name", User: "sandbox", Action: "exec",
Command: []string{"sh", "main.sh"},
NOutput: 4096,
},
},
After: &config.Step{
Box: ":name", User: "sandbox", Action: "stop",
NOutput: 4096,
},
},
},
"go": map[string]*config.Command{
"run": {
Engine: "docker",
@@ -231,6 +253,29 @@ func TestDockerRun(t *testing.T) {
t.Errorf("Stderr: unexpected value: %s", out.Stderr)
}
})
t.Run("directory traversal attack", func(t *testing.T) {
mem.Clear()
const fileName = "../../opt/codapi/codapi"
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
fileName: "hehe",
},
}
out := engine.Exec(req)
if out.OK {
t.Error("OK: expected false")
}
want := fmt.Sprintf("files[%s]: invalid name", fileName)
if out.Stderr != want {
t.Errorf("Stderr: unexpected value: %s", out.Stderr)
}
})
}
func TestDockerExec(t *testing.T) {
@@ -273,11 +318,55 @@ func TestDockerExec(t *testing.T) {
})
}
func TestDockerStop(t *testing.T) {
logx.Mock()
commands := map[string]execy.CmdOut{
"docker run": {Stdout: "c958ff2", Stderr: "", Err: nil},
"docker exec": {Stdout: "hello", Stderr: "", Err: nil},
"docker stop": {Stdout: "alpine_42", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
engine := NewDocker(dockerCfg, "alpine", "echo")
t.Run("success", func(t *testing.T) {
req := Request{
ID: "alpine_42",
Sandbox: "alpine",
Command: "echo",
Files: map[string]string{
"": "echo hello",
},
}
out := engine.Exec(req)
if out.ID != req.ID {
t.Errorf("ID: expected %s, got %s", req.ID, out.ID)
}
if !out.OK {
t.Error("OK: expected true")
}
want := "hello"
if out.Stdout != want {
t.Errorf("Stdout: expected %q, got %q", want, out.Stdout)
}
if out.Stderr != "" {
t.Errorf("Stderr: expected %q, got %q", "", out.Stdout)
}
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "docker run --rm --name alpine_42", "--detach")
mem.MustHave(t, "docker exec --interactive --user sandbox alpine_42 sh main.sh")
mem.MustHave(t, "docker stop alpine_42")
})
}
func Test_expandVars(t *testing.T) {
const name = "codapi_01"
commands := map[string]string{
"python main.py": "python main.py",
"sh create.sh :name": "sh create.sh " + name,
"python main.py": "python main.py",
"sh create.sh :name": "sh create.sh " + name,
"sh copy.sh :name new-:name": "sh copy.sh " + name + " new-" + name,
}
for cmd, want := range commands {
src := strings.Fields(cmd)

View File

@@ -62,6 +62,25 @@ func (err ExecutionError) Unwrap() error {
return err.inner
}
// An ArgumentError is returned if code execution failed
// due to the invalid value of the request agrument.
type ArgumentError struct {
name string
reason error
}
func NewArgumentError(name string, reason error) ArgumentError {
return ArgumentError{name: name, reason: reason}
}
func (err ArgumentError) Error() string {
return err.name + ": " + err.reason.Error()
}
func (err ArgumentError) Unwrap() error {
return err.reason
}
// Files are a collection of files to be executed by the engine.
type Files map[string]string

View File

@@ -48,7 +48,7 @@ func (p *Program) RunStdin(stdin io.Reader, id, name string, arg ...string) (std
cmd.Stdout = LimitWriter(&cmdout, p.nOutput)
cmd.Stderr = LimitWriter(&cmderr, p.nOutput)
err = execy.Run(cmd)
stdout = cmdout.String()
stderr = cmderr.String()
stdout = strings.TrimSpace(cmdout.String())
stderr = strings.TrimSpace(cmderr.String())
return
}

View File

@@ -12,8 +12,17 @@ import (
"strings"
)
// Exists checks if the specified path exists.
func Exists(path string) bool {
_, err := os.Stat(path)
// we need a double negation here, because
// errors.Is(err, os.ErrExist)
// does not work
return !errors.Is(err, os.ErrNotExist)
}
// CopyFile copies all files matching the pattern
// to the destination directory.
// to the destination directory. Does not overwrite existing file.
func CopyFiles(pattern string, dstDir string, perm fs.FileMode) error {
matches, err := filepath.Glob(pattern)
if err != nil {
@@ -28,6 +37,10 @@ func CopyFiles(pattern string, dstDir string, perm fs.FileMode) error {
defer src.Close()
dstFile := filepath.Join(dstDir, filepath.Base(match))
if Exists(dstFile) {
continue
}
dst, err := os.OpenFile(dstFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
if err != nil {
return err
@@ -62,23 +75,61 @@ func ReadJson[T any](path string) (T, error) {
// e.g. data:application/octet-stream;base64,MTIz
func WriteFile(path, content string, perm fs.FileMode) (err error) {
var data []byte
if strings.HasPrefix(content, "data:") {
// data-url encoded file
_, encoded, found := strings.Cut(content, ",")
if !found {
return errors.New("invalid data-url encoding")
}
data, err = base64.StdEncoding.DecodeString(encoded)
if err != nil {
return err
}
} else {
if !strings.HasPrefix(content, "data:") {
// text file
data = []byte(content)
return os.WriteFile(path, data, perm)
}
// data-url encoded file
meta, encoded, found := strings.Cut(content, ",")
if !found {
return errors.New("invalid data-url encoding")
}
if !strings.HasSuffix(meta, "base64") {
// no need to decode
data = []byte(encoded)
return os.WriteFile(path, data, perm)
}
// decode base64-encoded data
data, err = base64.StdEncoding.DecodeString(encoded)
if err != nil {
return err
}
return os.WriteFile(path, data, perm)
}
// JoinDir joins a directory path with a relative file path,
// making sure that the resulting path is still inside the directory.
// Returns an error otherwise.
func JoinDir(dir string, name string) (string, error) {
if dir == "" {
return "", errors.New("invalid dir")
}
cleanName := filepath.Clean(name)
if cleanName == "" {
return "", errors.New("invalid name")
}
if cleanName == "." || cleanName == "/" || filepath.IsAbs(cleanName) {
return "", errors.New("invalid name")
}
path := filepath.Join(dir, cleanName)
dirPrefix := filepath.Clean(dir)
if dirPrefix != "/" {
dirPrefix += string(os.PathSeparator)
}
if !strings.HasPrefix(path, dirPrefix) {
return "", errors.New("invalid name")
}
return path, nil
}
// MkdirTemp creates a new temporary directory with given permissions
// and returns the pathname of the new directory.
func MkdirTemp(perm fs.FileMode) (string, error) {

View File

@@ -8,57 +8,119 @@ import (
"testing"
)
func TestExists(t *testing.T) {
t.Run("exists", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "file.txt")
err := os.WriteFile(path, []byte{1, 2, 3}, 0444)
if err != nil {
t.Fatal(err)
}
if !Exists(path) {
t.Fatalf("Exists: %s does not exist", filepath.Base(path))
}
})
t.Run("does not exist", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "file.txt")
if Exists(path) {
t.Fatalf("Exists: %s should not exist", filepath.Base(path))
}
})
}
func TestCopyFiles(t *testing.T) {
// Create a temporary directory for testing
// create a temporary directory for testing
srcDir, err := os.MkdirTemp("", "src")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(srcDir)
// Create a source file
// create a source file
srcFile := filepath.Join(srcDir, "source.txt")
err = os.WriteFile(srcFile, []byte("test data"), 0644)
if err != nil {
t.Fatal(err)
}
// Specify the destination directory
// specify the destination directory
dstDir, err := os.MkdirTemp("", "dst")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dstDir)
// Call the CopyFiles function
const perm = fs.FileMode(0444)
pattern := filepath.Join(srcDir, "*.txt")
err = CopyFiles(pattern, dstDir, perm)
if err != nil {
t.Fatal(err)
}
t.Run("copy", func(t *testing.T) {
// call the CopyFiles function
const perm = fs.FileMode(0444)
pattern := filepath.Join(srcDir, "*.txt")
err = CopyFiles(pattern, dstDir, perm)
if err != nil {
t.Fatal(err)
}
// Verify that the file was copied correctly
dstFile := filepath.Join(dstDir, "source.txt")
fileInfo, err := os.Stat(dstFile)
if err != nil {
t.Fatalf("file not copied: %s", err)
}
if fileInfo.Mode() != perm {
t.Errorf("unexpected file permissions: got %v, want %v", fileInfo.Mode(), perm)
}
// verify that the file was copied correctly
dstFile := filepath.Join(dstDir, "source.txt")
fileInfo, err := os.Stat(dstFile)
if err != nil {
t.Fatalf("file not copied: %s", err)
}
if fileInfo.Mode() != perm {
t.Errorf("unexpected file permissions: got %v, want %v", fileInfo.Mode(), perm)
}
// Read the contents of the copied file
data, err := os.ReadFile(dstFile)
if err != nil {
t.Fatal(err)
}
// read the contents of the copied file
data, err := os.ReadFile(dstFile)
if err != nil {
t.Fatal(err)
}
// Verify the contents of the copied file
expected := []byte("test data")
if string(data) != string(expected) {
t.Errorf("unexpected file content: got %q, want %q", data, expected)
}
// verify the contents of the copied file
expected := []byte("test data")
if string(data) != string(expected) {
t.Errorf("unexpected file content: got %q, want %q", data, expected)
}
})
t.Run("skip existing", func(t *testing.T) {
// existing file in the destination dir
path := filepath.Join(dstDir, "existing.txt")
err := os.WriteFile(path, []byte("v1"), 0444)
if err != nil {
t.Fatal(err)
}
// same file in the source dir
path = filepath.Join(srcDir, "existing.txt")
err = os.WriteFile(path, []byte("v2"), 0444)
if err != nil {
t.Fatal(err)
}
// copy files
pattern := filepath.Join(srcDir, "*.txt")
err = CopyFiles(pattern, dstDir, 0444)
if err != nil {
t.Fatal(err)
}
// verify that the new file was copied correctly
newFile := filepath.Join(dstDir, "source.txt")
_, err = os.Stat(newFile)
if err != nil {
t.Fatalf("new file not copied: %s", err)
}
// verify that the existing file remained unchanged
existFile := filepath.Join(dstDir, "existing.txt")
data, err := os.ReadFile(existFile)
if err != nil {
t.Fatal(err)
}
expected := []byte("v1")
if string(data) != string(expected) {
t.Error("existing file got overwritten")
}
})
}
func TestReadJson(t *testing.T) {
@@ -111,8 +173,8 @@ func TestWriteFile(t *testing.T) {
}
})
t.Run("binary", func(t *testing.T) {
path := filepath.Join(dir, "data.bin")
t.Run("data-octet-stream", func(t *testing.T) {
path := filepath.Join(dir, "data-1.bin")
err = WriteFile(path, "data:application/octet-stream;base64,MTIz", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
@@ -127,6 +189,38 @@ func TestWriteFile(t *testing.T) {
}
})
t.Run("data-base64", func(t *testing.T) {
path := filepath.Join(dir, "data-2.bin")
err = WriteFile(path, "data:;base64,MTIz", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("123")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("data-text-plain", func(t *testing.T) {
path := filepath.Join(dir, "data-3.bin")
err = WriteFile(path, "data:text/plain;,123", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("123")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("perm", func(t *testing.T) {
const perm = 0444
path := filepath.Join(dir, "perm.txt")
@@ -153,13 +247,107 @@ func TestWriteFile(t *testing.T) {
t.Run("invalid binary value", func(t *testing.T) {
path := filepath.Join(dir, "data.bin")
err = WriteFile(path, "data:application/octet-stream;base64,12345", 0444)
err = WriteFile(path, "data:;base64,12345", 0444)
if err == nil {
t.Fatal("expected error, got nil")
}
})
}
func TestJoinDir(t *testing.T) {
tests := []struct {
name string
dir string
filename string
want string
wantErr bool
}{
{
name: "regular join",
dir: "/home/user",
filename: "docs/report.txt",
want: "/home/user/docs/report.txt",
wantErr: false,
},
{
name: "join with dot",
dir: "/home/user",
filename: ".",
want: "",
wantErr: true,
},
{
name: "join with absolute path",
dir: "/home/user",
filename: "/etc/passwd",
want: "",
wantErr: true,
},
{
name: "join with parent directory",
dir: "/home/user",
filename: "../user2/docs/report.txt",
want: "",
wantErr: true,
},
{
name: "empty directory",
dir: "",
filename: "report.txt",
want: "",
wantErr: true,
},
{
name: "empty filename",
dir: "/home/user",
filename: "",
want: "",
wantErr: true,
},
{
name: "directory with trailing slash",
dir: "/home/user/",
filename: "docs/report.txt",
want: "/home/user/docs/report.txt",
wantErr: false,
},
{
name: "filename with leading slash",
dir: "/home/user",
filename: "/docs/report.txt",
want: "",
wantErr: true,
},
{
name: "root directory",
dir: "/",
filename: "report.txt",
want: "/report.txt",
wantErr: false,
},
{
name: "dot dot slash filename",
dir: "/home/user",
filename: "..",
want: "",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := JoinDir(tt.dir, tt.filename)
if (err != nil) != tt.wantErr {
t.Errorf("JoinDir() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("JoinDir() = %v, want %v", got, tt.want)
}
})
}
}
func TestMkdirTemp(t *testing.T) {
t.Run("default permissions", func(t *testing.T) {
const perm = 0755

View File

@@ -30,9 +30,16 @@ func (m *Memory) WriteString(s string) {
}
// Has returns true if the memory has the message.
func (m *Memory) Has(msg string) bool {
func (m *Memory) Has(message ...string) bool {
for _, line := range m.Lines {
if strings.Contains(line, msg) {
containsAll := true
for _, part := range message {
if !strings.Contains(line, part) {
containsAll = false
break
}
}
if containsAll {
return true
}
}
@@ -40,16 +47,18 @@ func (m *Memory) Has(msg string) bool {
}
// MustHave checks if the memory has the message.
func (m *Memory) MustHave(t *testing.T, msg string) {
if !m.Has(msg) {
t.Errorf("%s must have: %s", m.Name, msg)
// If the message consists of several parts,
// they must all be in the same memory line.
func (m *Memory) MustHave(t *testing.T, message ...string) {
if !m.Has(message...) {
t.Errorf("%s must have: %v", m.Name, message)
}
}
// MustNotHave checks if the memory does not have the message.
func (m *Memory) MustNotHave(t *testing.T, msg string) {
if m.Has(msg) {
t.Errorf("%s must NOT have: %s", m.Name, msg)
func (m *Memory) MustNotHave(t *testing.T, message ...string) {
if m.Has(message...) {
t.Errorf("%s must NOT have: %v", m.Name, message)
}
}

View File

@@ -40,4 +40,23 @@ func TestMemory_Has(t *testing.T) {
if !mem.Has("hello world") {
t.Error("Has: unexpected false")
}
_, _ = mem.Write([]byte("one two three four"))
if !mem.Has("one two") {
t.Error("Has: one two: unexpected false")
}
if !mem.Has("two three") {
t.Error("Has: two three: unexpected false")
}
if mem.Has("one three") {
t.Error("Has: one three: unexpected true")
}
if !mem.Has("one", "three") {
t.Error("Has: [one three]: unexpected false")
}
if !mem.Has("one", "three", "four") {
t.Error("Has: [one three four]: unexpected false")
}
if !mem.Has("four", "three") {
t.Error("Has: [four three]: unexpected false")
}
}