27 Commits
0.6.0 ... main

Author SHA1 Message Date
Anton
4411aef6fc doc: api - fix broken link 2024-06-13 03:04:31 +05:00
Anton
2a95a86a2f doc: api 2024-06-13 03:03:31 +05:00
Anton
e0b733b18c doc: readme - funding 2024-06-13 03:01:09 +05:00
Anton Zhiyanov
d7e55541c3 doc: funding 2024-06-08 23:43:48 +05:00
Andreas Deininger
f250034cd4 infr: bump github action workflows (#11) 2024-03-28 09:12:01 +05:00
Andreas Deininger
38dfe1a380 fix: typos in comments (#10) 2024-03-28 09:10:03 +05:00
Anton
e6c7b053f9 doc: codapi 0.8.0 2024-03-11 00:12:07 +05:00
Anton
4468c6193c refactor: trim space in stdout and stderr 2024-03-03 20:13:26 +05:00
Anton
49dffc8f1d feat: docker stop action 2024-03-03 16:53:29 +05:00
Anton
d6945f0048 impr: backward-compatible compliance with rfc 2397 2024-02-20 12:10:21 +05:00
Anton
460493eeaa impr: comply with rfc 2397 for binary files in request 2024-02-20 11:46:06 +05:00
Stanislav Fesenko
cc3567f26e impr: unlimited replacements when expanding command vars (#8) 2024-02-20 11:30:22 +05:00
Anton
4218065e0e build: build on pull request 2024-02-20 11:22:19 +05:00
Anton
12f7e25a85 impr: request files have priority over box files 2024-02-11 18:23:37 +05:00
Anton
b312100bbc doc: codapi 0.7.1 2024-01-19 21:01:51 +05:00
Anton
bca91d71e5 fix: prevent directory traversal attack when writing request files 2024-01-19 20:47:09 +05:00
Anton
02473a2b61 impr: goreleaser-compatible version, commit and date 2024-01-18 00:34:49 +05:00
Anton
8178918c6e fix: engine - set 777 permissions for temp dir (#6) 2024-01-18 00:15:33 +05:00
Anton
314987ae43 impr: engine - copy files with 444 permissions 2024-01-17 23:27:59 +05:00
Anton
3fd59120f5 impr: explicit versioned boxes 2023-12-21 16:46:31 +05:00
Anton
ade821ff61 feat: support different versions of the same box 2023-12-20 22:43:07 +05:00
Anton
162ca55092 fix: access-control-allow-methods header naming and value 2023-12-20 11:12:06 +05:00
Anton
69d022c061 doc: readme 2023-12-18 00:59:53 +05:00
Anton
0385eadc08 impr: support binary files 2023-12-12 01:56:43 +05:00
Anton
81240dd80f doc: readme 2023-12-11 02:49:51 +05:00
Anton
c2ed6f1bcb doc: codapi in action 2023-12-05 22:15:05 +05:00
Anton
db61c053b1 doc: bump version 2023-12-05 13:47:15 +05:00
22 changed files with 920 additions and 160 deletions

1
.github/FUNDING.yml vendored Normal file
View File

@@ -0,0 +1 @@
github: [nalgeon]

View File

@@ -7,6 +7,12 @@ on:
- "docs/**"
- Makefile
- README.md
pull_request:
branches: [main]
paths-ignore:
- "docs/**"
- Makefile
- README.md
workflow_dispatch:
defaults:
@@ -18,10 +24,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: "stable"
@@ -29,7 +35,7 @@ jobs:
run: make test build
- name: Upload artifact
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: codapi
path: build/codapi

View File

@@ -14,15 +14,15 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: "stable"
- name: Release and publish
uses: goreleaser/goreleaser-action@v4
uses: goreleaser/goreleaser-action@v5
with:
args: release --clean
env:

View File

@@ -6,9 +6,7 @@ build_rev := "main"
ifneq ($(wildcard .git),)
build_rev := $(shell git rev-parse --short HEAD)
endif
build_date := $(shell date -u '+%Y%m%d')
version := $(build_date):$(build_rev)
build_date := $(shell date -u '+%Y-%m-%dT%H:%M:%S')
setup:
@go mod download
@@ -24,7 +22,7 @@ test:
build:
@go build -ldflags "-X main.Version=$(version)" -o build/codapi -v cmd/main.go
@go build -ldflags "-X main.commit=$(build_rev) -X main.date=$(build_date)" -o build/codapi -v cmd/main.go
run:
@./build/codapi

View File

@@ -1,8 +1,8 @@
# Embeddable code playgrounds
# Interactive code examples
_for education, documentation, and fun_ 🎉
_for documentation, education and fun_ 🎉
Codapi is a platform for embedding interactive code snippets directly into your product documentation, online course, or blog post.
Codapi is a platform for embedding interactive code snippets directly into your product documentation, online course or blog post.
```
┌───────────────────────────────┐
@@ -21,62 +21,22 @@ Codapi manages sandboxes (isolated execution environments) and provides an API t
Highlights:
- Custom sandboxes for any programming language, database, or software.
- Available as a cloud service and as a self-hosted version.
- Open source. Uses the permissive Apache-2.0 license.
- Automatically converts static code examples into mini-playgrounds.
- Lightweight and easy to integrate.
- Sandboxes for any programming language, database, or software.
- Open source. Uses the permissive Apache-2.0 license.
Learn more at [codapi.org](https://codapi.org/)
For an introduction to Codapi, see this post: [Interactive code examples for fun and profit](https://antonz.org/code-examples/).
## Installation
See [Installing Codapi](docs/install.md) for details.
## Usage (API)
## Usage
Call `/v1/exec` to run the code in a sandbox:
See [API](docs/api.md) to run sandboxed code using the HTTP API.
```http
POST https://api.codapi.org/v1/exec
content-type: application/json
{
"sandbox": "python",
"command": "run",
"files": {
"": "print('hello world')"
}
}
```
`sandbox` is the name of the pre-configured sandbox, and `command` is the name of a command supported by that sandbox. See [Adding a sandbox](docs/add-sandbox.md) for details on how to add a new sandbox.
`files` is a map, where the key is a filename and the value is its contents. When executing a single file, it should either be named as the `command` expects, or be an empty string (as in the example above).
Response:
```http
HTTP/1.1 200 OK
Content-Type: application/json
```
- `id` is the unique execution identifier.
- `ok` is `true` if the code executed without errors, or `false` otherwise.
- `duration` is the execution time in milliseconds.
- `stdout` is what the code printed to the standard output.
- `stderr` is what the code printed to the standard error, or a compiler/os error (if any).
## Usage (JavaScript)
See [codapi-js](https://github.com/nalgeon/codapi-js) to embed the JavaScript widget into a web page. The widget uses exactly the same API as described above.
## Contributing
Contributions are welcome. For anything other than bugfixes, please first open an issue to discuss what you want to change.
Be sure to add or update tests as appropriate.
See [codapi-js](https://github.com/nalgeon/codapi-js) to embed the JavaScript widget into a web page.
## Contributing
@@ -84,12 +44,12 @@ Contributions are welcome. For anything other than bugfixes, please first open a
Be sure to add or update tests as appropriate.
## Funding
Copyright 2023 [Anton Zhiyanov](https://antonz.org/).
Codapi is mostly a [one-man](https://antonz.org/) project, not backed by a VC fund or anything.
The software is available under the Apache-2.0 license.
If you find Codapi useful, please consider sponsoring it on GitHub. It really helps to move the project forward.
## Stay tuned
♥ [Become a sponsor](https://github.com/sponsors/nalgeon) to support Codapi.
★ [Subscribe](https://antonz.org/subscribe/) to stay on top of new features.

View File

@@ -13,11 +13,16 @@ import (
"github.com/nalgeon/codapi/internal/server"
)
var Version string = "main"
// set by the build process
var (
version = "main"
commit = "none"
date = "unknown"
)
// startServer starts the HTTP API sandbox server.
func startServer(port int) *server.Server {
logx.Log("codapi %s", Version)
logx.Log("codapi %s, commit %s, built at %s", version, commit, date)
logx.Log("listening on port %d...", port)
router := server.NewRouter()
srv := server.NewServer(port, router)

42
docs/api.md Normal file
View File

@@ -0,0 +1,42 @@
# API
Call `/v1/exec` to run the code in a sandbox:
```http
POST https://api.codapi.org/v1/exec
content-type: application/json
{
"sandbox": "python",
"command": "run",
"files": {
"": "print('hello world')"
}
}
```
`sandbox` is the name of the pre-configured sandbox, and `command` is the name of a command supported by that sandbox. See [Adding a sandbox](add-sandbox.md) for details on how to add a new sandbox.
`files` is a map, where the key is a filename and the value is its contents. When executing a single file, it should either be named as the `command` expects, or be an empty string (as in the example above).
Response:
```http
HTTP/1.1 200 OK
Content-Type: application/json
```
- `id` is the unique execution identifier.
- `ok` is `true` if the code executed without errors, or `false` otherwise.
- `duration` is the execution time in milliseconds.
- `stdout` is what the code printed to the standard output.
- `stderr` is what the code printed to the standard error, or a compiler/os error (if any).
-
- `id` is the unique execution identifier.
- `ok` is `true` if the code executed without errors, or `false` otherwise.
- `duration` is the execution time in milliseconds.
- `stdout` is what the code printed to the standard output.
- `stderr` is what the code printed to the standard error, or a compiler/os error (if any).
-

View File

@@ -28,10 +28,10 @@ docker run hello-world
```sh
cd /opt/codapi
curl -L -O "https://github.com/nalgeon/codapi/releases/download/0.5.0/codapi_0.5.0_linux_amd64.tar.gz"
tar xvzf codapi_0.5.0_linux_amd64.tar.gz
curl -L -O "https://github.com/nalgeon/codapi/releases/download/0.8.0/codapi_0.8.0_linux_amd64.tar.gz"
tar xvzf codapi_0.8.0_linux_amd64.tar.gz
chmod +x codapi
rm -f codapi_0.5.0_linux_amd64.tar.gz
rm -f codapi_0.8.0_linux_amd64.tar.gz
```
5. Build Docker images (as codapi):

2
go.mod
View File

@@ -1,3 +1,3 @@
module github.com/nalgeon/codapi
go 1.20
go 1.21

View File

@@ -6,7 +6,7 @@ import (
"sort"
)
// A Config describes application cofig.
// A Config describes application config.
type Config struct {
PoolSize int `json:"pool_size"`
Verbose bool `json:"verbose"`
@@ -96,8 +96,10 @@ type Command struct {
// A Step describes a single step of a command.
type Step struct {
Box string `json:"box"`
Version string `json:"version"`
User string `json:"user"`
Action string `json:"action"`
Detach bool `json:"detach"`
Stdin bool `json:"stdin"`
Command []string `json:"command"`
Timeout int `json:"timeout"`

View File

@@ -8,7 +8,6 @@ import (
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
@@ -24,6 +23,7 @@ var killTimeout = 5 * time.Second
const (
actionRun = "run"
actionExec = "exec"
actionStop = "stop"
)
// A Docker engine executes a specific sandbox command
@@ -42,7 +42,7 @@ func NewDocker(cfg *config.Config, sandbox, command string) Engine {
// Exec executes the command and returns the output.
func (e *Docker) Exec(req Request) Execution {
// all steps operate in the same temp directory
dir, err := os.MkdirTemp("", "")
dir, err := fileio.MkdirTemp(0777)
if err != nil {
err = NewExecutionError("create temp dir", err)
return Fail(req.ID, err)
@@ -54,7 +54,10 @@ func (e *Docker) Exec(req Request) Execution {
if e.cmd.Entry != "" {
// write request files to the temp directory
err = e.writeFiles(dir, req.Files)
if err != nil {
var argErr ArgumentError
if errors.As(err, &argErr) {
return Fail(req.ID, err)
} else if err != nil {
err = NewExecutionError("write files to temp dir", err)
return Fail(req.ID, err)
}
@@ -62,7 +65,7 @@ func (e *Docker) Exec(req Request) Execution {
// initialization step
if e.cmd.Before != nil {
out := e.execStep(e.cmd.Before, req.ID, dir, nil)
out := e.execStep(e.cmd.Before, req, dir, nil)
if !out.OK {
return out
}
@@ -70,14 +73,14 @@ func (e *Docker) Exec(req Request) Execution {
// the first step is required
first, rest := e.cmd.Steps[0], e.cmd.Steps[1:]
out := e.execStep(first, req.ID, dir, req.Files)
out := e.execStep(first, req, dir, req.Files)
// the rest are optional
if out.OK && len(rest) > 0 {
// each step operates on the results of the previous one,
// without using the source files - hence `nil` instead of `files`
for _, step := range rest {
out = e.execStep(step, req.ID, dir, nil)
out = e.execStep(step, req, dir, nil)
if !out.OK {
break
}
@@ -86,7 +89,7 @@ func (e *Docker) Exec(req Request) Execution {
// cleanup step
if e.cmd.After != nil {
afterOut := e.execStep(e.cmd.After, req.ID, dir, nil)
afterOut := e.execStep(e.cmd.After, req, dir, nil)
if out.OK && !afterOut.OK {
return afterOut
}
@@ -96,34 +99,67 @@ func (e *Docker) Exec(req Request) Execution {
}
// execStep executes a step using the docker container.
func (e *Docker) execStep(step *config.Step, reqID, dir string, files Files) Execution {
box := e.cfg.Boxes[step.Box]
err := e.copyFiles(box, dir)
func (e *Docker) execStep(step *config.Step, req Request, dir string, files Files) Execution {
box, err := e.getBox(step, req)
if err != nil {
err = NewExecutionError("copy files to temp dir", err)
return Fail(reqID, err)
return Fail(req.ID, err)
}
stdout, stderr, err := e.exec(box, step, reqID, dir, files)
err = e.copyFiles(box, dir)
if err != nil {
return Fail(reqID, err)
err = NewExecutionError("copy files to temp dir", err)
return Fail(req.ID, err)
}
stdout, stderr, err := e.exec(box, step, req, dir, files)
if err != nil {
return Fail(req.ID, err)
}
return Execution{
ID: reqID,
ID: req.ID,
OK: true,
Stdout: stdout,
Stderr: stderr,
}
}
// getBox selects an appropriate box for the step (if any).
func (e *Docker) getBox(step *config.Step, req Request) (*config.Box, error) {
if step.Action != actionRun {
// steps other than "run" use existing containers
// and do not spin up new ones
return nil, nil
}
var boxName string
// If the version is set in the step config, use it.
if step.Version != "" {
if step.Version == "latest" {
boxName = step.Box
} else {
boxName = step.Box + ":" + step.Version
}
} else if req.Version != "" {
// If the version is set in the request, use it.
boxName = step.Box + ":" + req.Version
} else {
// otherwise, use the latest version
boxName = step.Box
}
box, found := e.cfg.Boxes[boxName]
if !found {
return nil, fmt.Errorf("unknown box %s", boxName)
}
return box, nil
}
// copyFiles copies box files to the temporary directory.
func (e *Docker) copyFiles(box *config.Box, dir string) error {
if box == nil || len(box.Files) == 0 {
return nil
}
for _, pattern := range box.Files {
err := fileio.CopyFiles(pattern, dir)
err := fileio.CopyFiles(pattern, dir, 0444)
if err != nil {
return err
}
@@ -138,8 +174,13 @@ func (e *Docker) writeFiles(dir string, files Files) error {
if name == "" {
name = e.cmd.Entry
}
path := filepath.Join(dir, name)
err = os.WriteFile(path, []byte(content), 0444)
var path string
path, err = fileio.JoinDir(dir, name)
if err != nil {
err = NewArgumentError(fmt.Sprintf("files[%s]", name), err)
return false
}
err = fileio.WriteFile(path, content, 0444)
return err == nil
})
return err
@@ -147,18 +188,18 @@ func (e *Docker) writeFiles(dir string, files Files) error {
// exec executes the step in the docker container
// using the files from in the temporary directory.
func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, files Files) (stdout string, stderr string, err error) {
func (e *Docker) exec(box *config.Box, step *config.Step, req Request, dir string, files Files) (stdout string, stderr string, err error) {
// limit the stdout/stderr size
prog := NewProgram(step.Timeout, int64(step.NOutput))
args := e.buildArgs(box, step, reqID, dir)
args := e.buildArgs(box, step, req, dir)
if step.Stdin {
// pass files to container from stdin
stdin := filesReader(files)
stdout, stderr, err = prog.RunStdin(stdin, reqID, "docker", args...)
stdout, stderr, err = prog.RunStdin(stdin, req.ID, "docker", args...)
} else {
// pass files to container from temp directory
stdout, stderr, err = prog.Run(reqID, "docker", args...)
stdout, stderr, err = prog.Run(req.ID, "docker", args...)
}
if err == nil {
@@ -168,15 +209,15 @@ func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, fil
if err.Error() == "signal: killed" {
if step.Action == actionRun {
// we have to "docker kill" the container here, because the proccess
// we have to "docker kill" the container here, because the process
// inside the container is not related to the "docker run" process,
// and will hang forever after the "docker run" process is killed
go func() {
err = dockerKill(reqID)
err = dockerKill(req.ID)
if err == nil {
logx.Debug("%s: docker kill ok", reqID)
logx.Debug("%s: docker kill ok", req.ID)
} else {
logx.Log("%s: docker kill failed: %v", reqID, err)
logx.Log("%s: docker kill failed: %v", req.ID, err)
}
}()
}
@@ -202,28 +243,31 @@ func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, fil
}
// buildArgs prepares the arguments for the `docker` command.
func (e *Docker) buildArgs(box *config.Box, step *config.Step, name, dir string) []string {
func (e *Docker) buildArgs(box *config.Box, step *config.Step, req Request, dir string) []string {
var args []string
if step.Action == actionRun {
args = dockerRunArgs(box, step, name, dir)
} else if step.Action == actionExec {
args = dockerExecArgs(step)
} else {
switch step.Action {
case actionRun:
args = dockerRunArgs(box, step, req, dir)
case actionExec:
args = dockerExecArgs(step, req)
case actionStop:
args = dockerStopArgs(step, req)
default:
// should never happen if the config is valid
args = []string{"version"}
}
command := expandVars(step.Command, name)
command := expandVars(step.Command, req.ID)
args = append(args, command...)
logx.Debug("%v", args)
return args
}
// buildArgs prepares the arguments for the `docker run` command.
func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []string {
func dockerRunArgs(box *config.Box, step *config.Step, req Request, dir string) []string {
args := []string{
actionRun, "--rm",
"--name", name,
"--name", req.ID,
"--runtime", box.Runtime,
"--cpus", strconv.Itoa(box.CPU),
"--memory", fmt.Sprintf("%dm", box.Memory),
@@ -231,12 +275,15 @@ func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []strin
"--pids-limit", strconv.Itoa(box.NProc),
"--user", step.User,
}
if !box.Writable {
args = append(args, "--read-only")
if step.Detach {
args = append(args, "--detach")
}
if step.Stdin {
args = append(args, "--interactive")
}
if !box.Writable {
args = append(args, "--read-only")
}
if box.Storage != "" {
args = append(args, "--storage-opt", fmt.Sprintf("size=%s", box.Storage))
}
@@ -260,14 +307,23 @@ func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []strin
}
// dockerExecArgs prepares the arguments for the `docker exec` command.
func dockerExecArgs(step *config.Step) []string {
func dockerExecArgs(step *config.Step, req Request) []string {
// :name means executing in the container passed in the request
box := strings.Replace(step.Box, ":name", req.ID, 1)
return []string{
actionExec, "--interactive",
"--user", step.User,
step.Box,
box,
}
}
// dockerStopArgs prepares the arguments for the `docker stop` command.
func dockerStopArgs(step *config.Step, req Request) []string {
// :name means executing in the container passed in the request
box := strings.Replace(step.Box, ":name", req.ID, 1)
return []string{actionStop, box}
}
// filesReader creates a reader over an in-memory collection of files.
func filesReader(files Files) io.Reader {
var input strings.Builder
@@ -283,7 +339,7 @@ func expandVars(command []string, name string) []string {
expanded := make([]string, len(command))
copy(expanded, command)
for i, cmd := range expanded {
expanded[i] = strings.Replace(cmd, ":name", name, 1)
expanded[i] = strings.Replace(cmd, ":name", name, -1)
}
return expanded
}

View File

@@ -1,6 +1,7 @@
package engine
import (
"fmt"
"strings"
"testing"
@@ -11,8 +12,26 @@ import (
var dockerCfg = &config.Config{
Boxes: map[string]*config.Box{
"postgresql": {
Image: "codapi/postgresql",
"alpine": {
Image: "codapi/alpine",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
"go": {
Image: "codapi/go",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
"go:dev": {
Image: "codapi/go:dev",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
@@ -29,8 +48,56 @@ var dockerCfg = &config.Config{
NProc: 64,
},
},
"python:dev": {
Image: "codapi/python:dev",
Runtime: "runc",
Host: config.Host{
CPU: 1, Memory: 64, Network: "none",
Volume: "%s:/sandbox:ro",
NProc: 64,
},
},
},
Commands: map[string]config.SandboxCommands{
"alpine": map[string]*config.Command{
"echo": {
Engine: "docker",
Before: &config.Step{
Box: "alpine", User: "sandbox", Action: "run", Detach: true,
Command: []string{"echo", "before"},
NOutput: 4096,
},
Steps: []*config.Step{
{
Box: ":name", User: "sandbox", Action: "exec",
Command: []string{"sh", "main.sh"},
NOutput: 4096,
},
},
After: &config.Step{
Box: ":name", User: "sandbox", Action: "stop",
NOutput: 4096,
},
},
},
"go": map[string]*config.Command{
"run": {
Engine: "docker",
Steps: []*config.Step{
{
Box: "go", User: "sandbox", Action: "run",
Command: []string{"go", "build"},
NOutput: 4096,
},
{
Box: "alpine", Version: "latest",
User: "sandbox", Action: "run",
Command: []string{"./main"},
NOutput: 4096,
},
},
},
},
"postgresql": map[string]*config.Command{
"run": {
Engine: "docker",
@@ -75,9 +142,10 @@ func TestDockerRun(t *testing.T) {
"docker run": {Stdout: "hello world", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
engine := NewDocker(dockerCfg, "python", "run")
t.Run("success", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
@@ -103,8 +171,111 @@ func TestDockerRun(t *testing.T) {
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "codapi/python")
mem.MustHave(t, "python main.py")
})
t.Run("latest version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/python")
})
t.Run("custom version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Version: "dev",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/python:dev")
})
t.Run("step version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "go", "run")
req := Request{
ID: "http_42",
Sandbox: "go",
Version: "dev",
Command: "run",
Files: map[string]string{
"": "var n = 42",
},
}
out := engine.Exec(req)
if !out.OK {
t.Error("OK: expected true")
}
mem.MustHave(t, "codapi/go:dev")
mem.MustHave(t, "codapi/alpine")
})
t.Run("unsupported version", func(t *testing.T) {
mem.Clear()
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Version: "42",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
},
}
out := engine.Exec(req)
if out.OK {
t.Error("OK: expected false")
}
want := "unknown box python:42"
if out.Stderr != want {
t.Errorf("Stderr: unexpected value: %s", out.Stderr)
}
})
t.Run("directory traversal attack", func(t *testing.T) {
mem.Clear()
const fileName = "../../opt/codapi/codapi"
engine := NewDocker(dockerCfg, "python", "run")
req := Request{
ID: "http_42",
Sandbox: "python",
Command: "run",
Files: map[string]string{
"": "print('hello world')",
fileName: "hehe",
},
}
out := engine.Exec(req)
if out.OK {
t.Error("OK: expected false")
}
want := fmt.Sprintf("files[%s]: invalid name", fileName)
if out.Stderr != want {
t.Errorf("Stderr: unexpected value: %s", out.Stderr)
}
})
}
func TestDockerExec(t *testing.T) {
@@ -147,11 +318,55 @@ func TestDockerExec(t *testing.T) {
})
}
func TestDockerStop(t *testing.T) {
logx.Mock()
commands := map[string]execy.CmdOut{
"docker run": {Stdout: "c958ff2", Stderr: "", Err: nil},
"docker exec": {Stdout: "hello", Stderr: "", Err: nil},
"docker stop": {Stdout: "alpine_42", Stderr: "", Err: nil},
}
mem := execy.Mock(commands)
engine := NewDocker(dockerCfg, "alpine", "echo")
t.Run("success", func(t *testing.T) {
req := Request{
ID: "alpine_42",
Sandbox: "alpine",
Command: "echo",
Files: map[string]string{
"": "echo hello",
},
}
out := engine.Exec(req)
if out.ID != req.ID {
t.Errorf("ID: expected %s, got %s", req.ID, out.ID)
}
if !out.OK {
t.Error("OK: expected true")
}
want := "hello"
if out.Stdout != want {
t.Errorf("Stdout: expected %q, got %q", want, out.Stdout)
}
if out.Stderr != "" {
t.Errorf("Stderr: expected %q, got %q", "", out.Stdout)
}
if out.Err != nil {
t.Errorf("Err: expected nil, got %v", out.Err)
}
mem.MustHave(t, "docker run --rm --name alpine_42", "--detach")
mem.MustHave(t, "docker exec --interactive --user sandbox alpine_42 sh main.sh")
mem.MustHave(t, "docker stop alpine_42")
})
}
func Test_expandVars(t *testing.T) {
const name = "codapi_01"
commands := map[string]string{
"python main.py": "python main.py",
"sh create.sh :name": "sh create.sh " + name,
"python main.py": "python main.py",
"sh create.sh :name": "sh create.sh " + name,
"sh copy.sh :name new-:name": "sh copy.sh " + name + " new-" + name,
}
for cmd, want := range commands {
src := strings.Fields(cmd)

View File

@@ -12,13 +12,18 @@ import (
type Request struct {
ID string `json:"id"`
Sandbox string `json:"sandbox"`
Version string `json:"version,omitempty"`
Command string `json:"command"`
Files Files `json:"files"`
}
// GenerateID() sets a unique ID for the request.
func (r *Request) GenerateID() {
r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8))
if r.Version != "" {
r.ID = fmt.Sprintf("%s.%s_%s_%s", r.Sandbox, r.Version, r.Command, stringx.RandString(8))
} else {
r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8))
}
}
// An Execution is an output from the code execution engine.
@@ -57,6 +62,25 @@ func (err ExecutionError) Unwrap() error {
return err.inner
}
// An ArgumentError is returned if code execution failed
// due to the invalid value of the request argument.
type ArgumentError struct {
name string
reason error
}
func NewArgumentError(name string, reason error) ArgumentError {
return ArgumentError{name: name, reason: reason}
}
func (err ArgumentError) Error() string {
return err.name + ": " + err.reason.Error()
}
func (err ArgumentError) Unwrap() error {
return err.reason
}
// Files are a collection of files to be executed by the engine.
type Files map[string]string

View File

@@ -4,9 +4,34 @@ import (
"errors"
"reflect"
"sort"
"strings"
"testing"
)
func TestGenerateID(t *testing.T) {
t.Run("with version", func(t *testing.T) {
req := Request{
Sandbox: "python",
Version: "dev",
Command: "run",
}
req.GenerateID()
if !strings.HasPrefix(req.ID, "python.dev_run_") {
t.Errorf("ID: unexpected prefix %s", req.ID)
}
})
t.Run("without version", func(t *testing.T) {
req := Request{
Sandbox: "python",
Command: "run",
}
req.GenerateID()
if !strings.HasPrefix(req.ID, "python_run_") {
t.Errorf("ID: unexpected prefix %s", req.ID)
}
})
}
func TestExecutionError(t *testing.T) {
inner := errors.New("inner error")
err := NewExecutionError("failed", inner)

View File

@@ -48,7 +48,7 @@ func (p *Program) RunStdin(stdin io.Reader, id, name string, arg ...string) (std
cmd.Stdout = LimitWriter(&cmdout, p.nOutput)
cmd.Stderr = LimitWriter(&cmderr, p.nOutput)
err = execy.Run(cmd)
stdout = cmdout.String()
stderr = cmderr.String()
stdout = strings.TrimSpace(cmdout.String())
stderr = strings.TrimSpace(cmderr.String())
return
}

View File

@@ -2,15 +2,28 @@
package fileio
import (
"encoding/base64"
"encoding/json"
"errors"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
)
// Exists checks if the specified path exists.
func Exists(path string) bool {
_, err := os.Stat(path)
// we need a double negation here, because
// errors.Is(err, os.ErrExist)
// does not work
return !errors.Is(err, os.ErrNotExist)
}
// CopyFile copies all files matching the pattern
// to the destination directory.
func CopyFiles(pattern string, dstDir string) error {
// to the destination directory. Does not overwrite existing file.
func CopyFiles(pattern string, dstDir string, perm fs.FileMode) error {
matches, err := filepath.Glob(pattern)
if err != nil {
return err
@@ -24,7 +37,11 @@ func CopyFiles(pattern string, dstDir string) error {
defer src.Close()
dstFile := filepath.Join(dstDir, filepath.Base(match))
dst, err := os.Create(dstFile)
if Exists(dstFile) {
continue
}
dst, err := os.OpenFile(dstFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm)
if err != nil {
return err
}
@@ -52,3 +69,78 @@ func ReadJson[T any](path string) (T, error) {
}
return obj, err
}
// WriteFile writes the file to disk.
// The content can be text or binary (encoded as a data URL),
// e.g. data:application/octet-stream;base64,MTIz
func WriteFile(path, content string, perm fs.FileMode) (err error) {
var data []byte
if !strings.HasPrefix(content, "data:") {
// text file
data = []byte(content)
return os.WriteFile(path, data, perm)
}
// data-url encoded file
meta, encoded, found := strings.Cut(content, ",")
if !found {
return errors.New("invalid data-url encoding")
}
if !strings.HasSuffix(meta, "base64") {
// no need to decode
data = []byte(encoded)
return os.WriteFile(path, data, perm)
}
// decode base64-encoded data
data, err = base64.StdEncoding.DecodeString(encoded)
if err != nil {
return err
}
return os.WriteFile(path, data, perm)
}
// JoinDir joins a directory path with a relative file path,
// making sure that the resulting path is still inside the directory.
// Returns an error otherwise.
func JoinDir(dir string, name string) (string, error) {
if dir == "" {
return "", errors.New("invalid dir")
}
cleanName := filepath.Clean(name)
if cleanName == "" {
return "", errors.New("invalid name")
}
if cleanName == "." || cleanName == "/" || filepath.IsAbs(cleanName) {
return "", errors.New("invalid name")
}
path := filepath.Join(dir, cleanName)
dirPrefix := filepath.Clean(dir)
if dirPrefix != "/" {
dirPrefix += string(os.PathSeparator)
}
if !strings.HasPrefix(path, dirPrefix) {
return "", errors.New("invalid name")
}
return path, nil
}
// MkdirTemp creates a new temporary directory with given permissions
// and returns the pathname of the new directory.
func MkdirTemp(perm fs.FileMode) (string, error) {
dir, err := os.MkdirTemp("", "")
if err != nil {
return "", err
}
err = os.Chmod(dir, perm)
if err != nil {
os.Remove(dir)
return "", err
}
return dir, nil
}

View File

@@ -1,59 +1,126 @@
package fileio
import (
"io/fs"
"os"
"path/filepath"
"reflect"
"testing"
)
func TestExists(t *testing.T) {
t.Run("exists", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "file.txt")
err := os.WriteFile(path, []byte{1, 2, 3}, 0444)
if err != nil {
t.Fatal(err)
}
if !Exists(path) {
t.Fatalf("Exists: %s does not exist", filepath.Base(path))
}
})
t.Run("does not exist", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "file.txt")
if Exists(path) {
t.Fatalf("Exists: %s should not exist", filepath.Base(path))
}
})
}
func TestCopyFiles(t *testing.T) {
// Create a temporary directory for testing
// create a temporary directory for testing
srcDir, err := os.MkdirTemp("", "src")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(srcDir)
// Create a source file
// create a source file
srcFile := filepath.Join(srcDir, "source.txt")
err = os.WriteFile(srcFile, []byte("test data"), 0644)
if err != nil {
t.Fatal(err)
}
// Specify the destination directory
// specify the destination directory
dstDir, err := os.MkdirTemp("", "dst")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dstDir)
// Call the CopyFiles function
pattern := filepath.Join(srcDir, "*.txt")
err = CopyFiles(pattern, dstDir)
if err != nil {
t.Fatal(err)
}
t.Run("copy", func(t *testing.T) {
// call the CopyFiles function
const perm = fs.FileMode(0444)
pattern := filepath.Join(srcDir, "*.txt")
err = CopyFiles(pattern, dstDir, perm)
if err != nil {
t.Fatal(err)
}
// Verify that the file was copied correctly
dstFile := filepath.Join(dstDir, "source.txt")
_, err = os.Stat(dstFile)
if err != nil {
t.Fatalf("file not copied: %s", err)
}
// verify that the file was copied correctly
dstFile := filepath.Join(dstDir, "source.txt")
fileInfo, err := os.Stat(dstFile)
if err != nil {
t.Fatalf("file not copied: %s", err)
}
if fileInfo.Mode() != perm {
t.Errorf("unexpected file permissions: got %v, want %v", fileInfo.Mode(), perm)
}
// Read the contents of the copied file
data, err := os.ReadFile(dstFile)
if err != nil {
t.Fatal(err)
}
// read the contents of the copied file
data, err := os.ReadFile(dstFile)
if err != nil {
t.Fatal(err)
}
// Verify the contents of the copied file
expected := []byte("test data")
if string(data) != string(expected) {
t.Errorf("unexpected file content: got %q, want %q", data, expected)
}
// verify the contents of the copied file
expected := []byte("test data")
if string(data) != string(expected) {
t.Errorf("unexpected file content: got %q, want %q", data, expected)
}
})
t.Run("skip existing", func(t *testing.T) {
// existing file in the destination dir
path := filepath.Join(dstDir, "existing.txt")
err := os.WriteFile(path, []byte("v1"), 0444)
if err != nil {
t.Fatal(err)
}
// same file in the source dir
path = filepath.Join(srcDir, "existing.txt")
err = os.WriteFile(path, []byte("v2"), 0444)
if err != nil {
t.Fatal(err)
}
// copy files
pattern := filepath.Join(srcDir, "*.txt")
err = CopyFiles(pattern, dstDir, 0444)
if err != nil {
t.Fatal(err)
}
// verify that the new file was copied correctly
newFile := filepath.Join(dstDir, "source.txt")
_, err = os.Stat(newFile)
if err != nil {
t.Fatalf("new file not copied: %s", err)
}
// verify that the existing file remained unchanged
existFile := filepath.Join(dstDir, "existing.txt")
data, err := os.ReadFile(existFile)
if err != nil {
t.Fatal(err)
}
expected := []byte("v1")
if string(data) != string(expected) {
t.Error("existing file got overwritten")
}
})
}
func TestReadJson(t *testing.T) {
@@ -82,3 +149,237 @@ func TestReadJson(t *testing.T) {
}
})
}
func TestWriteFile(t *testing.T) {
dir, err := os.MkdirTemp("", "files")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
t.Run("text", func(t *testing.T) {
path := filepath.Join(dir, "data.txt")
err = WriteFile(path, "hello", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("hello")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("data-octet-stream", func(t *testing.T) {
path := filepath.Join(dir, "data-1.bin")
err = WriteFile(path, "data:application/octet-stream;base64,MTIz", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("123")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("data-base64", func(t *testing.T) {
path := filepath.Join(dir, "data-2.bin")
err = WriteFile(path, "data:;base64,MTIz", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("123")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("data-text-plain", func(t *testing.T) {
path := filepath.Join(dir, "data-3.bin")
err = WriteFile(path, "data:text/plain;,123", 0444)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read file: expected nil err, got %v", err)
}
want := []byte("123")
if !reflect.DeepEqual(got, want) {
t.Errorf("read file: expected %v, got %v", want, got)
}
})
t.Run("perm", func(t *testing.T) {
const perm = 0444
path := filepath.Join(dir, "perm.txt")
err = WriteFile(path, "hello", perm)
if err != nil {
t.Fatalf("expected nil err, got %v", err)
}
fileInfo, err := os.Stat(path)
if err != nil {
t.Fatalf("file not created: %s", err)
}
if fileInfo.Mode().Perm() != perm {
t.Errorf("unexpected file permissions: expected %o, got %o", perm, fileInfo.Mode().Perm())
}
})
t.Run("missing data-url separator", func(t *testing.T) {
path := filepath.Join(dir, "data.bin")
err = WriteFile(path, "data:application/octet-stream:MTIz", 0444)
if err == nil {
t.Fatal("expected error, got nil")
}
})
t.Run("invalid binary value", func(t *testing.T) {
path := filepath.Join(dir, "data.bin")
err = WriteFile(path, "data:;base64,12345", 0444)
if err == nil {
t.Fatal("expected error, got nil")
}
})
}
func TestJoinDir(t *testing.T) {
tests := []struct {
name string
dir string
filename string
want string
wantErr bool
}{
{
name: "regular join",
dir: "/home/user",
filename: "docs/report.txt",
want: "/home/user/docs/report.txt",
wantErr: false,
},
{
name: "join with dot",
dir: "/home/user",
filename: ".",
want: "",
wantErr: true,
},
{
name: "join with absolute path",
dir: "/home/user",
filename: "/etc/passwd",
want: "",
wantErr: true,
},
{
name: "join with parent directory",
dir: "/home/user",
filename: "../user2/docs/report.txt",
want: "",
wantErr: true,
},
{
name: "empty directory",
dir: "",
filename: "report.txt",
want: "",
wantErr: true,
},
{
name: "empty filename",
dir: "/home/user",
filename: "",
want: "",
wantErr: true,
},
{
name: "directory with trailing slash",
dir: "/home/user/",
filename: "docs/report.txt",
want: "/home/user/docs/report.txt",
wantErr: false,
},
{
name: "filename with leading slash",
dir: "/home/user",
filename: "/docs/report.txt",
want: "",
wantErr: true,
},
{
name: "root directory",
dir: "/",
filename: "report.txt",
want: "/report.txt",
wantErr: false,
},
{
name: "dot dot slash filename",
dir: "/home/user",
filename: "..",
want: "",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := JoinDir(tt.dir, tt.filename)
if (err != nil) != tt.wantErr {
t.Errorf("JoinDir() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("JoinDir() = %v, want %v", got, tt.want)
}
})
}
}
func TestMkdirTemp(t *testing.T) {
t.Run("default permissions", func(t *testing.T) {
const perm = 0755
dir, err := MkdirTemp(perm)
if err != nil {
t.Fatalf("failed to create temp directory: %v", err)
}
defer os.Remove(dir)
info, err := os.Stat(dir)
if err != nil {
t.Fatalf("failed to stat temp directory: %v", err)
}
if info.Mode().Perm() != perm {
t.Errorf("unexpected permissions: expected %o, got %o", perm, info.Mode().Perm())
}
})
t.Run("non-default permissions", func(t *testing.T) {
const perm = 0777
dir, err := MkdirTemp(perm)
if err != nil {
t.Fatalf("failed to create temp directory: %v", err)
}
defer os.Remove(dir)
info, err := os.Stat(dir)
if err != nil {
t.Fatalf("failed to stat temp directory: %v", err)
}
if info.Mode().Perm() != perm {
t.Errorf("unexpected permissions: expected %o, got %o", perm, info.Mode().Perm())
}
})
}

View File

@@ -30,9 +30,16 @@ func (m *Memory) WriteString(s string) {
}
// Has returns true if the memory has the message.
func (m *Memory) Has(msg string) bool {
func (m *Memory) Has(message ...string) bool {
for _, line := range m.Lines {
if strings.Contains(line, msg) {
containsAll := true
for _, part := range message {
if !strings.Contains(line, part) {
containsAll = false
break
}
}
if containsAll {
return true
}
}
@@ -40,22 +47,29 @@ func (m *Memory) Has(msg string) bool {
}
// MustHave checks if the memory has the message.
func (m *Memory) MustHave(t *testing.T, msg string) {
if !m.Has(msg) {
t.Errorf("%s must have: %s", m.Name, msg)
// If the message consists of several parts,
// they must all be in the same memory line.
func (m *Memory) MustHave(t *testing.T, message ...string) {
if !m.Has(message...) {
t.Errorf("%s must have: %v", m.Name, message)
}
}
// MustNotHave checks if the memory does not have the message.
func (m *Memory) MustNotHave(t *testing.T, msg string) {
if m.Has(msg) {
t.Errorf("%s must NOT have: %s", m.Name, msg)
func (m *Memory) MustNotHave(t *testing.T, message ...string) {
if m.Has(message...) {
t.Errorf("%s must NOT have: %v", m.Name, message)
}
}
// Clear clears the memory.
func (m *Memory) Clear() {
m.Lines = []string{}
}
// Print prints memory lines to stdout.
func (m *Memory) Print() {
for _, line := range m.Lines {
fmt.Print(line)
fmt.Println(line)
}
}

View File

@@ -40,4 +40,23 @@ func TestMemory_Has(t *testing.T) {
if !mem.Has("hello world") {
t.Error("Has: unexpected false")
}
_, _ = mem.Write([]byte("one two three four"))
if !mem.Has("one two") {
t.Error("Has: one two: unexpected false")
}
if !mem.Has("two three") {
t.Error("Has: two three: unexpected false")
}
if mem.Has("one three") {
t.Error("Has: one three: unexpected true")
}
if !mem.Has("one", "three") {
t.Error("Has: [one three]: unexpected false")
}
if !mem.Has("one", "three", "four") {
t.Error("Has: [one three four]: unexpected false")
}
if !mem.Has("four", "three") {
t.Error("Has: [four three]: unexpected false")
}
}

View File

@@ -20,7 +20,7 @@ var engineConstr = map[string]func(*config.Config, string, string) engine.Engine
}
// engines is the registry of command executors.
// Each engine executes a specific command in a specifix sandbox.
// Each engine executes a specific command in a specific sandbox.
// sandbox : command : engine
// TODO: Maybe it's better to create a single instance of each engine
// and pass the sandbox and command as arguments to the Exec.

View File

@@ -7,7 +7,7 @@ import "net/http"
func enableCORS(handler func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("access-control-allow-origin", "*")
w.Header().Set("access-control-allow-method", "post")
w.Header().Set("access-control-allow-methods", "options, post")
w.Header().Set("access-control-allow-headers", "authorization, content-type")
w.Header().Set("access-control-max-age", "3600")
if r.Method == http.MethodOptions {

View File

@@ -31,8 +31,8 @@ func Test_enableCORS(t *testing.T) {
if w.Header().Get("access-control-allow-origin") != "*" {
t.Errorf("invalid access-control-allow-origin")
}
if w.Header().Get("access-control-allow-method") != "post" {
t.Errorf("invalid access-control-allow-method")
if w.Header().Get("access-control-allow-methods") != "options, post" {
t.Errorf("invalid access-control-allow-methods")
}
if w.Header().Get("access-control-allow-headers") != "authorization, content-type" {
t.Errorf("invalid access-control-allow-headers")