From 8447197d0f2f9aa3a55fee58449fe5efd57ed2ab Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 25 Nov 2023 04:02:45 +0500 Subject: [PATCH] feat: initial public version --- .github/workflows/build.yml | 41 +++++ LICENSE | 4 +- Makefile | 62 +++++++ README.md | 99 +++++++++++- boxes.json | 5 + cmd/main.go | 64 ++++++++ codapi.service | 15 ++ commands.json | 14 ++ config.json | 21 +++ config/config.go | 165 +++++++++++++++++++ config/config_test.go | 157 ++++++++++++++++++ config/load.go | 94 +++++++++++ config/load_test.go | 38 +++++ config/testdata/boxes.json | 5 + config/testdata/commands.json | 25 +++ config/testdata/config.json | 10 ++ docs/docker-xfs.md | 70 ++++++++ docs/install.md | 101 ++++++++++++ engine/docker.go | 297 ++++++++++++++++++++++++++++++++++ engine/docker_test.go | 164 +++++++++++++++++++ engine/engine.go | 116 +++++++++++++ engine/engine_test.go | 137 ++++++++++++++++ engine/exec.go | 54 +++++++ engine/exec_test.go | 74 +++++++++ engine/http.go | 166 +++++++++++++++++++ engine/http_test.go | 179 ++++++++++++++++++++ engine/io.go | 31 ++++ engine/io_test.go | 56 +++++++ engine/testdata/example.txt | 1 + execy/execy.go | 31 ++++ execy/execy_test.go | 29 ++++ execy/mock.go | 44 +++++ execy/mock_test.go | 33 ++++ fileio/fileio.go | 39 +++++ fileio/fileio_test.go | 56 +++++++ go.mod | 3 + go.sum | 0 httpx/httpx.go | 19 +++ httpx/httpx_test.go | 40 +++++ httpx/mock.go | 96 +++++++++++ httpx/mock_test.go | 34 ++++ httpx/testdata/example.json | 1 + httpx/testdata/example.txt | 1 + images/alpine/Dockerfile | 6 + logx/logx.go | 52 ++++++ logx/logx_test.go | 70 ++++++++ logx/memory.go | 61 +++++++ logx/memory_test.go | 43 +++++ sandbox/config.go | 43 +++++ sandbox/config_test.go | 59 +++++++ sandbox/sandbox.go | 47 ++++++ sandbox/sandbox_test.go | 115 +++++++++++++ sandbox/semaphore.go | 45 ++++++ sandbox/semaphore_test.go | 45 ++++++ server/io.go | 49 ++++++ server/io_test.go | 85 ++++++++++ server/middleware.go | 19 +++ server/middleware_test.go | 45 ++++++ server/router.go | 79 +++++++++ server/router_test.go | 156 ++++++++++++++++++ server/server.go | 59 +++++++ server/server_test.go | 31 ++++ stringx/stringx.go | 33 ++++ stringx/stringx_test.go | 51 ++++++ 64 files changed, 3880 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 Makefile create mode 100644 boxes.json create mode 100644 cmd/main.go create mode 100644 codapi.service create mode 100644 commands.json create mode 100644 config.json create mode 100644 config/config.go create mode 100644 config/config_test.go create mode 100644 config/load.go create mode 100644 config/load_test.go create mode 100644 config/testdata/boxes.json create mode 100644 config/testdata/commands.json create mode 100644 config/testdata/config.json create mode 100644 docs/docker-xfs.md create mode 100644 docs/install.md create mode 100644 engine/docker.go create mode 100644 engine/docker_test.go create mode 100644 engine/engine.go create mode 100644 engine/engine_test.go create mode 100644 engine/exec.go create mode 100644 engine/exec_test.go create mode 100644 engine/http.go create mode 100644 engine/http_test.go create mode 100644 engine/io.go create mode 100644 engine/io_test.go create mode 100644 engine/testdata/example.txt create mode 100644 execy/execy.go create mode 100644 execy/execy_test.go create mode 100644 execy/mock.go create mode 100644 execy/mock_test.go create mode 100644 fileio/fileio.go create mode 100644 fileio/fileio_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 httpx/httpx.go create mode 100644 httpx/httpx_test.go create mode 100644 httpx/mock.go create mode 100644 httpx/mock_test.go create mode 100644 httpx/testdata/example.json create mode 100644 httpx/testdata/example.txt create mode 100644 images/alpine/Dockerfile create mode 100644 logx/logx.go create mode 100644 logx/logx_test.go create mode 100644 logx/memory.go create mode 100644 logx/memory_test.go create mode 100644 sandbox/config.go create mode 100644 sandbox/config_test.go create mode 100644 sandbox/sandbox.go create mode 100644 sandbox/sandbox_test.go create mode 100644 sandbox/semaphore.go create mode 100644 sandbox/semaphore_test.go create mode 100644 server/io.go create mode 100644 server/io_test.go create mode 100644 server/middleware.go create mode 100644 server/middleware_test.go create mode 100644 server/router.go create mode 100644 server/router_test.go create mode 100644 server/server.go create mode 100644 server/server_test.go create mode 100644 stringx/stringx.go create mode 100644 stringx/stringx_test.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..8e7208c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,41 @@ +name: build + +on: + push: + branches: [main] + paths-ignore: + - "docs/**" + - Makefile + - README.md + workflow_dispatch: + +defaults: + run: + shell: bash + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "stable" + + - name: Test and build + run: make test build + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: codapi + path: | + build/codapi + images/ + *.json + codapi.service + Makefile + retention-days: 7 diff --git a/LICENSE b/LICENSE index 0ad25db..b0bbd76 100644 --- a/LICENSE +++ b/LICENSE @@ -629,8 +629,8 @@ to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. - - Copyright (C) + Codapi - Embeddable code playgrounds + Copyright (C) 2023+ Anton Zhiyanov This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d349f2e --- /dev/null +++ b/Makefile @@ -0,0 +1,62 @@ +.PHONY: build images + +# Development + +build_rev := "main" +ifneq ($(wildcard .git),) + build_rev := $(shell git rev-parse --short HEAD) +endif + +build_date := $(shell date -u '+%Y%m%d') +version := $(build_date):$(build_rev) + +setup: + @go mod download + +lint: + @golangci-lint run --print-issued-lines=false --out-format=colored-line-number ./... + +vet: + @go vet ./... + +test: + @go test ./... -v + + +build: + @go build -ldflags "-X main.Version=$(version)" -o build/codapi -v cmd/main.go + +run: + @./build/codapi + + +# Containers + +images: + docker build --file images/alpine/Dockerfile --tag codapi/alpine:latest images/alpine/ + +network: + docker network create --internal codapi + +# Host OS + +mount-tmp: + mount -t tmpfs tmpfs /tmp -o rw,exec,nosuid,nodev,size=64m,mode=1777 + +# Deployment + +app-download: + @curl -L -o codapi.zip "https://api.github.com/repos/nalgeon/codapi/actions/artifacts/$(id)/zip" + @unzip -ou codapi.zip + @chmod +x build/codapi + @rm -f codapi.zip + @echo "OK" + +app-start: + @nohup build/codapi > codapi.log 2>&1 & echo $$! > codapi.pid + @echo "started codapi" + +app-stop: + @kill $(shell cat codapi.pid) + @rm -f codapi.pid + @echo "stopped codapi" diff --git a/README.md b/README.md index 8922d2e..5610471 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,97 @@ -# codapi -Embeddable code playgrounds. +# Embeddable code playgrounds + +> For education, documentation, and fun. + +Codapi is a platform for embedding interactive code snippets directly into your product documentation, online course, or blog post. + +``` + python +┌───────────────────────────────┐ +│ msg = "Hello, World!" │ +│ print(msg) │ +│ │ +│ │ +│ run ► │ +└───────────────────────────────┘ + ✓ took 387 ms +┌───────────────────────────────┐ +│ Hello, World! │ +└───────────────────────────────┘ +``` + +Codapi manages sandboxes (isolated execution environments) and provides an API to execute code in these sandboxes. It also provides a JavaScript widget [codapi-js](https://github.com/nalgeon/codapi-js) for easier integration. + +Highlights: + +- Supports dozens of playgrounds out of the box, plus custom sandboxes if you need them. +- Available as a cloud service and as a self-hosted version. +- Open source. Uses the AGPL license. Committed to remaining open source forever. +- Lightweight and easy to integrate. + +Learn more at [**codapi.org**](https://codapi.org/) + +## Installation + +See [Installing Codapi](docs/install.md) for details. + +## Usage (API) + +Call `/v1/exec` to run the code in a sandbox: + +```http +POST https://api.codapi.org/v1/exec +content-type: application/json + +{ + "sandbox": "python", + "command": "run", + "files": { + "": "print('hello world')" + } +} +``` + +`sandbox` is the name of the pre-configured sandbox, and `command` is the name of a command supported by that sandbox. See [Configuration](docs/config.md) for details. + +`files` is a map, where the key is a filename and the value is its contents. When executing a single file, it should either be named as the `command` expects, or be an empty string (as in the example above). + +Response: + +```http +HTTP/1.1 200 OK +Content-Type: application/json + +{ + "id": "python_run_9b7b1afd", + "ok": true, + "duration": 314, + "stdout": "hello world\n", + "stderr": "" +} +``` + +- `id` is the unique execution identifier. +- `ok` is `true` if the code executed without errors, or `false` otherwise. +- `duration` is the execution time in milliseconds. +- `stdout` is what the code printed to the standard output. +- `stderr` is what the code printed to the standard error, or a compiler/os error (if any). + +## Usage (JavaScript) + +See [codapi-js](https://github.com/nalgeon/codapi-js) to embed the JavaScript widget into a web page. The widget uses exactly the same API as described above. + +## Contributing + +Pull requests are welcome. For new features or improvements, please first open an issue to discuss what you would like to change. + +Be sure to add or update tests as appropriate. + +## License + +Copyright 2023+ [Anton Zhiyanov](https://antonz.org/). + +The software is available under the AGPL License. + +## Stay tuned + +[**★ Subscribe**](https://antonz.org/subscribe/) to stay on top of new features. diff --git a/boxes.json b/boxes.json new file mode 100644 index 0000000..ed0eb66 --- /dev/null +++ b/boxes.json @@ -0,0 +1,5 @@ +{ + "alpine": { + "image": "codapi/alpine" + } +} diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..ece9ea7 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,64 @@ +// Codapi safely executes code snippets using sandboxes. +package main + +import ( + "flag" + "os" + "os/signal" + "syscall" + + "github.com/nalgeon/codapi/config" + "github.com/nalgeon/codapi/logx" + "github.com/nalgeon/codapi/sandbox" + "github.com/nalgeon/codapi/server" +) + +var Version string = "main" + +// startServer starts the HTTP API sandbox server. +func startServer(port int) *server.Server { + logx.Log("codapi %s", Version) + logx.Log("listening on port %d...", port) + router := server.NewRouter() + srv := server.NewServer(port, router) + srv.Start() + return srv +} + +// listenSignals listens for termination signals +// and performs graceful shutdown. +func listenSignals(srv *server.Server) { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + <-sigs + logx.Log("stopping...") + err := srv.Stop() + if err != nil { + logx.Log("failed to stop: %v", err) + } +} + +func main() { + port := flag.Int("port", 1313, "server port") + flag.Parse() + + cfg, err := config.Read("config.json", "boxes.json", "commands.json") + if err != nil { + logx.Log("missing config file") + os.Exit(1) + } + + err = sandbox.ApplyConfig(cfg) + if err != nil { + logx.Log("invalid config: %v", err) + os.Exit(1) + } + + srv := startServer(*port) + logx.Verbose = cfg.Verbose + logx.Log("workers: %d", cfg.PoolSize) + logx.Log("boxes: %v", cfg.BoxNames()) + logx.Log("commands: %v", cfg.CommandNames()) + + listenSignals(srv) +} diff --git a/codapi.service b/codapi.service new file mode 100644 index 0000000..6bec4c5 --- /dev/null +++ b/codapi.service @@ -0,0 +1,15 @@ +[Unit] +Description=Code playgrounds +After=network.target + +[Service] +Type=simple +User=codapi +WorkingDirectory=/opt/codapi +ExecStart=/opt/codapi/build/codapi +Restart=on-failure +StandardOutput=file:/opt/codapi/codapi.log +StandardError=file:/opt/codapi/codapi.log + +[Install] +WantedBy=multi-user.target diff --git a/commands.json b/commands.json new file mode 100644 index 0000000..00f8e95 --- /dev/null +++ b/commands.json @@ -0,0 +1,14 @@ +{ + "sh": { + "run": { + "engine": "docker", + "entry": "main.sh", + "steps": [ + { + "box": "alpine", + "command": ["sh", "main.sh"] + } + ] + } + } +} diff --git a/config.json b/config.json new file mode 100644 index 0000000..7416ac3 --- /dev/null +++ b/config.json @@ -0,0 +1,21 @@ +{ + "pool_size": 8, + "verbose": true, + "box": { + "runtime": "runc", + "cpu": 1, + "memory": 64, + "network": "none", + "writable": false, + "volume": "%s:/sandbox:ro", + "cap_drop": ["all"], + "ulimit": ["nofile=96"], + "nproc": 64 + }, + "step": { + "user": "sandbox", + "action": "run", + "timeout": 5, + "noutput": 4096 + } +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..3fc4036 --- /dev/null +++ b/config/config.go @@ -0,0 +1,165 @@ +// Package config reads application config. +package config + +import ( + "encoding/json" + "sort" +) + +// A Config describes application cofig. +type Config struct { + PoolSize int `json:"pool_size"` + Verbose bool `json:"verbose"` + Box *Box `json:"box"` + Step *Step `json:"step"` + HTTP *HTTP `json:"http"` + + // These are the available containers ("boxes"). + Boxes map[string]*Box `json:"boxes"` + + // These are the "sandboxes". Each sandbox can contain + // multiple commands, and each command can contain + // multiple steps. Each step is executed in a specific box. + Commands map[string]SandboxCommands `json:"commands"` +} + +// BoxNames returns configured box names. +func (cfg *Config) BoxNames() []string { + names := make([]string, 0, len(cfg.Boxes)) + for name := range cfg.Boxes { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// CommandNames returns configured command names. +func (cfg *Config) CommandNames() []string { + names := make([]string, 0, len(cfg.Commands)) + for name := range cfg.Commands { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// ToJSON returns JSON-encoded config with indentation. +func (cfg *Config) ToJSON() string { + data, _ := json.MarshalIndent(cfg, "", " ") + return string(data) +} + +// A Box describes a specific container. +// There is an important difference between a "sandbox" and a "box". +// A box is a single container. A sandbox is an environment in which we run commands. +// A sandbox command can contain multiple steps, each of which runs in a separate box. +// So the relation sandbox -> box is 1 -> 1+. +type Box struct { + Image string `json:"image"` + Runtime string `json:"runtime"` + Host + + Files []string `json:"files"` +} + +// A Host describes container Host attributes. +type Host struct { + CPU int `json:"cpu"` + Memory int `json:"memory"` + Storage string `json:"storage"` + Network string `json:"network"` + Writable bool `json:"writable"` + Volume string `json:"volume"` + Tmpfs []string `json:"tmpfs"` + CapAdd []string `json:"cap_add"` + CapDrop []string `json:"cap_drop"` + Ulimit []string `json:"ulimit"` + // do not use the ulimit nproc because it is + // a per-user setting, not a per-container setting + NProc int `json:"nproc"` +} + +// SandboxCommands describes all commands available for a sandbox. +// command name : command +type SandboxCommands map[string]*Command + +// A Command describes a specific set of actions to take +// when executing a command in a sandbox. +type Command struct { + Engine string `json:"engine"` + Entry string `json:"entry"` + Before *Step `json:"before"` + Steps []*Step `json:"steps"` + After *Step `json:"after"` +} + +// A Step describes a single step of a command. +type Step struct { + Box string `json:"box"` + User string `json:"user"` + Action string `json:"action"` + Stdin bool `json:"stdin"` + Command []string `json:"command"` + Timeout int `json:"timeout"` + NOutput int `json:"noutput"` +} + +// An HTTP describes HTTP engine settings. +type HTTP struct { + Hosts map[string]string `json:"hosts"` +} + +// setBoxDefaults sets default box properties +// instead of zero values. +func setBoxDefaults(box, defs *Box) { + if box.Runtime == "" { + box.Runtime = defs.Runtime + } + if box.CPU == 0 { + box.CPU = defs.CPU + } + if box.Memory == 0 { + box.Memory = defs.Memory + } + if box.Storage == "" { + box.Storage = defs.Storage + } + if box.Network == "" { + box.Network = defs.Network + } + if box.Volume == "" { + box.Volume = defs.Volume + } + if box.Tmpfs == nil { + box.Tmpfs = defs.Tmpfs + } + if box.CapAdd == nil { + box.CapAdd = defs.CapAdd + } + if box.CapDrop == nil { + box.CapDrop = defs.CapDrop + } + if box.Ulimit == nil { + box.Ulimit = defs.Ulimit + } + if box.NProc == 0 { + box.NProc = defs.NProc + } +} + +// setStepDefaults sets default command step +// properties instead of zero values. +func setStepDefaults(step, defs *Step) { + if step.User == "" { + step.User = defs.User + } + if step.Action == "" { + step.Action = defs.Action + } + if step.Timeout == 0 { + step.Timeout = defs.Timeout + } + if step.NOutput == 0 { + step.NOutput = defs.NOutput + } +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..19d2ec6 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,157 @@ +package config + +import ( + "reflect" + "strings" + "testing" +) + +func TestConfig_BoxNames(t *testing.T) { + cfg := &Config{ + Boxes: map[string]*Box{ + "go": {}, + "python": {}, + }, + } + + want := []string{"go", "python"} + got := cfg.BoxNames() + if !reflect.DeepEqual(got, want) { + t.Errorf("BoxNames: expected %v, got %v", want, got) + } +} + +func TestConfig_CommandNames(t *testing.T) { + cfg := &Config{ + Commands: map[string]SandboxCommands{ + "go": map[string]*Command{ + "run": {}, + }, + "python": map[string]*Command{ + "run": {}, + "test": {}, + }, + }, + } + + want := []string{"go", "python"} + got := cfg.CommandNames() + if !reflect.DeepEqual(got, want) { + t.Errorf("CommandNames: expected %v, got %v", want, got) + } +} + +func TestConfig_ToJSON(t *testing.T) { + cfg := &Config{ + PoolSize: 8, + Boxes: map[string]*Box{ + "go": {}, + "python": {}, + }, + Commands: map[string]SandboxCommands{ + "go": map[string]*Command{ + "run": {}, + }, + "python": map[string]*Command{ + "run": {}, + "test": {}, + }, + }, + } + + got := cfg.ToJSON() + if !strings.Contains(got, `"pool_size": 8`) { + t.Error("ToJSON: expected pool_size = 8") + } +} + +func Test_setBoxDefaults(t *testing.T) { + box := &Box{} + defs := &Box{ + Image: "codapi/python", + Runtime: "runc", + Host: Host{ + CPU: 1, Memory: 64, Storage: "16m", + Network: "none", Writable: true, + Volume: "%s:/sandbox:ro", + Tmpfs: []string{"/tmp:rw,size=16m"}, + CapAdd: []string{"all"}, + CapDrop: []string{"none"}, + Ulimit: []string{"nofile=96"}, + NProc: 96, + }, + Files: []string{"config.py"}, + } + setBoxDefaults(box, defs) + if box.Image != "" { + t.Error("Image: should not set default value") + } + if box.Runtime != defs.Runtime { + t.Errorf("Runtime: expected %s, got %s", defs.Runtime, box.Runtime) + } + if box.CPU != defs.CPU { + t.Errorf("CPU: expected %d, got %d", defs.CPU, box.CPU) + } + if box.Memory != defs.Memory { + t.Errorf("Memory: expected %d, got %d", defs.Memory, box.Memory) + } + if box.Storage != defs.Storage { + t.Errorf("Storage: expected %s, got %s", defs.Storage, box.Storage) + } + if box.Network != defs.Network { + t.Errorf("Network: expected %s, got %s", defs.Network, box.Network) + } + if box.Volume != defs.Volume { + t.Errorf("Volume: expected %s, got %s", defs.Volume, box.Volume) + } + if !reflect.DeepEqual(box.Tmpfs, defs.Tmpfs) { + t.Errorf("Tmpfs: expected %v, got %v", defs.Tmpfs, box.Tmpfs) + } + if !reflect.DeepEqual(box.CapAdd, defs.CapAdd) { + t.Errorf("CapAdd: expected %v, got %v", defs.CapAdd, box.CapAdd) + } + if !reflect.DeepEqual(box.CapDrop, defs.CapDrop) { + t.Errorf("CapDrop: expected %v, got %v", defs.CapDrop, box.CapDrop) + } + if !reflect.DeepEqual(box.Ulimit, defs.Ulimit) { + t.Errorf("Ulimit: expected %v, got %v", defs.Ulimit, box.Ulimit) + } + if box.NProc != defs.NProc { + t.Errorf("NProc: expected %d, got %d", defs.NProc, box.NProc) + } + if len(box.Files) != 0 { + t.Error("Files: should not set default value") + } +} + +func Test_setStepDefaults(t *testing.T) { + step := &Step{} + defs := &Step{ + Box: "python", + User: "sandbox", + Action: "run", + Command: []string{"python", "main.py"}, + Timeout: 3, + NOutput: 4096, + } + + setStepDefaults(step, defs) + if step.Box != "" { + t.Error("Box: should not set default value") + } + if step.User != defs.User { + t.Errorf("User: expected %s, got %s", defs.User, step.User) + } + if step.Action != defs.Action { + t.Errorf("Action: expected %s, got %s", defs.Action, step.Action) + } + if len(step.Command) != 0 { + t.Error("Command: should not set default value") + } + if step.Timeout != defs.Timeout { + t.Errorf("Timeout: expected %d, got %d", defs.Timeout, step.Timeout) + } + if step.NOutput != defs.NOutput { + t.Errorf("NOutput: expected %d, got %d", defs.NOutput, step.NOutput) + } +} diff --git a/config/load.go b/config/load.go new file mode 100644 index 0000000..aa417bb --- /dev/null +++ b/config/load.go @@ -0,0 +1,94 @@ +package config + +import ( + "encoding/json" + "os" +) + +// Read reads application config from JSON files. +func Read(cfgPath, boxPath, cmdPath string) (*Config, error) { + cfg, err := ReadConfig(cfgPath) + if err != nil { + return nil, err + } + + cfg, err = ReadBoxes(cfg, boxPath) + if err != nil { + return nil, err + } + + cfg, err = ReadCommands(cfg, cmdPath) + if err != nil { + return nil, err + } + + return cfg, err +} + +// ReadConfig reads application config from a JSON file. +func ReadConfig(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + cfg := &Config{} + err = json.Unmarshal(data, cfg) + if err != nil { + return nil, err + } + + return cfg, err +} + +// ReadBoxes reads boxes config from a JSON file. +func ReadBoxes(cfg *Config, path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + boxes := make(map[string]*Box) + err = json.Unmarshal(data, &boxes) + if err != nil { + return nil, err + } + + for _, box := range boxes { + setBoxDefaults(box, cfg.Box) + } + + cfg.Boxes = boxes + return cfg, err +} + +// ReadCommands reads commands config from a JSON file. +func ReadCommands(cfg *Config, path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + commands := make(map[string]SandboxCommands) + err = json.Unmarshal(data, &commands) + if err != nil { + return nil, err + } + + for _, playCmds := range commands { + for _, cmd := range playCmds { + if cmd.Before != nil { + setStepDefaults(cmd.Before, cfg.Step) + } + for _, step := range cmd.Steps { + setStepDefaults(step, cfg.Step) + } + if cmd.After != nil { + setStepDefaults(cmd.After, cfg.Step) + } + } + } + + cfg.Commands = commands + return cfg, err +} diff --git a/config/load_test.go b/config/load_test.go new file mode 100644 index 0000000..14e8993 --- /dev/null +++ b/config/load_test.go @@ -0,0 +1,38 @@ +package config + +import ( + "path/filepath" + "testing" +) + +func TestRead(t *testing.T) { + cfgPath := filepath.Join("testdata", "config.json") + boxPath := filepath.Join("testdata", "boxes.json") + cmdPath := filepath.Join("testdata", "commands.json") + cfg, err := Read(cfgPath, boxPath, cmdPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if cfg.PoolSize != 8 { + t.Errorf("PoolSize: expected 8, got %d", cfg.PoolSize) + } + if !cfg.Verbose { + t.Error("Verbose: expected true") + } + if cfg.Box.Memory != 64 { + t.Errorf("Box.Memory: expected 64, got %d", cfg.Box.Memory) + } + if cfg.Step.User != "sandbox" { + t.Errorf("Step.User: expected sandbox, got %s", cfg.Step.User) + } + if _, ok := cfg.Boxes["python"]; !ok { + t.Error("Boxes: missing python box") + } + if _, ok := cfg.Commands["python"]; !ok { + t.Error("Commands: missing python sandbox") + } + if _, ok := cfg.Commands["python"]["run"]; !ok { + t.Error("Commands[python]: missing run command") + } +} diff --git a/config/testdata/boxes.json b/config/testdata/boxes.json new file mode 100644 index 0000000..f44aead --- /dev/null +++ b/config/testdata/boxes.json @@ -0,0 +1,5 @@ +{ + "python": { + "image": "codapi/python" + } +} diff --git a/config/testdata/commands.json b/config/testdata/commands.json new file mode 100644 index 0000000..0e425c3 --- /dev/null +++ b/config/testdata/commands.json @@ -0,0 +1,25 @@ +{ + "python": { + "run": { + "engine": "docker", + "entry": "main.py", + "steps": [ + { + "box": "python", + "command": ["python", "main.py"] + } + ] + }, + "test": { + "engine": "docker", + "entry": "test_main.py", + "steps": [ + { + "box": "python", + "command": ["python", "-m", "unittest"], + "noutput": 8192 + } + ] + } + } +} diff --git a/config/testdata/config.json b/config/testdata/config.json new file mode 100644 index 0000000..e84b95b --- /dev/null +++ b/config/testdata/config.json @@ -0,0 +1,10 @@ +{ + "pool_size": 8, + "verbose": true, + "box": { + "memory": 64 + }, + "step": { + "user": "sandbox" + } +} diff --git a/docs/docker-xfs.md b/docs/docker-xfs.md new file mode 100644 index 0000000..5b86753 --- /dev/null +++ b/docs/docker-xfs.md @@ -0,0 +1,70 @@ +# XFS filesystem for Docker + +1. Install the necessary packages: + +```bash +sudo apt-get update +sudo apt-get install xfsprogs +``` + +2. Identify the disk or partition you want to use for the new filesystem. You can use the `lsblk` or `fdisk -l` command to list the available disks and partitions. Make sure you select the correct one as this process will erase all data on it. + +3. If the partition you want to use is not formatted, format it with an XFS filesystem. Replace `/dev/sdX` with the appropriate device identifier: + +```bash +sudo mkfs.xfs /dev/sdX +``` + +4. Once the partition is formatted, create a mount point. This will be the directory where the new filesystem will be mounted: + +```bash +sudo mkdir /mnt/docker +``` + +5. Update the `/etc/fstab` file to automatically mount the new filesystem at boot: + +``` +/dev/sdX /mnt/docker xfs defaults,nofail,discard,noatime,quota,prjquota,pquota,gquota 0 2 +``` + +6. Mount the new filesystem and verify that it is working: + +```bash +sudo mount -a +df -h +``` + +The output of `df -h` should show the new filesystem mounted at `/mnt/docker`. + +7. Stop the docker daemon: + +```bash +systemctl stop docker +``` + +8. Update the `/etc/docker/daemon.json` file to point docker to the new mount point: + +```json +{ + "data-root": "/mnt/docker" +} +``` + +9. Start the docker daemon: + +```bash +systemctl start docker +``` + +10. Build the images: + +```bash +su - codapi +make images +``` + +11. Verify that docker can now limit the storage size: + +```bash +docker run -it --storage-opt size=16m codapi/alpine /bin/df -h | grep overlay +``` diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000..0cbb380 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,101 @@ +# Installing Codapi + +Steps for Debian (11/12) or Ubuntu (20.04/22.04). + +1. Install necessary packages (as root): + +```sh +apt update && apt install -y ca-certificates curl docker.io make unzip +systemctl enable docker.service +systemctl restart docker.service +``` + +2. Create Codapi user (as root): + +```sh +useradd --groups docker --shell /usr/bin/bash --create-home --home /opt/codapi codapi +``` + +3. Verify that Docker is working (as codapi): + +```sh +docker run hello-world +``` + +4. Install Codapi (as codapi): + +```sh +cd /opt/codapi +curl -L -o codapi.zip "https://api.github.com/repos/nalgeon/codapi/actions/artifacts/926428361/zip" +unzip -u codapi.zip +chmod +x build/codapi +rm -f codapi.zip +``` + +6. Build Docker images (as codapi): + +```sh +cd /opt/codapi +make images +``` + +7. Verify that Codapi starts without errors (as codapi): + +```sh +cd /opt/codapi +./build/codapi +``` + +Should print the `alpine` box and the `sh` command: + +``` +2023/09/16 15:18:05 codapi 20230915:691d224 +2023/09/16 15:18:05 listening on port 1313... +2023/09/16 15:18:05 workers: 8 +2023/09/16 15:18:05 boxes: [alpine] +2023/09/16 15:18:05 commands: [sh] +``` + +Stop it with Ctrl+C. + +8. Configure Codapi as systemd service (as root): + +```sh +mv /opt/codapi/codapi.service /etc/systemd/system/ +chown root:root /etc/systemd/system/codapi.service +systemctl enable codapi.service +systemctl start codapi.service +``` + +Verify that the Codapi service is running: + +```sh +systemctl status codapi.service +``` + +Should print `active (running)`: + +``` +codapi.service - Code playgrounds + Loaded: loaded (/etc/systemd/system/codapi.service; enabled; preset: enabled) + Active: active (running) +... +``` + +9. Verify that Codapi is working: + +```sh +curl -H "content-type: application/json" -d '{ "sandbox": "sh", "command": "run", "files": {"": "echo hello" }}' http://localhost:1313/v1/exec +``` + +Should print `ok` = `true`: + +```json +{ + "id": "sh_run_dd27ed27", + "ok": true, + "duration": 650, + "stdout": "hello\n", + "stderr": "" +} +``` diff --git a/engine/docker.go b/engine/docker.go new file mode 100644 index 0000000..d577931 --- /dev/null +++ b/engine/docker.go @@ -0,0 +1,297 @@ +// Execute commands using Docker. +package engine + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/nalgeon/codapi/config" + "github.com/nalgeon/codapi/execy" + "github.com/nalgeon/codapi/fileio" + "github.com/nalgeon/codapi/logx" +) + +var killTimeout = 5 * time.Second + +const ( + actionRun = "run" + actionExec = "exec" +) + +// A Docker engine executes a specific sandbox command +// using Docker `run` or `exec` actions. +type Docker struct { + cfg *config.Config + cmd *config.Command +} + +// NewDocker creates a new Docker engine for a specific command. +func NewDocker(cfg *config.Config, sandbox, command string) Engine { + cmd := cfg.Commands[sandbox][command] + return &Docker{cfg, cmd} +} + +// Exec executes the command and returns the output. +func (e *Docker) Exec(req Request) Execution { + // all steps operate in the same temp directory + dir, err := os.MkdirTemp("", "") + if err != nil { + err = NewExecutionError("create temp dir", err) + return Fail(req.ID, err) + } + defer os.RemoveAll(dir) + + // if the command entry point file is not defined, + // there is no need to store request files in the temp directory + if e.cmd.Entry != "" { + // write request files to the temp directory + err = e.writeFiles(dir, req.Files) + if err != nil { + err = NewExecutionError("write files to temp dir", err) + return Fail(req.ID, err) + } + } + + // initialization step + if e.cmd.Before != nil { + out := e.execStep(e.cmd.Before, req.ID, dir, nil) + if !out.OK { + return out + } + } + + // the first step is required + first, rest := e.cmd.Steps[0], e.cmd.Steps[1:] + out := e.execStep(first, req.ID, dir, req.Files) + + // the rest are optional + if out.OK && len(rest) > 0 { + // each step operates on the results of the previous one, + // without using the source files - hence `nil` instead of `files` + for _, step := range rest { + out = e.execStep(step, req.ID, dir, nil) + if !out.OK { + break + } + } + } + + // cleanup step + if e.cmd.After != nil { + afterOut := e.execStep(e.cmd.After, req.ID, dir, nil) + if out.OK && !afterOut.OK { + return afterOut + } + } + + return out +} + +// execStep executes a step using the docker container. +func (e *Docker) execStep(step *config.Step, reqID, dir string, files Files) Execution { + box := e.cfg.Boxes[step.Box] + err := e.copyFiles(box, dir) + if err != nil { + err = NewExecutionError("copy files to temp dir", err) + return Fail(reqID, err) + } + + stdout, stderr, err := e.exec(box, step, reqID, dir, files) + if err != nil { + return Fail(reqID, err) + } + + return Execution{ + ID: reqID, + OK: true, + Stdout: stdout, + Stderr: stderr, + } +} + +// copyFiles copies box files to the temporary directory. +func (e *Docker) copyFiles(box *config.Box, dir string) error { + if box == nil || len(box.Files) == 0 { + return nil + } + for _, pattern := range box.Files { + err := fileio.CopyFiles(pattern, dir) + if err != nil { + return err + } + } + return nil +} + +// writeFiles writes request files to the temporary directory. +func (e *Docker) writeFiles(dir string, files Files) error { + var err error + files.Range(func(name, content string) bool { + if name == "" { + name = e.cmd.Entry + } + path := filepath.Join(dir, name) + err = os.WriteFile(path, []byte(content), 0444) + return err == nil + }) + return err +} + +// exec executes the step in the docker container +// using the files from in the temporary directory. +func (e *Docker) exec(box *config.Box, step *config.Step, reqID, dir string, files Files) (stdout string, stderr string, err error) { + // limit the stdout/stderr size + prog := NewProgram(step.Timeout, int64(step.NOutput)) + args := e.buildArgs(box, step, reqID, dir) + + if step.Stdin { + // pass files to container from stdin + stdin := filesReader(files) + stdout, stderr, err = prog.RunStdin(stdin, reqID, "docker", args...) + } else { + // pass files to container from temp directory + stdout, stderr, err = prog.Run(reqID, "docker", args...) + } + + if err == nil { + // success + return + } + + if err.Error() == "signal: killed" { + if step.Action == actionRun { + // we have to "docker kill" the container here, because the proccess + // inside the container is not related to the "docker run" process, + // and will hang forever after the "docker run" process is killed + go func() { + err = dockerKill(reqID) + if err == nil { + logx.Debug("%s: docker kill ok", reqID) + } else { + logx.Log("%s: docker kill failed: %v", reqID, err) + } + }() + } + // context timeout + err = ErrTimeout + return + } + + exitErr := new(exec.ExitError) + if errors.As(err, &exitErr) { + // the problem (if any) is the code, not the execution + // so we return the error without wrapping into ExecutionError + stderr, stdout = stdout+stderr, "" + if stderr != "" { + err = fmt.Errorf("%s (%s)", stderr, err) + } + return + } + + // other execution error + err = NewExecutionError("execute code", err) + return +} + +// buildArgs prepares the arguments for the `docker` command. +func (e *Docker) buildArgs(box *config.Box, step *config.Step, name, dir string) []string { + var args []string + if step.Action == actionRun { + args = dockerRunArgs(box, step, name, dir) + } else if step.Action == actionExec { + args = dockerExecArgs(step) + } else { + // should never happen if the config is valid + args = []string{"version"} + } + + command := expandVars(step.Command, name) + args = append(args, command...) + logx.Debug("%v", args) + return args +} + +// buildArgs prepares the arguments for the `docker run` command. +func dockerRunArgs(box *config.Box, step *config.Step, name, dir string) []string { + args := []string{ + actionRun, "--rm", + "--name", name, + "--runtime", box.Runtime, + "--cpus", strconv.Itoa(box.CPU), + "--memory", fmt.Sprintf("%dm", box.Memory), + "--network", box.Network, + "--pids-limit", strconv.Itoa(box.NProc), + "--user", step.User, + } + if !box.Writable { + args = append(args, "--read-only") + } + if step.Stdin { + args = append(args, "--interactive") + } + if box.Storage != "" { + args = append(args, "--storage-opt", fmt.Sprintf("size=%s", box.Storage)) + } + if dir != "" { + args = append(args, "--volume", fmt.Sprintf(box.Volume, dir)) + } + for _, fs := range box.Tmpfs { + args = append(args, "--tmpfs", fs) + } + for _, cap := range box.CapAdd { + args = append(args, "--cap-add", cap) + } + for _, cap := range box.CapDrop { + args = append(args, "--cap-drop", cap) + } + for _, lim := range box.Ulimit { + args = append(args, "--ulimit", lim) + } + args = append(args, box.Image) + return args +} + +// dockerExecArgs prepares the arguments for the `docker exec` command. +func dockerExecArgs(step *config.Step) []string { + return []string{ + actionExec, "--interactive", + "--user", step.User, + step.Box, + } +} + +// filesReader creates a reader over an in-memory collection of files. +func filesReader(files Files) io.Reader { + var input strings.Builder + for _, content := range files { + input.WriteString(content) + } + return strings.NewReader(input.String()) +} + +// expandVars replaces variables in command arguments with values. +// The only supported variable is :name = container name. +func expandVars(command []string, name string) []string { + expanded := make([]string, len(command)) + copy(expanded, command) + for i, cmd := range expanded { + expanded[i] = strings.Replace(cmd, ":name", name, 1) + } + return expanded +} + +// dockerKill kills the container with the specified id/name. +func dockerKill(id string) error { + ctx, cancel := context.WithTimeout(context.Background(), killTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "docker", "kill", id) + return execy.Run(cmd) +} diff --git a/engine/docker_test.go b/engine/docker_test.go new file mode 100644 index 0000000..89dff9b --- /dev/null +++ b/engine/docker_test.go @@ -0,0 +1,164 @@ +package engine + +import ( + "strings" + "testing" + + "github.com/nalgeon/codapi/config" + "github.com/nalgeon/codapi/execy" + "github.com/nalgeon/codapi/logx" +) + +var dockerCfg = &config.Config{ + Boxes: map[string]*config.Box{ + "postgresql": { + Image: "codapi/postgresql", + Runtime: "runc", + Host: config.Host{ + CPU: 1, Memory: 64, Network: "none", + Volume: "%s:/sandbox:ro", + NProc: 64, + }, + }, + "python": { + Image: "codapi/python", + Runtime: "runc", + Host: config.Host{ + CPU: 1, Memory: 64, Network: "none", + Volume: "%s:/sandbox:ro", + NProc: 64, + }, + }, + }, + Commands: map[string]config.SandboxCommands{ + "postgresql": map[string]*config.Command{ + "run": { + Engine: "docker", + Before: &config.Step{ + Box: "postgres", User: "sandbox", Action: "exec", + Command: []string{"psql", "-f", "create.sql"}, + NOutput: 4096, + }, + Steps: []*config.Step{ + { + Box: "postgres", User: "sandbox", Action: "exec", Stdin: true, + Command: []string{"psql", "--user=:name"}, + NOutput: 4096, + }, + }, + After: &config.Step{ + Box: "postgres", User: "sandbox", Action: "exec", + Command: []string{"psql", "-f", "drop.sql"}, + NOutput: 4096, + }, + }, + }, + "python": map[string]*config.Command{ + "run": { + Engine: "docker", + Entry: "main.py", + Steps: []*config.Step{ + { + Box: "python", User: "sandbox", Action: "run", + Command: []string{"python", "main.py"}, + NOutput: 4096, + }, + }, + }, + }, + }, +} + +func TestDockerRun(t *testing.T) { + logx.Mock() + commands := map[string]execy.CmdOut{ + "docker run": {Stdout: "hello world", Stderr: "", Err: nil}, + } + mem := execy.Mock(commands) + engine := NewDocker(dockerCfg, "python", "run") + + t.Run("success", func(t *testing.T) { + req := Request{ + ID: "http_42", + Sandbox: "python", + Command: "run", + Files: map[string]string{ + "": "print('hello world')", + }, + } + out := engine.Exec(req) + if out.ID != req.ID { + t.Errorf("ID: expected %s, got %s", req.ID, out.ID) + } + if !out.OK { + t.Error("OK: expected true") + } + want := "hello world" + if out.Stdout != want { + t.Errorf("Stdout: expected %q, got %q", want, out.Stdout) + } + if out.Stderr != "" { + t.Errorf("Stderr: expected %q, got %q", "", out.Stdout) + } + if out.Err != nil { + t.Errorf("Err: expected nil, got %v", out.Err) + } + mem.MustHave(t, "python main.py") + }) +} + +func TestDockerExec(t *testing.T) { + logx.Mock() + commands := map[string]execy.CmdOut{ + "docker exec": {Stdout: "hello world", Stderr: "", Err: nil}, + } + mem := execy.Mock(commands) + engine := NewDocker(dockerCfg, "postgresql", "run") + + t.Run("success", func(t *testing.T) { + req := Request{ + ID: "http_42", + Sandbox: "postgresql", + Command: "run", + Files: map[string]string{ + "": "select 'hello world'", + }, + } + out := engine.Exec(req) + if out.ID != req.ID { + t.Errorf("ID: expected %s, got %s", req.ID, out.ID) + } + if !out.OK { + t.Error("OK: expected true") + } + want := "hello world" + if out.Stdout != want { + t.Errorf("Stdout: expected %q, got %q", want, out.Stdout) + } + if out.Stderr != "" { + t.Errorf("Stderr: expected %q, got %q", "", out.Stdout) + } + if out.Err != nil { + t.Errorf("Err: expected nil, got %v", out.Err) + } + mem.MustHave(t, "psql -f create.sql") + mem.MustHave(t, "psql --user=http_42") + mem.MustHave(t, "psql -f drop.sql") + }) +} + +func Test_expandVars(t *testing.T) { + const name = "codapi_01" + commands := map[string]string{ + "python main.py": "python main.py", + "sh create.sh :name": "sh create.sh " + name, + } + for cmd, want := range commands { + src := strings.Fields(cmd) + exp := expandVars(src, name) + got := strings.Join(exp, " ") + if got != want { + t.Errorf("%q: expected %q, got %q", cmd, got, want) + } + } +} diff --git a/engine/engine.go b/engine/engine.go new file mode 100644 index 0000000..0252350 --- /dev/null +++ b/engine/engine.go @@ -0,0 +1,116 @@ +// Package engine provides code execution engines. +package engine + +import ( + "errors" + "fmt" + + "github.com/nalgeon/codapi/stringx" +) + +// A Request initiates code execution. +type Request struct { + ID string `json:"id"` + Sandbox string `json:"sandbox"` + Command string `json:"command"` + Files Files `json:"files"` +} + +// GenerateID() sets a unique ID for the request. +func (r *Request) GenerateID() { + r.ID = fmt.Sprintf("%s_%s_%s", r.Sandbox, r.Command, stringx.RandString(8)) +} + +// An Execution is an output from the code execution engine. +type Execution struct { + ID string `json:"id"` + OK bool `json:"ok"` + Duration int `json:"duration"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + Err error `json:"-"` +} + +// An ErrTimeout is returned if code execution did not complete +// in the allowed timeframe. +var ErrTimeout = errors.New("code execution timeout") + +// An ErrBusy is returned when there are no engines available. +var ErrBusy = errors.New("busy: try again later") + +// An ExecutionError is returned if code execution failed +// due to the application problems, not due to the problems with the code. +type ExecutionError struct { + msg string + inner error +} + +func NewExecutionError(msg string, err error) ExecutionError { + return ExecutionError{msg: msg, inner: err} +} + +func (err ExecutionError) Error() string { + return err.msg + ": " + err.inner.Error() +} + +func (err ExecutionError) Unwrap() error { + return err.inner +} + +// Files are a collection of files to be executed by the engine. +type Files map[string]string + +// First returns the contents of the first file. +func (f Files) First() string { + for _, content := range f { + return content + } + return "" +} + +// Range iterates over the files, calling fn for each one. +func (f Files) Range(fn func(name, content string) bool) { + for name, content := range f { + ok := fn(name, content) + if !ok { + break + } + } +} + +// Count returns the number of files. +func (f Files) Count() int { + return len(f) +} + +// An Engine executes a specific sandbox command on the code. +// Engines must be concurrent-safe, since they can be accessed by multiple goroutines. +type Engine interface { + // Exec executes the command and returns the output. + Exec(req Request) Execution +} + +// Fail creates an output from an error. +func Fail(id string, err error) Execution { + if _, ok := err.(ExecutionError); ok { + return Execution{ + ID: id, + OK: false, + Stderr: "internal error", + Err: err, + } + } + if errors.Is(err, ErrBusy) { + return Execution{ + ID: id, + OK: false, + Stderr: err.Error(), + Err: err, + } + } + return Execution{ + ID: id, + OK: false, + Stderr: err.Error(), + } +} diff --git a/engine/engine_test.go b/engine/engine_test.go new file mode 100644 index 0000000..e14afb2 --- /dev/null +++ b/engine/engine_test.go @@ -0,0 +1,137 @@ +package engine + +import ( + "errors" + "reflect" + "sort" + "testing" +) + +func TestExecutionError(t *testing.T) { + inner := errors.New("inner error") + err := NewExecutionError("failed", inner) + if err.Error() != "failed: inner error" { + t.Errorf("Error: expected %q, got %q", "failed: inner error", err.Error()) + } + unwrapped := err.Unwrap() + if unwrapped != inner { + t.Errorf("Unwrap: expected %#v, got %#v", inner, unwrapped) + } +} + +func TestFiles_Count(t *testing.T) { + var files Files = map[string]string{ + "first": "alice", + "second": "bob", + "third": "cindy", + } + if files.Count() != 3 { + t.Errorf("Count: expected 3, got %d", files.Count()) + } +} + +func TestFiles_Range(t *testing.T) { + var files Files = map[string]string{ + "first": "alice", + "second": "bob", + "third": "cindy", + } + + t.Run("range", func(t *testing.T) { + names := []string{} + contents := []string{} + files.Range(func(name, content string) bool { + names = append(names, name) + contents = append(contents, content) + return true + }) + sort.Strings(names) + if !reflect.DeepEqual(names, []string{"first", "second", "third"}) { + t.Errorf("unexpected names: %v", names) + } + sort.Strings(contents) + if !reflect.DeepEqual(contents, []string{"alice", "bob", "cindy"}) { + t.Errorf("unexpected contents: %v", contents) + } + }) + + t.Run("break", func(t *testing.T) { + names := []string{} + contents := []string{} + files.Range(func(name, content string) bool { + names = append(names, name) + contents = append(contents, content) + return false + }) + if len(names) != 1 { + t.Fatalf("expected names len = 1, got %d", len(names)) + } + if len(contents) != 1 { + t.Fatalf("expected contents len = 1, got %d", len(contents)) + } + if files[names[0]] != contents[0] { + t.Fatalf("name does not match content: %v -> %v", names[0], contents[0]) + } + }) +} + +func TestFail(t *testing.T) { + t.Run("ExecutionError", func(t *testing.T) { + err := NewExecutionError("failed", errors.New("inner error")) + out := Fail("42", err) + if out.ID != "42" { + t.Errorf("ID: expected 42, got %v", out.ID) + } + if out.OK { + t.Error("OK: expected false") + } + if out.Stderr != "internal error" { + t.Errorf("Stderr: expected %q, got %q", "internal error", out.Stderr) + } + if out.Stdout != "" { + t.Errorf("Stdout: expected empty, got %q", out.Stdout) + } + if out.Err != err { + t.Errorf("Err: expected %#v, got %#v", err, out.Err) + } + }) + t.Run("ErrBusy", func(t *testing.T) { + err := ErrBusy + out := Fail("42", err) + if out.ID != "42" { + t.Errorf("ID: expected 42, got %v", out.ID) + } + if out.OK { + t.Error("OK: expected false") + } + if out.Stderr != err.Error() { + t.Errorf("Stderr: expected %q, got %q", err.Error(), out.Stderr) + } + if out.Stdout != "" { + t.Errorf("Stdout: expected empty, got %q", out.Stdout) + } + if out.Err != err { + t.Errorf("Err: expected %#v, got %#v", err, out.Err) + } + }) + t.Run("Error", func(t *testing.T) { + err := errors.New("user error") + out := Fail("42", err) + if out.ID != "42" { + t.Errorf("ID: expected 42, got %v", out.ID) + } + if out.OK { + t.Error("OK: expected false") + } + if out.Stderr != err.Error() { + t.Errorf("Stderr: expected %q, got %q", err.Error(), out.Stderr) + } + if out.Stdout != "" { + t.Errorf("Stdout: expected empty, got %q", out.Stdout) + } + if out.Err != nil { + t.Errorf("Err: expected nil, got %#v", out.Err) + } + }) + +} diff --git a/engine/exec.go b/engine/exec.go new file mode 100644 index 0000000..7fc0dd2 --- /dev/null +++ b/engine/exec.go @@ -0,0 +1,54 @@ +package engine + +import ( + "context" + "io" + "os/exec" + "strings" + "time" + + "github.com/nalgeon/codapi/execy" + "github.com/nalgeon/codapi/logx" +) + +// A Program is an executable program. +type Program struct { + timeout time.Duration + nOutput int64 +} + +// NewProgram creates a new program. +func NewProgram(timeoutSec int, nOutput int64) *Program { + return &Program{ + timeout: time.Duration(timeoutSec) * time.Second, + nOutput: nOutput, + } +} + +// Run starts the program and waits for it to complete (or timeout). +func (p *Program) Run(id, name string, arg ...string) (stdout string, stderr string, err error) { + return p.RunStdin(nil, id, name, arg...) +} + +// RunStdin starts the program with data from stdin +// and waits for it to complete (or timeout). +func (p *Program) RunStdin(stdin io.Reader, id, name string, arg ...string) (stdout string, stderr string, err error) { + ctx, cancel := context.WithTimeout(context.Background(), p.timeout) + defer cancel() + + var cmdout, cmderr strings.Builder + cmd := exec.CommandContext(ctx, name, arg...) + cmd.Cancel = func() error { + err := cmd.Process.Kill() + logx.Debug("%s: execution timeout, killed process=%d, err=%v", id, cmd.Process.Pid, err) + return err + } + + cmd.Stdin = stdin + cmd.Stdout = LimitWriter(&cmdout, p.nOutput) + cmd.Stderr = LimitWriter(&cmderr, p.nOutput) + err = execy.Run(cmd) + stdout = cmdout.String() + stderr = cmderr.String() + return +} diff --git a/engine/exec_test.go b/engine/exec_test.go new file mode 100644 index 0000000..3771785 --- /dev/null +++ b/engine/exec_test.go @@ -0,0 +1,74 @@ +package engine + +import ( + "errors" + "strings" + "testing" + + "github.com/nalgeon/codapi/execy" +) + +func TestProgram_Run(t *testing.T) { + commands := map[string]execy.CmdOut{ + "mock stdout": {Stdout: "stdout", Stderr: "", Err: nil}, + "mock stderr": {Stdout: "", Stderr: "stderr", Err: nil}, + "mock outerr": {Stdout: "stdout", Stderr: "stderr", Err: nil}, + "mock err": {Stdout: "", Stderr: "stderr", Err: errors.New("error")}, + } + mem := execy.Mock(commands) + + for key, want := range commands { + t.Run(key, func(t *testing.T) { + p := NewProgram(3, 100) + name, arg, _ := strings.Cut(key, " ") + stdout, stderr, err := p.Run("mock_42", name, arg) + if !mem.Has(key) { + t.Errorf("Run: command %q not run", key) + } + if stdout != want.Stdout { + t.Errorf("stdout: want %#v, got %#v", want.Stdout, stdout) + } + if stderr != want.Stderr { + t.Errorf("stderr: want %#v, got %#v", want.Stderr, stderr) + } + if err != want.Err { + t.Errorf("err: want %#v, got %#v", want.Err, err) + } + }) + } +} + +func TestProgram_LimitOutput(t *testing.T) { + commands := map[string]execy.CmdOut{ + "mock stdout": {Stdout: "1234567890", Stderr: ""}, + "mock stderr": {Stdout: "", Stderr: "1234567890"}, + "mock outerr": {Stdout: "1234567890", Stderr: "1234567890"}, + } + execy.Mock(commands) + + const nOutput = 5 + { + p := NewProgram(3, nOutput) + stdout, _, _ := p.Run("mock_42", "mock", "stdout") + if stdout != "12345" { + t.Errorf("stdout: want %#v, got %#v", "12345", stdout) + } + } + { + p := NewProgram(3, nOutput) + _, stderr, _ := p.Run("mock_42", "mock", "stderr") + if stderr != "12345" { + t.Errorf("stderr: want %#v, got %#v", "12345", stderr) + } + } + { + p := NewProgram(3, nOutput) + stdout, stderr, _ := p.Run("mock_42", "mock", "outerr") + if stdout != "12345" { + t.Errorf("stdout: want %#v, got %#v", "12345", stdout) + } + if stderr != "12345" { + t.Errorf("stderr: want %#v, got %#v", "12345", stderr) + } + } +} diff --git a/engine/http.go b/engine/http.go new file mode 100644 index 0000000..5a0e307 --- /dev/null +++ b/engine/http.go @@ -0,0 +1,166 @@ +// Send HTTP request according to the specification. +package engine + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/nalgeon/codapi/config" + "github.com/nalgeon/codapi/httpx" + "github.com/nalgeon/codapi/logx" +) + +// An HTTP engine sends HTTP requests. +type HTTP struct { + hosts map[string]string +} + +// NewHTTP creates a new HTTP engine. +func NewHTTP(cfg *config.Config, sandbox, command string) Engine { + if len(cfg.HTTP.Hosts) == 0 { + msg := fmt.Sprintf("%s %s: http engine requires at least one allowed URL", sandbox, command) + panic(msg) + } + return &HTTP{hosts: cfg.HTTP.Hosts} +} + +// Exec sends an HTTP request according to the spec +// and returns the response as text with status, headers and body. +func (e *HTTP) Exec(req Request) Execution { + // build request from spec + httpReq, err := e.parse(req.Files.First()) + if err != nil { + err = fmt.Errorf("parse spec: %w", err) + return Fail(req.ID, err) + } + + // send request and receive response + allowed := e.translateHost(httpReq) + if !allowed { + err = fmt.Errorf("host not allowed: %s", httpReq.Host) + return Fail(req.ID, err) + } + + logx.Log("%s: %s %s", req.ID, httpReq.Method, httpReq.URL.String()) + resp, err := httpx.Do(httpReq) + if err != nil { + err = fmt.Errorf("http request: %w", err) + return Fail(req.ID, err) + } + defer resp.Body.Close() + + // read response body + body, err := io.ReadAll(resp.Body) + if err != nil { + err = NewExecutionError("read response", err) + return Fail(req.ID, err) + } + + // build text representation of request + stdout := e.responseText(resp, body) + return Execution{ + ID: req.ID, + OK: true, + Stdout: stdout, + } +} + +// parse parses the request specification. +func (e *HTTP) parse(text string) (*http.Request, error) { + lines := strings.Split(text, "\n") + if len(lines) == 0 { + return nil, errors.New("empty request") + } + + lineIdx := 0 + + // parse method and URL + var method, url string + methodURL := strings.Fields(lines[0]) + if len(methodURL) >= 2 { + method = methodURL[0] + url = methodURL[1] + } else { + method = http.MethodGet + url = methodURL[0] + } + + lineIdx++ + + // parse URL parameters + var urlParams strings.Builder + for i := lineIdx; i < len(lines); i++ { + line := strings.TrimSpace(lines[i]) + if strings.HasPrefix(line, "?") || strings.HasPrefix(line, "&") { + urlParams.WriteString(line) + lineIdx++ + } else { + break + } + } + + // parse headers + headers := make(http.Header) + for i := lineIdx; i < len(lines); i++ { + line := strings.TrimSpace(lines[i]) + if line == "" { + break + } + headerParts := strings.SplitN(line, ":", 2) + if len(headerParts) == 2 { + headers.Add(strings.TrimSpace(headerParts[0]), strings.TrimSpace(headerParts[1])) + lineIdx++ + } + } + + lineIdx += 1 + + // parse body + var bodyRdr io.Reader + if lineIdx < len(lines) { + body := strings.Join(lines[lineIdx:], "\n") + bodyRdr = strings.NewReader(body) + } + + // create request + req, err := http.NewRequest(method, url+urlParams.String(), bodyRdr) + if err != nil { + return nil, err + } + req.Header = headers + return req, nil +} + +// translateHost translates the requested host into the allowed one. +// Returns false if the requested host is not allowed. +func (e *HTTP) translateHost(req *http.Request) bool { + host := e.hosts[req.Host] + if host == "" { + return false + } + req.URL.Host = host + return true +} + +// responseText returns the response as text with status, headers and body. +func (e *HTTP) responseText(resp *http.Response, body []byte) string { + var b bytes.Buffer + // status line + b.WriteString( + fmt.Sprintf("%s %d %s\n", resp.Proto, resp.StatusCode, http.StatusText(resp.StatusCode)), + ) + // headers + for name := range resp.Header { + b.WriteString(fmt.Sprintf("%s: %s\n", name, resp.Header.Get(name))) + } + // body + if len(body) > 0 { + b.WriteByte('\n') + b.Write(body) + } + return b.String() +} diff --git a/engine/http_test.go b/engine/http_test.go new file mode 100644 index 0000000..2961f78 --- /dev/null +++ b/engine/http_test.go @@ -0,0 +1,179 @@ +package engine + +import ( + "io" + "net/http" + "testing" + + "github.com/nalgeon/codapi/config" + "github.com/nalgeon/codapi/httpx" + "github.com/nalgeon/codapi/logx" +) + +var httpCfg = &config.Config{ + HTTP: &config.HTTP{ + Hosts: map[string]string{"codapi.org": "localhost"}, + }, + Commands: map[string]config.SandboxCommands{ + "http": map[string]*config.Command{ + "run": {Engine: "http"}, + }, + }, +} + +func TestHTTP_Exec(t *testing.T) { + logx.Mock() + httpx.Mock() + engine := NewHTTP(httpCfg, "http", "run") + + t.Run("success", func(t *testing.T) { + req := Request{ + ID: "http_42", + Sandbox: "http", + Command: "run", + Files: map[string]string{ + "": "GET https://codapi.org/example.txt", + }, + } + out := engine.Exec(req) + if out.ID != req.ID { + t.Errorf("ID: expected %s, got %s", req.ID, out.ID) + } + if !out.OK { + t.Error("OK: expected true") + } + want := `HTTP/1.1 200 OK +Content-Type: text/plain + +hello` + if out.Stdout != want { + t.Errorf("Stdout: expected %q, got %q", want, out.Stdout) + } + if out.Stderr != "" { + t.Errorf("Stderr: expected %q, got %q", "", out.Stdout) + } + if out.Err != nil { + t.Errorf("Err: expected nil, got %#v", out.Err) + } + }) + t.Run("hostname not allowed", func(t *testing.T) { + req := Request{ + ID: "http_42", + Sandbox: "http", + Command: "run", + Files: map[string]string{ + "": "GET https://example.com/get", + }, + } + out := engine.Exec(req) + if out.Err != nil { + t.Errorf("Err: expected nil, got %#v", out.Err) + } + if out.Stderr != "host not allowed: example.com" { + t.Errorf("Stderr: unexpected value %q", out.Stderr) + } + }) +} + +func TestHTTP_parse(t *testing.T) { + logx.Mock() + httpx.Mock() + engine := NewHTTP(httpCfg, "http", "run").(*HTTP) + + t.Run("request line", func(t *testing.T) { + const uri = "https://codapi.org/head" + text := "HEAD " + uri + req, err := engine.parse(text) + if err != nil { + t.Fatalf("unexpected error %#v", err) + } + if req.Method != http.MethodHead { + t.Errorf("Method: expected %s, got %s", http.MethodHead, req.Method) + } + if req.URL.String() != uri { + t.Errorf("URL: expected %q, got %q", uri, req.URL.String()) + } + }) + t.Run("headers", func(t *testing.T) { + const uri = "https://codapi.org/get" + text := "GET " + uri + "\naccept: text/plain\nx-secret: 42" + req, err := engine.parse(text) + if err != nil { + t.Fatalf("unexpected error %#v", err) + } + if req.Method != http.MethodGet { + t.Errorf("Method: expected %s, got %s", http.MethodGet, req.Method) + } + if req.URL.String() != uri { + t.Errorf("URL: expected %q, got %q", uri, req.URL.String()) + } + if len(req.Header) != 2 { + t.Fatalf("Header: expected 2 headers, got %d", len(req.Header)) + } + if req.Header.Get("accept") != "text/plain" { + t.Fatalf("Header: expected accept = %q, got %q", "text/plain", req.Header.Get("accept")) + } + if req.Header.Get("x-secret") != "42" { + t.Fatalf("Header: expected x-secret = %q, got %q", "42", req.Header.Get("x-secret")) + } + }) + t.Run("body", func(t *testing.T) { + const uri = "https://codapi.org/post" + const body = "{\"name\":\"alice\"}" + text := "POST " + uri + "\ncontent-type: application/json\n\n" + body + req, err := engine.parse(text) + if err != nil { + t.Fatalf("unexpected error %#v", err) + } + if req.Method != http.MethodPost { + t.Errorf("Method: expected %s, got %s", http.MethodPost, req.Method) + } + if req.URL.String() != uri { + t.Errorf("URL: expected %q, got %q", uri, req.URL.String()) + } + if req.Header.Get("content-type") != "application/json" { + t.Errorf("Header: expected content-type = %q, got %q", + "application/json", req.Header.Get("content-type")) + } + b, _ := io.ReadAll(req.Body) + got := string(b) + if got != body { + t.Errorf("Body: expected %q, got %q", body, got) + } + }) + t.Run("invalid", func(t *testing.T) { + _, err := engine.parse("on,e two three") + if err == nil { + t.Fatal("expected error, got nil") + } + }) +} + +func TestHTTP_translateHost(t *testing.T) { + logx.Mock() + httpx.Mock() + engine := NewHTTP(httpCfg, "http", "run").(*HTTP) + + t.Run("known url", func(t *testing.T) { + const uri = "http://codapi.org/get" + req, _ := http.NewRequest(http.MethodGet, uri, nil) + ok := engine.translateHost(req) + if !ok { + t.Errorf("%s: should be allowed", uri) + } + if req.URL.Hostname() != "localhost" { + t.Errorf("%s: expected %s, got %s", uri, "localhost", req.URL.Hostname()) + } + }) + t.Run("unknown url", func(t *testing.T) { + const uri = "http://example.com/get" + req, _ := http.NewRequest(http.MethodGet, uri, nil) + ok := engine.translateHost(req) + if ok { + t.Errorf("%s: should not be allowed", uri) + } + if req.URL.Hostname() != "example.com" { + t.Errorf("%s: expected %s, got %s", uri, "example.com", req.URL.Hostname()) + } + }) +} diff --git a/engine/io.go b/engine/io.go new file mode 100644 index 0000000..7cd6c85 --- /dev/null +++ b/engine/io.go @@ -0,0 +1,31 @@ +package engine + +import "io" + +// A LimitedWriter writes to w but limits the amount +// of data to only n bytes. After reaching the limit, +// silently discards the rest of the data without errors. +type LimitedWriter struct { + w io.Writer + n int64 +} + +// LimitWriter returns a writer that writes no more +// than n bytes and silently discards the rest. +func LimitWriter(w io.Writer, n int64) io.Writer { + return &LimitedWriter{w, n} +} + +// Write implements the io.Writer interface. +func (w *LimitedWriter) Write(p []byte) (int, error) { + lenp := len(p) + if w.n <= 0 { + return lenp, nil + } + if int64(lenp) > w.n { + p = p[:w.n] + } + n, err := w.w.Write(p) + w.n -= int64(n) + return lenp, err +} diff --git a/engine/io_test.go b/engine/io_test.go new file mode 100644 index 0000000..2d4181c --- /dev/null +++ b/engine/io_test.go @@ -0,0 +1,56 @@ +package engine + +import ( + "bytes" + "reflect" + "testing" +) + +func TestLimitedWriter(t *testing.T) { + var b bytes.Buffer + w := LimitWriter(&b, 5) + + { + src := []byte{1, 2, 3} + n, err := w.Write(src) + if n != 3 { + t.Fatalf("write(1,2,3): expected n = 3, got %d", n) + } + if err != nil { + t.Fatalf("write(1,2,3): expected nil err, got %v", err) + } + if !reflect.DeepEqual(b.Bytes(), src) { + t.Fatalf("write(1,2,3): expected %v, got %v", src, b.Bytes()) + } + } + + { + src := []byte{4, 5} + n, err := w.Write(src) + if n != 2 { + t.Fatalf("+write(4,5): expected n = 2, got %d", n) + } + if err != nil { + t.Fatalf("+write(4,5): expected nil err, got %v", err) + } + want := []byte{1, 2, 3, 4, 5} + if !reflect.DeepEqual(b.Bytes(), want) { + t.Fatalf("+write(4,5): expected %v, got %v", want, b.Bytes()) + } + } + + { + src := []byte{6, 7, 8} + n, err := w.Write(src) + if n != 3 { + t.Fatalf("+write(6,7,8): expected n = 3, got %d", n) + } + if err != nil { + t.Fatalf("+write(6,7,8): expected nil err, got %v", err) + } + want := []byte{1, 2, 3, 4, 5} + if !reflect.DeepEqual(b.Bytes(), want) { + t.Fatalf("+write(6,7,8): expected %v, got %v", want, b.Bytes()) + } + } +} diff --git a/engine/testdata/example.txt b/engine/testdata/example.txt new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/engine/testdata/example.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/execy/execy.go b/execy/execy.go new file mode 100644 index 0000000..0ad4703 --- /dev/null +++ b/execy/execy.go @@ -0,0 +1,31 @@ +// Package execy runs external commands. +package execy + +import ( + "os/exec" +) + +var runner = Runner(&osRunner{}) + +// Runner executes external commands. +type Runner interface { + Run(cmd *exec.Cmd) error +} + +// osRunner runs OS programs. +type osRunner struct{} + +func (r *osRunner) Run(cmd *exec.Cmd) error { + return cmd.Run() +} + +func Run(cmd *exec.Cmd) error { + return runner.Run(cmd) +} + +// CmdOut represents the result of the command run. +type CmdOut struct { + Stdout string + Stderr string + Err error +} diff --git a/execy/execy_test.go b/execy/execy_test.go new file mode 100644 index 0000000..280ba6c --- /dev/null +++ b/execy/execy_test.go @@ -0,0 +1,29 @@ +package execy + +import ( + "context" + "os/exec" + "strings" + "testing" +) + +func TestRunner(t *testing.T) { + const want = "hello world" + ctx := context.Background() + cmd := exec.CommandContext(ctx, "echo", "-n", want) + outb := new(strings.Builder) + errb := new(strings.Builder) + cmd.Stdout = outb + cmd.Stderr = errb + + err := Run(cmd) + if err != nil { + t.Fatalf("Err: expected nil, got %v", err) + } + if outb.String() != want { + t.Errorf("Stdout: expected %q, got %q", want, outb.String()) + } + if errb.String() != "" { + t.Errorf("Stderr: expected %q, got %q", "", errb.String()) + } +} diff --git a/execy/mock.go b/execy/mock.go new file mode 100644 index 0000000..a816ad9 --- /dev/null +++ b/execy/mock.go @@ -0,0 +1,44 @@ +package execy + +import ( + "os/exec" + "strings" + + "github.com/nalgeon/codapi/logx" +) + +// Mock installs mock outputs for given commands. +func Mock(commands map[string]CmdOut) *logx.Memory { + if commands != nil { + mockCommands = commands + } + mem := logx.NewMemory("exec") + runner = &mockRunner{mem} + return mem +} + +// mockRunner returns mock outputs +// without running OS programs. +type mockRunner struct { + mem *logx.Memory +} + +// Run returns a mock output from the registry +// that matches the given command name and argument. +func (r *mockRunner) Run(cmd *exec.Cmd) error { + cmdStr := strings.Join(cmd.Args, " ") + r.mem.WriteString(cmdStr) + + key := cmd.Args[0] + " " + cmd.Args[1] + out, ok := mockCommands[key] + if !ok { + // command is not in the registry, + // so let's return an empty "success" result + out = CmdOut{} + } + _, _ = cmd.Stdout.Write([]byte(out.Stdout)) + _, _ = cmd.Stderr.Write([]byte(out.Stderr)) + return out.Err +} + +var mockCommands map[string]CmdOut = map[string]CmdOut{} diff --git a/execy/mock_test.go b/execy/mock_test.go new file mode 100644 index 0000000..94a4deb --- /dev/null +++ b/execy/mock_test.go @@ -0,0 +1,33 @@ +package execy + +import ( + "context" + "os/exec" + "strings" + "testing" +) + +func TestMock(t *testing.T) { + const want = "hello world" + out := CmdOut{Stdout: want, Stderr: "", Err: nil} + mem := Mock(map[string]CmdOut{"echo -n": out}) + + ctx := context.Background() + cmd := exec.CommandContext(ctx, "echo", "-n", want) + outb := new(strings.Builder) + errb := new(strings.Builder) + cmd.Stdout = outb + cmd.Stderr = errb + + err := Run(cmd) + if err != nil { + t.Fatalf("Err: expected nil, got %v", err) + } + if outb.String() != want { + t.Errorf("Stdout: expected %q, got %q", want, outb.String()) + } + if errb.String() != "" { + t.Errorf("Stderr: expected %q, got %q", "", errb.String()) + } + mem.MustHave(t, "echo -n hello world") +} diff --git a/fileio/fileio.go b/fileio/fileio.go new file mode 100644 index 0000000..6c77a19 --- /dev/null +++ b/fileio/fileio.go @@ -0,0 +1,39 @@ +// Package fileio provides high-level file operations. +package fileio + +import ( + "io" + "os" + "path/filepath" +) + +// CopyFile copies all files matching the pattern +// to the destination directory. +func CopyFiles(pattern string, dstDir string) error { + matches, err := filepath.Glob(pattern) + if err != nil { + return err + } + + for _, match := range matches { + src, err := os.Open(match) + if err != nil { + return err + } + defer src.Close() + + dstFile := filepath.Join(dstDir, filepath.Base(match)) + dst, err := os.Create(dstFile) + if err != nil { + return err + } + defer dst.Close() + + _, err = io.Copy(dst, src) + if err != nil { + return err + } + } + + return nil +} diff --git a/fileio/fileio_test.go b/fileio/fileio_test.go new file mode 100644 index 0000000..8ac2f71 --- /dev/null +++ b/fileio/fileio_test.go @@ -0,0 +1,56 @@ +package fileio + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCopyFiles(t *testing.T) { + // Create a temporary directory for testing + srcDir, err := os.MkdirTemp("", "src") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(srcDir) + + // Create a source file + srcFile := filepath.Join(srcDir, "source.txt") + err = os.WriteFile(srcFile, []byte("test data"), 0644) + if err != nil { + t.Fatal(err) + } + + // Specify the destination directory + dstDir, err := os.MkdirTemp("", "dst") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dstDir) + + // Call the CopyFiles function + pattern := filepath.Join(srcDir, "*.txt") + err = CopyFiles(pattern, dstDir) + if err != nil { + t.Fatal(err) + } + + // Verify that the file was copied correctly + dstFile := filepath.Join(dstDir, "source.txt") + _, err = os.Stat(dstFile) + if err != nil { + t.Fatalf("file not copied: %s", err) + } + + // Read the contents of the copied file + data, err := os.ReadFile(dstFile) + if err != nil { + t.Fatal(err) + } + + // Verify the contents of the copied file + expected := []byte("test data") + if string(data) != string(expected) { + t.Errorf("unexpected file content: got %q, want %q", data, expected) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..202b675 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/nalgeon/codapi + +go 1.20 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e69de29 diff --git a/httpx/httpx.go b/httpx/httpx.go new file mode 100644 index 0000000..f7b1535 --- /dev/null +++ b/httpx/httpx.go @@ -0,0 +1,19 @@ +// Package httpx provides helper functions for making HTTP requests. +package httpx + +import ( + "net/http" + "time" +) + +var client = Client(&http.Client{Timeout: 5 * time.Second}) + +// Client is something that can send HTTP requests. +type Client interface { + Do(req *http.Request) (*http.Response, error) +} + +// Do sends an HTTP request and returns an HTTP response. +func Do(req *http.Request) (*http.Response, error) { + return client.Do(req) +} diff --git a/httpx/httpx_test.go b/httpx/httpx_test.go new file mode 100644 index 0000000..b5a0acd --- /dev/null +++ b/httpx/httpx_test.go @@ -0,0 +1,40 @@ +package httpx + +import ( + "net/http" + "testing" +) + +func TestDo(t *testing.T) { + srv := MockServer() + defer srv.Close() + + t.Run("ok", func(t *testing.T) { + uri := srv.URL + "/example.json" + req, _ := http.NewRequest(http.MethodGet, uri, nil) + + resp, err := Do(req) + if err != nil { + t.Errorf("Do: unexpected error %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Do: expected status=%d, got %v", http.StatusOK, resp.StatusCode) + } + }) + t.Run("not found", func(t *testing.T) { + uri := srv.URL + "/not-found.json" + req, _ := http.NewRequest(http.MethodGet, uri, nil) + + resp, err := Do(req) + if err != nil { + t.Errorf("Do: unexpected error %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("Do: expected status=%d, got %v", http.StatusNotFound, resp.StatusCode) + } + }) +} diff --git a/httpx/mock.go b/httpx/mock.go new file mode 100644 index 0000000..7ce77aa --- /dev/null +++ b/httpx/mock.go @@ -0,0 +1,96 @@ +package httpx + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" +) + +var contentTypes = map[string]string{ + ".json": "application/json", + ".txt": "text/plain", +} + +// MockClient serves responses from the file system instead of remote calls. +// Should be used for testing purposes only. +type MockClient struct { + dir string +} + +// Mock creates a new MockClient and installs it instead of the default one. +func Mock(path ...string) *MockClient { + dir := filepath.Join("testdata", filepath.Join(path...)) + c := &MockClient{dir: dir} + client = c + return c +} + +// Do serves the file according to the request URL. +func (c *MockClient) Do(req *http.Request) (*http.Response, error) { + filename := filepath.Join(c.dir, path.Base(req.URL.Path)) + + data, err := os.ReadFile(filename) + if err != nil { + resp := http.Response{ + Status: http.StatusText(http.StatusNotFound), + StatusCode: http.StatusNotFound, + } + return &resp, nil + } + + cType, ok := contentTypes[path.Ext(filename)] + if !ok { + cType = "application/octet-stream" + } + rdr := respond(cType, data) + resp, err := http.ReadResponse(bufio.NewReader(rdr), req) + if err != nil { + panic(err) + } + return resp, nil +} + +func respond(cType string, data []byte) io.Reader { + buf := bytes.Buffer{} + buf.WriteString("HTTP/1.1 200 OK\n") + buf.WriteString(fmt.Sprintf("Content-Type: %s\n\n", cType)) + _, err := buf.Write(data) + if err != nil { + panic(err) + } + return &buf +} + +// MockServer creates a mock HTTP server and installs its client +// instead of the default one. Serves responses from the file system +// instead of remote calls. Should be used for testing purposes only. +func MockServer() *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + filename := filepath.Join("testdata", path.Base(r.URL.Path)) + + data, err := os.ReadFile(filename) + if err != nil { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + cType, ok := contentTypes[path.Ext(filename)] + if !ok { + cType = "application/octet-stream" + } + + w.Header().Set("content-type", cType) + _, err = w.Write(data) + if err != nil { + panic(err) + } + })) + client = srv.Client() + return srv +} diff --git a/httpx/mock_test.go b/httpx/mock_test.go new file mode 100644 index 0000000..e280d2f --- /dev/null +++ b/httpx/mock_test.go @@ -0,0 +1,34 @@ +package httpx + +import ( + "io" + "net/http" + "testing" +) + +func TestMockClient(t *testing.T) { + Mock() + + const url = "https://codapi.org/example.txt" + req, _ := http.NewRequest("GET", url, nil) + + resp, err := Do(req) + if err != nil { + t.Errorf("Do: unexpected error %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Do: expected status code %d, got %d", http.StatusOK, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Errorf("io.ReadAll: unexpected error %v", err) + } + + want := "hello" + if string(body) != want { + t.Errorf("Do: expected %v, got %v", want, string(body)) + } +} diff --git a/httpx/testdata/example.json b/httpx/testdata/example.json new file mode 100644 index 0000000..bb66bca --- /dev/null +++ b/httpx/testdata/example.json @@ -0,0 +1 @@ +{ "name": "alice" } diff --git a/httpx/testdata/example.txt b/httpx/testdata/example.txt new file mode 100644 index 0000000..b6fc4c6 --- /dev/null +++ b/httpx/testdata/example.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/images/alpine/Dockerfile b/images/alpine/Dockerfile new file mode 100644 index 0000000..e12de74 --- /dev/null +++ b/images/alpine/Dockerfile @@ -0,0 +1,6 @@ +FROM alpine:3.18 + +RUN adduser --home /sandbox --disabled-password sandbox + +USER sandbox +WORKDIR /sandbox diff --git a/logx/logx.go b/logx/logx.go new file mode 100644 index 0000000..8791e23 --- /dev/null +++ b/logx/logx.go @@ -0,0 +1,52 @@ +// Package logx provides helper functions for logging. +package logx + +import ( + "io" + "log" + "os" +) + +var logger = log.New(os.Stderr, "", log.LstdFlags) +var Verbose = false + +// SetOutput sets the output destination. +func SetOutput(w io.Writer) { + logger.SetOutput(w) +} + +// Printf prints a formatted message. +func Printf(format string, v ...any) { + logger.Printf(format, v...) +} + +// Println prints a message. +func Println(v ...any) { + logger.Println(v...) +} + +// Log prints a message. +func Log(message string, args ...any) { + if len(args) == 0 { + logger.Println(message) + } else { + logger.Printf(message+"\n", args...) + } +} + +// Debug prints a message if the verbose mode is on. +func Debug(message string, args ...any) { + if !Verbose { + return + } + Log(message, args...) +} + +// Mock creates a new Memory and installs it as the logger output +// instead of the default one. Should be used for testing purposes only. +func Mock(path ...string) *Memory { + memory := NewMemory("log") + SetOutput(memory) + Verbose = true + return memory +} diff --git a/logx/logx_test.go b/logx/logx_test.go new file mode 100644 index 0000000..20c8144 --- /dev/null +++ b/logx/logx_test.go @@ -0,0 +1,70 @@ +package logx + +import "testing" + +func TestSetOutput(t *testing.T) { + mem := NewMemory("log") + SetOutput(mem) + Log("hello") + if !mem.Has("hello") { + t.Error("SetOutput: memory not set as output") + } +} + +func TestLog(t *testing.T) { + mem := NewMemory("log") + SetOutput(mem) + { + Log("value: %d", 42) + if len(mem.Lines) != 1 { + t.Errorf("Log: expected line count %v", len(mem.Lines)) + } + if !mem.Has("value: 42") { + t.Errorf("Log: expected output: %v", mem.Lines) + } + } + { + Log("value: %d", 84) + if len(mem.Lines) != 2 { + t.Errorf("Log: expected line count %v", len(mem.Lines)) + } + if !mem.Has("value: 42") || !mem.Has("value: 84") { + t.Errorf("Log: expected output: %v", mem.Lines) + } + } +} + +func TestDebug(t *testing.T) { + t.Run("enabled", func(t *testing.T) { + mem := NewMemory("log") + SetOutput(mem) + Verbose = true + { + Debug("value: %d", 42) + if len(mem.Lines) != 1 { + t.Errorf("Log: expected line count %v", len(mem.Lines)) + } + if !mem.Has("value: 42") { + t.Errorf("Log: expected output: %v", mem.Lines) + } + } + { + Debug("value: %d", 84) + if len(mem.Lines) != 2 { + t.Errorf("Log: expected line count %v", len(mem.Lines)) + } + if !mem.Has("value: 42") || !mem.Has("value: 84") { + t.Errorf("Log: expected output: %v", mem.Lines) + } + } + }) + t.Run("disabled", func(t *testing.T) { + mem := NewMemory("log") + SetOutput(mem) + Verbose = false + Debug("value: %d", 42) + if len(mem.Lines) != 0 { + t.Errorf("Log: expected line count %v", len(mem.Lines)) + } + }) +} diff --git a/logx/memory.go b/logx/memory.go new file mode 100644 index 0000000..7d78ace --- /dev/null +++ b/logx/memory.go @@ -0,0 +1,61 @@ +package logx + +import ( + "fmt" + "strings" + "testing" +) + +// Memory stores logged messages in a slice. +type Memory struct { + Name string + Lines []string +} + +// NewMemory creates a new memory destination. +func NewMemory(name string) *Memory { + return &Memory{Name: name, Lines: []string{}} +} + +// Write implements the io.Writer interface. +func (m *Memory) Write(p []byte) (n int, err error) { + msg := string(p) + m.Lines = append(m.Lines, msg) + return len(p), nil +} + +// WriteString writes a string to the memory. +func (m *Memory) WriteString(s string) { + m.Lines = append(m.Lines, s) +} + +// Has returns true if the memory has the message. +func (m *Memory) Has(msg string) bool { + for _, line := range m.Lines { + if strings.Contains(line, msg) { + return true + } + } + return false +} + +// MustHave checks if the memory has the message. +func (m *Memory) MustHave(t *testing.T, msg string) { + if !m.Has(msg) { + t.Errorf("%s must have: %s", m.Name, msg) + } +} + +// MustNotHave checks if the memory does not have the message. +func (m *Memory) MustNotHave(t *testing.T, msg string) { + if m.Has(msg) { + t.Errorf("%s must NOT have: %s", m.Name, msg) + } +} + +// Print prints memory lines to stdout. +func (m *Memory) Print() { + for _, line := range m.Lines { + fmt.Print(line) + } +} diff --git a/logx/memory_test.go b/logx/memory_test.go new file mode 100644 index 0000000..fac8646 --- /dev/null +++ b/logx/memory_test.go @@ -0,0 +1,43 @@ +package logx + +import "testing" + +func TestMemory_Name(t *testing.T) { + mem := NewMemory("log") + if mem.Name != "log" { + t.Errorf("Name: unexpected name %q", mem.Name) + } +} + +func TestMemory_Write(t *testing.T) { + mem := NewMemory("log") + if len(mem.Lines) != 0 { + t.Fatalf("Write: unexpected line count %v", len(mem.Lines)) + } + + n, err := mem.Write([]byte("hello world")) + if err != nil { + t.Fatalf("Write: unexpected error %v", err) + } + if n != 11 { + t.Errorf("Write: unexpected byte count %v", n) + } + + if len(mem.Lines) != 1 { + t.Fatalf("Write: unexpected line count %v", len(mem.Lines)) + } + if mem.Lines[0] != "hello world" { + t.Errorf("Write: unexpected line #0 %q", mem.Lines[0]) + } +} + +func TestMemory_Has(t *testing.T) { + mem := NewMemory("log") + if mem.Has("hello world") { + t.Error("Has: unexpected true") + } + _, _ = mem.Write([]byte("hello world")) + if !mem.Has("hello world") { + t.Error("Has: unexpected false") + } +} diff --git a/sandbox/config.go b/sandbox/config.go new file mode 100644 index 0000000..925bb06 --- /dev/null +++ b/sandbox/config.go @@ -0,0 +1,43 @@ +// Creates sandboxes according to the configuration. +package sandbox + +import ( + "fmt" + + "github.com/nalgeon/codapi/config" + "github.com/nalgeon/codapi/engine" +) + +// A semaphore represents available concurrent workers +// that are responsible for executing code in sandboxes. +// The workers themselves are external to this package +// (the calling goroutines are workers). +var semaphore *Semaphore + +var engineConstr = map[string]func(*config.Config, string, string) engine.Engine{ + "docker": engine.NewDocker, + "http": engine.NewHTTP, +} + +// engines is the registry of command executors. +// Each engine executes a specific command in a specifix sandbox. +// sandbox : command : engine +// TODO: Maybe it's better to create a single instance of each engine +// and pass the sandbox and command as arguments to the Exec. +var engines = map[string]map[string]engine.Engine{} + +// ApplyConfig fills engine registry according to the configuration. +func ApplyConfig(cfg *config.Config) error { + semaphore = NewSemaphore(cfg.PoolSize) + for sandName, sandCmds := range cfg.Commands { + engines[sandName] = make(map[string]engine.Engine) + for cmdName, cmd := range sandCmds { + constructor, ok := engineConstr[cmd.Engine] + if !ok { + return fmt.Errorf("unknown engine: %s", cmd.Engine) + } + engines[sandName][cmdName] = constructor(cfg, sandName, cmdName) + } + } + return nil +} diff --git a/sandbox/config_test.go b/sandbox/config_test.go new file mode 100644 index 0000000..66255dc --- /dev/null +++ b/sandbox/config_test.go @@ -0,0 +1,59 @@ +package sandbox + +import ( + "testing" + + "github.com/nalgeon/codapi/config" + "github.com/nalgeon/codapi/engine" +) + +var cfg = &config.Config{ + PoolSize: 8, + HTTP: &config.HTTP{ + Hosts: map[string]string{"localhost": "localhost"}, + }, + Boxes: map[string]*config.Box{ + "http": {}, + "python": {}, + }, + Commands: map[string]config.SandboxCommands{ + "http": map[string]*config.Command{ + "run": {Engine: "http"}, + }, + "python": map[string]*config.Command{ + "run": { + Engine: "docker", + Entry: "main.py", + Steps: []*config.Step{ + {Box: "python", Action: "run", NOutput: 4096}, + }, + }, + "test": {Engine: "docker"}, + }, + }, +} + +func TestApplyConfig(t *testing.T) { + err := ApplyConfig(cfg) + if err != nil { + t.Fatalf("ApplyConfig: expected nil err, got %v", err) + } + if semaphore.Size() != cfg.PoolSize { + t.Errorf("semaphore.Size: expected %d, got %d", cfg.PoolSize, semaphore.Size()) + } + if len(engines) != 2 { + t.Errorf("len(engines): expected 2, got %d", len(engines)) + } + if len(engines["http"]) != 1 { + t.Errorf("len(engine = http): expected 1, got %d", len(engines["http"])) + } + if _, ok := engines["http"]["run"].(*engine.HTTP); !ok { + t.Error("engine = http: expected HTTP engine") + } + if len(engines["python"]) != 2 { + t.Errorf("len(engine = python): expected 2, got %d", len(engines["python"])) + } + if _, ok := engines["python"]["run"].(*engine.Docker); !ok { + t.Error("engine = python: expected Docker engine") + } +} diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go new file mode 100644 index 0000000..7101a33 --- /dev/null +++ b/sandbox/sandbox.go @@ -0,0 +1,47 @@ +// Package sandbox provides a registry of sandboxes +// for code execution. +package sandbox + +import ( + "errors" + "strings" + "time" + + "github.com/nalgeon/codapi/engine" +) + +var ErrUnknownSandbox = errors.New("unknown sandbox") +var ErrUnknownCommand = errors.New("unknown command") +var ErrEmptyRequest = errors.New("empty request") + +// Validate checks if the code execution request is valid. +func Validate(in engine.Request) error { + box, ok := engines[in.Sandbox] + if !ok { + return ErrUnknownSandbox + } + _, ok = box[in.Command] + if !ok { + return ErrUnknownCommand + } + if len(in.Files) < 2 && strings.TrimSpace(in.Files.First()) == "" { + return ErrEmptyRequest + } + return nil +} + +// Exec executes the code using the appropriate sandbox. +// Allows no more than pool.Size() concurrent workers at any given time. +// The request must already be validated by Validate(). +func Exec(in engine.Request) engine.Execution { + err := semaphore.Acquire() + defer semaphore.Release() + if err == ErrBusy { + return engine.Fail(in.ID, engine.ErrBusy) + } + start := time.Now() + engine := engines[in.Sandbox][in.Command] + out := engine.Exec(in) + out.Duration = int(time.Since(start).Milliseconds()) + return out +} diff --git a/sandbox/sandbox_test.go b/sandbox/sandbox_test.go new file mode 100644 index 0000000..6bc0b6b --- /dev/null +++ b/sandbox/sandbox_test.go @@ -0,0 +1,115 @@ +package sandbox + +import ( + "errors" + "testing" + + "github.com/nalgeon/codapi/engine" + "github.com/nalgeon/codapi/execy" +) + +func TestValidate(t *testing.T) { + _ = ApplyConfig(cfg) + t.Run("valid", func(t *testing.T) { + + req := engine.Request{ + ID: "http_42", + Sandbox: "python", + Command: "run", + Files: map[string]string{ + "": "print('hello')", + }, + } + err := Validate(req) + if err != nil { + t.Errorf("Validate: expected nil err, got %v", err) + } + }) + t.Run("unknown sandbox", func(t *testing.T) { + req := engine.Request{ + ID: "http_42", + Sandbox: "rust", + Command: "run", + Files: nil, + } + err := Validate(req) + if !errors.Is(err, ErrUnknownSandbox) { + t.Errorf("Validate: expected ErrUnknownSandbox, got %T(%s)", err, err) + } + }) + t.Run("unknown command", func(t *testing.T) { + req := engine.Request{ + ID: "http_42", + Sandbox: "python", + Command: "deploy", + Files: nil, + } + err := Validate(req) + if !errors.Is(err, ErrUnknownCommand) { + t.Errorf("Validate: expected ErrUnknownCommand, got %T(%s)", err, err) + } + }) + t.Run("empty request", func(t *testing.T) { + req := engine.Request{ + ID: "http_42", + Sandbox: "python", + Command: "run", + Files: nil, + } + err := Validate(req) + if !errors.Is(err, ErrEmptyRequest) { + t.Errorf("Validate: expected ErrEmptyRequest, got %T(%s)", err, err) + } + }) +} + +func TestExec(t *testing.T) { + _ = ApplyConfig(cfg) + t.Run("exec", func(t *testing.T) { + execy.Mock(map[string]execy.CmdOut{ + "docker run": {Stdout: "hello"}, + }) + req := engine.Request{ + ID: "http_42", + Sandbox: "python", + Command: "run", + Files: map[string]string{ + "": "print('hello')", + }, + } + out := Exec(req) + if out.ID != req.ID { + t.Errorf("ID: expected %s, got %s", req.ID, out.ID) + } + if !out.OK { + t.Error("OK: expected true") + } + if out.Stdout != "hello" { + t.Errorf("Stdout: expected hello, got %s", out.Stdout) + } + if out.Stderr != "" { + t.Errorf("Stderr: expected empty string, got %s", out.Stderr) + } + if out.Err != nil { + t.Errorf("Err: expected nil, got %v", out.Err) + } + }) + t.Run("busy", func(t *testing.T) { + for i := 0; i < cfg.PoolSize; i++ { + _ = semaphore.Acquire() + } + req := engine.Request{ + ID: "http_42", + Sandbox: "python", + Command: "run", + Files: map[string]string{ + "": "print('hello')", + }, + } + out := Exec(req) + if out.Err != engine.ErrBusy { + t.Errorf("Err: expected ErrBusy, got %v", out.Err) + } + }) + +} diff --git a/sandbox/semaphore.go b/sandbox/semaphore.go new file mode 100644 index 0000000..776f6fa --- /dev/null +++ b/sandbox/semaphore.go @@ -0,0 +1,45 @@ +package sandbox + +import "errors" + +var ErrBusy = errors.New("busy") + +// A Semaphore manages a limited number of tokens +// that can be acquired or released. +type Semaphore struct { + tokens chan struct{} +} + +// NewSemaphore creates a new semaphore of the specified size. +func NewSemaphore(size int) *Semaphore { + tokens := make(chan struct{}, size) + for i := 0; i < size; i++ { + tokens <- struct{}{} + } + return &Semaphore{tokens} +} + +// Acquire acquires a token. Returns ErrBusy if no tokens are available. +func (q *Semaphore) Acquire() error { + select { + case <-q.tokens: + return nil + default: + return ErrBusy + } +} + +// Release releases a token. +func (q *Semaphore) Release() { + select { + case q.tokens <- struct{}{}: + return + default: + return + } +} + +// Size returns the size of the semaphore. +func (q *Semaphore) Size() int { + return len(q.tokens) +} diff --git a/sandbox/semaphore_test.go b/sandbox/semaphore_test.go new file mode 100644 index 0000000..9486ec4 --- /dev/null +++ b/sandbox/semaphore_test.go @@ -0,0 +1,45 @@ +package sandbox + +import "testing" + +func TestSemaphore(t *testing.T) { + t.Run("size", func(t *testing.T) { + sem := NewSemaphore(3) + if sem.Size() != 3 { + t.Errorf("Size: expected 3, got %d", sem.Size()) + } + }) + t.Run("acquire", func(t *testing.T) { + sem := NewSemaphore(2) + err := sem.Acquire() + if err != nil { + t.Fatalf("acquire #1: expected nil err") + } + err = sem.Acquire() + if err != nil { + t.Fatalf("acquire #2: expected nil err") + } + err = sem.Acquire() + if err != ErrBusy { + t.Fatalf("acquire #3: expected ErrBusy") + } + }) + t.Run("release", func(t *testing.T) { + sem := NewSemaphore(2) + _ = sem.Acquire() + _ = sem.Acquire() + _ = sem.Acquire() + + sem.Release() + err := sem.Acquire() + if err != nil { + t.Fatalf("acquire after release: expected nil err") + } + }) + t.Run("release free", func(t *testing.T) { + sem := NewSemaphore(2) + sem.Release() + sem.Release() + sem.Release() + }) +} diff --git a/server/io.go b/server/io.go new file mode 100644 index 0000000..a4b06dd --- /dev/null +++ b/server/io.go @@ -0,0 +1,49 @@ +// Reading requests and writing responses. +package server + +import ( + "encoding/json" + "errors" + "io" + "net/http" +) + +// readJson decodes the request body from JSON. +func readJson[T any](r *http.Request) (T, error) { + var obj T + if r.Header.Get("content-type") != "application/json" { + return obj, errors.New(http.StatusText(http.StatusUnsupportedMediaType)) + } + data, err := io.ReadAll(r.Body) + if err != nil { + return obj, err + } + err = json.Unmarshal(data, &obj) + if err != nil { + return obj, err + } + return obj, err +} + +// writeJson encodes an object into JSON and writes it to the response. +func writeJson(w http.ResponseWriter, obj any) error { + data, err := json.Marshal(obj) + if err != nil { + return err + } + + w.Header().Set("content-type", "application/json") + _, err = w.Write(data) + if err != nil { + return err + } + return nil +} + +// writeError encodes an error object into JSON and writes it to the response. +func writeError(w http.ResponseWriter, code int, obj any) { + data, _ := json.Marshal(obj) + w.Header().Set("content-type", "application/json") + w.WriteHeader(code) + w.Write(data) //nolint:errcheck +} diff --git a/server/io_test.go b/server/io_test.go new file mode 100644 index 0000000..13e061d --- /dev/null +++ b/server/io_test.go @@ -0,0 +1,85 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/nalgeon/codapi/engine" +) + +func Test_readJson(t *testing.T) { + t.Run("success", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/example", + strings.NewReader(`{"sandbox": "python", "command": "run"}`)) + req.Header.Set("Content-Type", "application/json") + + got, err := readJson[engine.Request](req) + if err != nil { + t.Errorf("expected nil err, got %v", err) + } + + want := engine.Request{ + Sandbox: "python", Command: "run", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + }) + t.Run("unsupported media type", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/example", nil) + req.Header.Set("Content-Type", "text/plain") + + _, err := readJson[engine.Request](req) + if err == nil || err.Error() != "Unsupported Media Type" { + t.Errorf("unexpected error %v", err) + } + }) + t.Run("error", func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/example", strings.NewReader("hello world")) + req.Header.Set("Content-Type", "application/json") + + _, err := readJson[engine.Request](req) + if err == nil { + t.Error("expected unmarshaling error") + } + }) +} + +func Test_writeJson(t *testing.T) { + w := httptest.NewRecorder() + obj := engine.Request{ + ID: "42", Sandbox: "python", Command: "run", + } + + err := writeJson(w, obj) + if err != nil { + t.Errorf("expected nil err, got %v", err) + } + + body := w.Body.String() + contentType := w.Header().Get("content-type") + if contentType != "application/json" { + t.Errorf("unexpected content-type header %s", contentType) + } + + want := `{"id":"42","sandbox":"python","command":"run","files":null}` + if body != want { + t.Errorf("expected %s, got %s", body, want) + } +} + +func Test_writeError(t *testing.T) { + w := httptest.NewRecorder() + obj := time.Date(2020, 10, 15, 0, 0, 0, 0, time.UTC) + writeError(w, http.StatusForbidden, obj) + if w.Code != http.StatusForbidden { + t.Errorf("expected status code %d, got %d", http.StatusForbidden, w.Code) + } + if w.Body.String() != `"2020-10-15T00:00:00Z"` { + t.Errorf("unexpected body %s", w.Body.String()) + } +} diff --git a/server/middleware.go b/server/middleware.go new file mode 100644 index 0000000..eec6f5c --- /dev/null +++ b/server/middleware.go @@ -0,0 +1,19 @@ +// HTTP middlewares. +package server + +import "net/http" + +// enableCORS allows cross-site requests for a given handler. +func enableCORS(handler func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("access-control-allow-origin", "*") + w.Header().Set("access-control-allow-method", "post") + w.Header().Set("access-control-allow-headers", "authorization, content-type") + w.Header().Set("access-control-max-age", "3600") + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusOK) + return + } + handler(w, r) + } +} diff --git a/server/middleware_test.go b/server/middleware_test.go new file mode 100644 index 0000000..45739eb --- /dev/null +++ b/server/middleware_test.go @@ -0,0 +1,45 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func Test_enableCORS(t *testing.T) { + t.Run("options", func(t *testing.T) { + w := httptest.NewRecorder() + r, _ := http.NewRequest("OPTIONS", "/v1/exec", nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + fn := enableCORS(handler) + fn(w, r) + + if w.Header().Get("access-control-allow-origin") != "*" { + t.Errorf("invalid access-control-allow-origin") + } + if w.Code != 200 { + t.Errorf("expected status code 200, got %d", w.Code) + } + }) + t.Run("post", func(t *testing.T) { + w := httptest.NewRecorder() + r, _ := http.NewRequest("POST", "/v1/exec", nil) + handler := func(w http.ResponseWriter, r *http.Request) {} + fn := enableCORS(handler) + fn(w, r) + + if w.Header().Get("access-control-allow-origin") != "*" { + t.Errorf("invalid access-control-allow-origin") + } + if w.Header().Get("access-control-allow-method") != "post" { + t.Errorf("invalid access-control-allow-method") + } + if w.Header().Get("access-control-allow-headers") != "authorization, content-type" { + t.Errorf("invalid access-control-allow-headers") + } + if w.Header().Get("access-control-max-age") != "3600" { + t.Errorf("access-control-max-age") + } + }) + +} diff --git a/server/router.go b/server/router.go new file mode 100644 index 0000000..99b6820 --- /dev/null +++ b/server/router.go @@ -0,0 +1,79 @@ +// HTTP routes and handlers. +package server + +import ( + "errors" + "fmt" + "net/http" + + "github.com/nalgeon/codapi/engine" + "github.com/nalgeon/codapi/logx" + "github.com/nalgeon/codapi/sandbox" + "github.com/nalgeon/codapi/stringx" +) + +// NewRouter creates HTTP routes and handlers for them. +func NewRouter() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/v1/exec", enableCORS(exec)) + return mux +} + +// exec runs a sandbox command on the supplied code. +func exec(w http.ResponseWriter, r *http.Request) { + // only POST is allowed + if r.Method != http.MethodPost { + err := fmt.Errorf("unsupported method: %s", r.Method) + writeError(w, http.StatusMethodNotAllowed, engine.Fail("-", err)) + return + } + + // read the input data - language, command, code + in, err := readJson[engine.Request](r) + if err != nil { + writeError(w, http.StatusBadRequest, engine.Fail("-", err)) + return + } + in.GenerateID() + + // validate the input data + err = sandbox.Validate(in) + if errors.Is(err, sandbox.ErrUnknownSandbox) || errors.Is(err, sandbox.ErrUnknownCommand) { + writeError(w, http.StatusNotFound, engine.Fail(in.ID, err)) + return + } + if err != nil { + writeError(w, http.StatusBadRequest, engine.Fail(in.ID, err)) + return + } + + // execute the code using the sandbox + out := sandbox.Exec(in) + + // fail on application error + if out.Err != nil { + logx.Log("✗ %s: %s", out.ID, out.Err) + if errors.Is(out.Err, engine.ErrBusy) { + writeError(w, http.StatusTooManyRequests, out) + } else { + writeError(w, http.StatusInternalServerError, out) + } + return + } + + // log results + if out.OK { + logx.Log("✓ %s: took %d ms", out.ID, out.Duration) + } else { + msg := stringx.Compact(stringx.Shorten(out.Stderr, 80)) + logx.Log("✗ %s: %s", out.ID, msg) + } + + // write the response + err = writeJson(w, out) + if err != nil { + err = engine.NewExecutionError("write response", err) + writeError(w, http.StatusInternalServerError, engine.Fail(in.ID, err)) + return + } +} diff --git a/server/router_test.go b/server/router_test.go new file mode 100644 index 0000000..78b0087 --- /dev/null +++ b/server/router_test.go @@ -0,0 +1,156 @@ +package server + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/nalgeon/codapi/config" + "github.com/nalgeon/codapi/engine" + "github.com/nalgeon/codapi/execy" + "github.com/nalgeon/codapi/sandbox" +) + +var cfg = &config.Config{ + PoolSize: 8, + Boxes: map[string]*config.Box{ + "python": {}, + }, + Commands: map[string]config.SandboxCommands{ + "python": map[string]*config.Command{ + "run": { + Engine: "docker", + Entry: "main.py", + Steps: []*config.Step{ + {Box: "python", Action: "run", NOutput: 4096}, + }, + }, + "test": {Engine: "docker"}, + }, + }, +} + +type server struct { + srv *httptest.Server + cli *http.Client +} + +func newServer() *server { + router := NewRouter() + srv := httptest.NewServer(router) + return &server{srv, srv.Client()} +} + +func (s *server) post(uri string, val any) (*http.Response, error) { + body, _ := json.Marshal(val) + req, _ := http.NewRequest("POST", s.srv.URL+uri, bytes.NewReader(body)) + req.Header.Set("content-type", "application/json") + return s.cli.Do(req) +} + +func (s *server) close() { + s.srv.Close() +} + +func Test_exec(t *testing.T) { + _ = sandbox.ApplyConfig(cfg) + execy.Mock(map[string]execy.CmdOut{ + "docker run": {Stdout: "hello"}, + }) + + srv := newServer() + defer srv.close() + + t.Run("success", func(t *testing.T) { + in := engine.Request{ + Sandbox: "python", + Command: "run", + Files: map[string]string{ + "": "print('hello')", + }, + } + resp, err := srv.post("/v1/exec", in) + if err != nil { + t.Fatalf("POST /exec: expected nil err, got %v", err) + } + out := decodeResp[engine.Execution](t, resp) + if !out.OK { + t.Error("OK: expected true") + } + if out.Stdout != "hello" { + t.Errorf("Stdout: expected hello, got %s", out.Stdout) + } + if out.Stderr != "" { + t.Errorf("Stderr: expected empty string, got %s", out.Stderr) + } + if out.Err != nil { + t.Errorf("Err: expected nil, got %v", out.Err) + } + }) + t.Run("error not found", func(t *testing.T) { + in := engine.Request{ + Sandbox: "rust", + Command: "run", + Files: nil, + } + resp, err := srv.post("/v1/exec", in) + if err != nil { + t.Fatalf("POST /exec: expected nil err, got %v", err) + } + if resp.StatusCode != http.StatusNotFound { + t.Errorf("StatusCode: expected 404, got %v", resp.StatusCode) + } + out := decodeResp[engine.Execution](t, resp) + if out.OK { + t.Error("OK: expected false") + } + if out.Stdout != "" { + t.Errorf("Stdout: expected empty string, got %s", out.Stdout) + } + if out.Stderr != "unknown sandbox" { + t.Errorf("Stderr: expected error, got %s", out.Stderr) + } + if out.Err != nil { + t.Errorf("Err: expected nil, got %v", out.Err) + } + }) + t.Run("error bad request", func(t *testing.T) { + in := engine.Request{ + Sandbox: "python", + Command: "run", + Files: nil, + } + resp, err := srv.post("/v1/exec", in) + if err != nil { + t.Fatalf("POST /exec: expected nil err, got %v", err) + } + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("StatusCode: expected 400, got %v", resp.StatusCode) + } + out := decodeResp[engine.Execution](t, resp) + if out.OK { + t.Error("OK: expected false") + } + if out.Stdout != "" { + t.Errorf("Stdout: expected empty string, got %s", out.Stdout) + } + if out.Stderr != "empty request" { + t.Errorf("Stderr: expected error, got %s", out.Stderr) + } + if out.Err != nil { + t.Errorf("Err: expected nil, got %v", out.Err) + } + }) +} + +func decodeResp[T any](t *testing.T, resp *http.Response) T { + defer resp.Body.Close() + var val T + err := json.NewDecoder(resp.Body).Decode(&val) + if err != nil { + t.Fatal(err) + } + return val +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..9f0b094 --- /dev/null +++ b/server/server.go @@ -0,0 +1,59 @@ +// Package server provides an HTTP API for running code in a sandbox. +package server + +import ( + "context" + "fmt" + "net/http" + "sync" + "time" + + "github.com/nalgeon/codapi/logx" +) + +// The maximum duration of the server graceful shutdown. +const ShutdownTimeout = 3 * time.Second + +// A Server is an HTTP sandbox server. +type Server struct { + srv *http.Server + wg *sync.WaitGroup +} + +// NewServer creates a new Server. +func NewServer(port int, handler http.Handler) *Server { + addr := fmt.Sprintf(":%d", port) + return &Server{ + srv: &http.Server{Addr: addr, Handler: handler}, + wg: &sync.WaitGroup{}, + } +} + +// Start starts the server. +func (s *Server) Start() { + // run the server inside a goroutine so that + // it does not block the main goroutine, and allow it + // to start other processes and listen for signals + s.wg.Add(1) + go func() { + defer s.wg.Done() + err := s.srv.ListenAndServe() + if err != http.ErrServerClosed { + logx.Log(err.Error()) + } + }() +} + +// Stop stops the server. +func (s *Server) Stop() error { + // perform a graceful shutdown, but not longer + // than the duration of ShutdownTimeout + ctx, cancel := context.WithTimeout(context.Background(), ShutdownTimeout) + defer cancel() + err := s.srv.Shutdown(ctx) + if err != nil { + return err + } + s.wg.Wait() + return nil +} diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 0000000..0e4c5d8 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,31 @@ +package server + +import ( + "net/http" + "testing" +) + +func TestServer(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + srv := NewServer(8585, handler) + if srv.srv.Addr != ":8585" { + t.Fatalf("NewServer: expected port :8585 got %s", srv.srv.Addr) + } + + srv.Start() + resp, err := http.Get("http://localhost:8585/get") + if err != nil { + t.Fatalf("GET: expected nil err, got %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("GET: expected status code 200, got %d", resp.StatusCode) + } + + err = srv.Stop() + if err != nil { + t.Fatalf("Stop: expected nil err, got %v", err) + } +} diff --git a/stringx/stringx.go b/stringx/stringx.go new file mode 100644 index 0000000..744f950 --- /dev/null +++ b/stringx/stringx.go @@ -0,0 +1,33 @@ +// Package stringx provides helper functions for working with strings. +package stringx + +import ( + "crypto/rand" + "encoding/hex" + "regexp" +) + +var compactRE = regexp.MustCompile(`\s+`) + +// Shorten shortens a string to a specified number of characters. +func Shorten(s string, maxlen int) string { + var short = []rune(s) + if len(short) > maxlen { + short = short[:maxlen] + short = append(short, []rune(" [truncated]")...) + } + return string(short) +} + +// Compact replaces consecutive whitespaces with a single space. +func Compact(s string) string { + return compactRE.ReplaceAllString(string(s), " ") +} + +// RandString generates a random string. +// length must be even. +func RandString(length int) string { + b := make([]byte, length/2) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/stringx/stringx_test.go b/stringx/stringx_test.go new file mode 100644 index 0000000..54b7412 --- /dev/null +++ b/stringx/stringx_test.go @@ -0,0 +1,51 @@ +package stringx + +import "testing" + +func TestShorten(t *testing.T) { + t.Run("shorten", func(t *testing.T) { + const src = "Hello, World!" + const want = "Hello [truncated]" + got := Shorten(src, 5) + if got != want { + t.Errorf("expected %q, got %q", got, want) + } + }) + t.Run("ignore", func(t *testing.T) { + const src = "Hello, World!" + const want = src + got := Shorten(src, 20) + if got != want { + t.Errorf("expected %q, got %q", got, want) + } + }) +} + +func TestCompact(t *testing.T) { + t.Run("compact", func(t *testing.T) { + const src = "go\nis awesome" + const want = "go is awesome" + got := Compact(src) + if got != want { + t.Errorf("expected %q, got %q", got, want) + } + }) + t.Run("ignore", func(t *testing.T) { + const src = "go is awesome" + const want = src + got := Compact(src) + if got != want { + t.Errorf("expected %q, got %q", got, want) + } + }) +} + +func TestRandString(t *testing.T) { + lengths := []int{2, 4, 6, 8, 10} + for _, n := range lengths { + s := RandString(n) + if len(s) != n { + t.Errorf("%d: expected len(s) = %d, got %d", n, n, len(s)) + } + } +}