Go library

Build your own tools on gorun

Everything the gorun command does is in the gorun Go package: load a Gorunfile, pick servers, run tasks in parallel over SSH, and get a progress event for every step. Here are two small apps built on it, a web dashboard and a CLI, with their code.

go get github.com/jenmud/gorun@latest

Needs Go 1.26 or newer. Reference: pkg.go.dev/github.com/jenmud/gorun.

Your first program

Load a Gorunfile, set OnEvent, run. That's a complete program: it runs every server's tasks in parallel and prints each result as it lands.

package main

import (
	"context"
	"fmt"
	"log"

	"github.com/jenmud/gorun"
)

func main() {
	var cfg gorun.Config
	if err := cfg.LoadFile("Gorunfile"); err != nil {
		log.Fatal(err)
	}

	// Called for every progress event, from one goroutine per server.
	cfg.OnEvent = func(event any) {
		switch ev := event.(type) {
		case gorun.TaskEvent:
			if ev.State == gorun.TaskDone || ev.State == gorun.TaskFailed {
				fmt.Printf("%-10s %-16s exit %d in %s\n", ev.Server, ev.Task, ev.ExitCode, ev.Duration)
			}
		case gorun.ProcessFinishedEvent:
			fmt.Printf("%d tasks, %d failed\n", ev.TotalTasks, ev.Failed)
		}
	}

	if err := cfg.Run(context.Background()); err != nil {
		log.Fatal(err)
	}
}

Progress events

OnEvent receives these values as the run happens. Switch on the type:

EventSentUseful fields
ProcessStartedEventonce, when the run startsTitle, Servers, StartedAt, Hostname
PlanEventonce per server, before its first taskServer, Address, Tasks (in run order), TaskMeta
TaskEventevery time a task changes stateServer, Task, State, Stdout, Stderr, ExitCode, Duration, Attempt, Attempts, Command
ServerErrorEventwhen a server can't run at all (e.g. SSH refused)Server, Err
ProcessFinishedEventonce, when every server is doneSuccess, Duration, TotalTasks, Failed, Skipped

A TaskEvent's State is TaskRunning, TaskDone, TaskFailed or TaskSkipped. A running event with Attempt above 1 is a retry.

Lock your state. Each server runs in its own goroutine, so OnEvent is called concurrently. Guard whatever it updates with a mutex, as both examples do.

Choosing what to run

// the same servers as: gorun Gorunfile --tag web
servers, err := cfg.SelectServers(gorun.ServerFilter{Tags: []string{"web"}})

// run everything those servers normally run
err = cfg.Run(ctx, servers...)

// or just one task (and what it depends on), on any servers
err = cfg.RunTask(ctx, cfg.Tasks["restart-nginx"], servers...)

// or only tasks marked local = true, on this machine
err = cfg.RunLocal(ctx, "build", "package")

Logs

// gorun logs every task to stdout by default; send it elsewhere
f, _ := gorun.OpenLogFile(gorun.LogFilePath()) // ~/.gorun/gorun.log
gorun.SetLogOutput(f)

Example 1 · web app

Service Deck: manage services across a fleet

A small Go web app for the people who look after the servers. Pick a tier, press an action, and watch every server work through it. Each action is a gorun task (restart nginx runs its config test first), and every progress event streams to the browser over server-sent events with Datastar. A health check finds a broken worker; the deploy runs pull, migrate, restart and verify on eight servers at once.

Service Deck, a web app built on the gorun library, deploying to eight servers and running health checks

Starting a run from a button

Select servers the way --tag does, hand gorun the event callback, and run the task in the background. The request returns straight away.

cfg, err := d.load()
if err != nil {
	_ = sse.ConsoleError(err)
	return
}

// pick servers the way gorun's --tag flag does
filter := gorun.ServerFilter{}
if signals.Target != "" && signals.Target != "all" {
	filter.Tags = []string{signals.Target}
}
servers, err := cfg.SelectServers(filter)
if err != nil {
	_ = sse.ConsoleError(err)
	return
}

d.mu.Lock()
if d.running {
	d.mu.Unlock()
	return // one run at a time
}
d.running = true
d.board = board{Title: cfg.Title, Action: act.Label, Target: signals.Target, Started: time.Now()}
for _, s := range servers {
	d.board.Servers = append(d.board.Servers, &serverState{Name: s.Name, Tags: s.Tags, State: "queued"})
}
d.version++
d.mu.Unlock()

// Every progress event lands here, from many goroutines at once: one per server.
cfg.OnEvent = d.onEvent

go func() {
	err := cfg.RunTask(context.Background(), cfg.Tasks[act.Task], servers...)
	d.mu.Lock()
	defer d.mu.Unlock()
	d.running = false
	if err != nil && d.board.Summary == "" {
		d.board.Summary = err.Error()
		d.board.Failed = true
	}
	d.version++
}()

Turning events into the board

Every card, progress bar and log line on the page comes from this switch. A stream handler redraws the board whenever it changes.

// onEvent turns gorun's events into board state.
func (d *deck) onEvent(event any) {
	d.mu.Lock()
	defer d.mu.Unlock()
	defer func() { d.version++ }()

	switch ev := event.(type) {
	case gorun.ProcessStartedEvent:
		d.logLocked("start", fmt.Sprintf("%s on %d servers", d.board.Action, len(ev.Servers)))

	case gorun.PlanEvent:
		if s := d.server(ev.Server); s != nil {
			s.Tasks = ev.Tasks
			d.logLocked("plan", fmt.Sprintf("%s: %s", ev.Server, strings.Join(ev.Tasks, " → ")))
		}

	case gorun.TaskEvent:
		s := d.server(ev.Server)
		if s == nil {
			return
		}
		switch ev.State {
		case gorun.TaskRunning:
			s.State, s.Current, s.Attempt = "running", ev.Task, ""
			if ev.Attempt > 1 {
				s.Attempt = fmt.Sprintf("retry %d/%d", ev.Attempt-1, ev.Attempts-1)
			}
			d.logLocked("running", fmt.Sprintf("%s › %s", ev.Server, ev.Task))
		case gorun.TaskDone:
			s.Done++
			s.LastLine = lastLine(ev.Stdout)
			s.Elapsed = ev.Duration.Round(10 * time.Millisecond).String()
			if s.Done == len(s.Tasks) {
				s.State, s.Current = "done", ""
			}
			d.logLocked("done", fmt.Sprintf("%s › %s in %s", ev.Server, ev.Task, s.Elapsed))
		case gorun.TaskFailed:
			s.State, s.Current = "failed", ev.Task
			s.LastLine = lastLine(ev.Stderr)
			d.logLocked("failed", fmt.Sprintf("%s › %s exit %d: %s", ev.Server, ev.Task, ev.ExitCode, s.LastLine))
		case gorun.TaskSkipped:
			d.logLocked("skipped", fmt.Sprintf("%s › %s", ev.Server, ev.Task))
		}

	case gorun.ServerErrorEvent:
		if s := d.server(ev.Server); s != nil {
			s.State, s.LastLine = "failed", fmt.Sprint(ev.Err)
		}
		d.logLocked("failed", fmt.Sprintf("%s: %v", ev.Server, ev.Err))

	case gorun.ProcessFinishedEvent:
		d.board.Failed = !ev.Success
		d.board.Summary = fmt.Sprintf("%d tasks, %d failed, %d skipped in %s",
			ev.TotalTasks, ev.Failed, ev.Skipped, ev.Duration.Round(100*time.Millisecond))
		d.logLocked("finish", d.board.Summary)
	}
}

Run it: the demo fleet is local and only echoes and sleeps. Give the hosts ssh tables and real commands (they're in the Gorunfile's comments) to manage real servers.

git clone https://github.com/jenmud/gorun
cd gorun/examples/servicedeck
go run .                      # http://localhost:8090
Full source on GitHub →

Example 2 · command line

rollout: a CLI with its own progress display

A release tool that draws its own progress bars from the same events: one line per server, the running task and how long it has taken, a failed install that is retried before it gives up, and a summary at the end. It's a couple of hundred lines, with no dependencies beyond gorun.

rollout, a command-line tool built on the gorun library, rolling a release out to six servers

Load, select, run

// gorun logs every task to stdout unless told otherwise. This tool draws on the
// terminal itself, so send the logs to gorun's log file (~/.gorun/gorun.log).
if logFile, err := gorun.OpenLogFile(gorun.LogFilePath()); err == nil {
	gorun.SetLogOutput(logFile)
	defer logFile.Close()
}

var cfg gorun.Config
if err := cfg.LoadFile(*file); err != nil {
	fail(err)
}
cfg.ResolvePaths(filepath.Dir(*file))

filter := gorun.ServerFilter{}
if *tag != "" {
	filter.Tags = []string{*tag}
}
servers, err := cfg.SelectServers(filter)
if err != nil {
	fail(err)
}

p := newProgress(cfg.Title, servers)
cfg.OnEvent = p.onEvent // every progress event, from every server, as it happens

ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
defer stop()

done := make(chan struct{})
go p.draw(done) // redraw ten times a second
err = cfg.Run(ctx, servers...)
close(done)
p.render() // the final frame

if err != nil || p.failed > 0 {
	os.Exit(1)
}

Events in, progress out

// onEvent is called by gorun from one goroutine per server, so it locks.
func (p *progress) onEvent(event any) {
	p.mu.Lock()
	defer p.mu.Unlock()

	switch ev := event.(type) {
	case gorun.PlanEvent: // the server's tasks, in the order they will run
		p.servers[ev.Server].total = len(ev.Tasks)

	case gorun.TaskEvent: // a task changed state
		r := p.servers[ev.Server]
		switch ev.State {
		case gorun.TaskRunning:
			r.task, r.started, r.note = ev.Task, time.Now(), ""
			if ev.Attempt > 1 {
				r.note = fmt.Sprintf("retry %d of %d", ev.Attempt-1, ev.Attempts-1)
			}
		case gorun.TaskDone:
			r.done++
			r.finished = r.done == r.total
		case gorun.TaskFailed:
			r.done++
			r.task = ev.Task
			r.note = fmt.Sprintf("exit %d: %s", ev.ExitCode, clip(lastLine(ev.Stderr), 32))
			r.failed, r.finished = true, true
		case gorun.TaskSkipped:
			r.done++
		}

	case gorun.ServerErrorEvent: // the server couldn't run at all (e.g. SSH refused)
		r := p.servers[ev.Server]
		r.failed, r.finished, r.note = true, true, clip(fmt.Sprint(ev.Err), 32)

	case gorun.ProcessFinishedEvent: // the whole run is over
		p.failed = ev.Failed
		p.summary = fmt.Sprintf("%d tasks · %d failed · %d skipped · %s",
			ev.TotalTasks, ev.Failed, ev.Skipped, ev.Duration.Round(100*time.Millisecond))
	}
}
git clone https://github.com/jenmud/gorun
cd gorun/examples/rollout
go run .                      # every server
go run . -tag edge            # only the edge servers
Full source on GitHub →

More ideas

A Slack bot that runs @deploy and posts each server's result. A nightly job that patches a fleet and opens a ticket for the failures. A Kubernetes operator step for the machines outside the cluster. Same five events every time.

Library reference in the docs