From 6040eff2d2b4618f19d4615d246ea6b0704acdf8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 21:00:08 -0500 Subject: Add domain and TLS management via skiff domains add/remove/list Replaces the single server_name string in meta.json with a domain list (one primary, others redirect to it) and TLSDomains tracking what the app's certificate actually covers. Certs are issued/expanded via certbot certonly (webroot, never the --nginx plugin, since Skiff owns and regenerates skiff.d/*.conf itself) as one SAN cert per app, matching how a real multi-domain vhost is normally issued. Domain/TLS changes never touch the jail - no restart, no health check - since they only affect nginx and the certificate. `add` is safe to retry if DNS isn't ready yet; `remove` never calls certbot at all, so it can't fail on an ACME/network problem. Also updates README/USER_GUIDE for the new command surface, one-time TLS host setup (certbot register, renewal cron, reload hook), the --ip same-subnet requirement for jails to reach the network at build time, and the actual base-image provisioning sequence (pkg needs an explicit bootstrap after bsdinstall; versioned port names like go125 need a default symlink) - all found and fixed live while getting this working end-to-end on a real FreeBSD VM. Co-Authored-By: Claude Sonnet 5 --- README.md | 111 +++++++++++++++++++++++++++- USER_GUIDE.md | 149 +++++++++++++++++++++++++++++++++++++- cmd/skiff/main.go | 116 +++++++++++++++++++++++++++-- internal/appmeta/appmeta.go | 33 +++++++-- internal/certbot/certbot.go | 26 +++++++ internal/deploy/deploy.go | 2 +- internal/domains/domains.go | 128 ++++++++++++++++++++++++++++++++ internal/layout/layout.go | 13 ++++ internal/nginx/nginx.go | 53 ++++++++++++-- internal/tmpl/nginx-app.conf.tmpl | 38 +++++++++- internal/tmpl/tmpl.go | 21 +++++- 11 files changed, 656 insertions(+), 34 deletions(-) create mode 100644 internal/certbot/certbot.go create mode 100644 internal/domains/domains.go diff --git a/README.md b/README.md index 4373008..006d67a 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,23 @@ needs to provide](#what-an-app-needs-to-provide). - FreeBSD 15, ZFS root - `nginx` and `git` - neither ships in base; install with `pkg install nginx git-lite` +- `certbot` - only needed if you'll route real domains via `skiff domains`; + install with `pkg install py312-certbot` (package name tracks the default + Python version, so this may drift) - `jail_enable="YES"` in `/etc/rc.conf` (`sysrc jail_enable=YES`) - `service jail start`/`restart` refuse to run without it - Whatever toolchain your apps' Makefiles invoke to build - `go`, `python`, `rustc`, `node`, etc. - baked into the base jail image (see [One-time host setup](#one-time-host-setup)); builds run inside the jail via `jexec`, not on the host itself. `make` itself is already in FreeBSD base (bmake), so - it's the one build tool Skiff never asks you to install. + it's the one build tool Skiff never asks you to install. Watch for + versioned port names (`go125`, `python312`, ...) - FreeBSD installs the + literal versioned binary, not a bare `go`/`python3`, so symlink a default + into the base image's `/usr/local/bin` if a Makefile invokes the + unversioned name. +- The build step needs real outbound network access from inside the jail + (module/package downloads) - see the `--ip` note under [First + deploy](#first-deploy). ## One-time host setup @@ -33,6 +43,42 @@ needs to provide](#what-an-app-needs-to-provide). # zfs create zroot/skiff # zfs create -o mountpoint=/skiff/base zroot/skiff/base # bsdinstall jail /skiff/base +``` + +`bsdinstall jail` only lays down the bare base system - it ships `pkg(8)`'s +bootstrap stub but never actually runs it, so a jail cloned from this alone +has no working `pkg` of its own (files can still be `pkg -c`-installed into +it from the host, but nothing *inside* the jail can manage them, and +`jexec pkg ...` fails with "the package management tool is not yet +installed"). Bootstrap pkg into the image itself, before snapshotting: + +``` +# mount -t devfs devfs /skiff/base/dev +# env ASSUME_ALWAYS_YES=yes pkg -c /skiff/base bootstrap -f +``` + +(`-f` because the stub already counts as "pkg is present" otherwise; devfs +is needed for pkg's own network/TLS calls inside a bare chroot - `fetch(1)` +wants `/dev/null` and `/dev/random`.) Now install whatever your apps' +Makefiles actually need to build: + +``` +# pkg -c /skiff/base install -y go125 +``` + +Watch for versioned port names - FreeBSD installs `go125` as literally +`go125`, not `go`, so a Makefile invoking the bare command needs a default +symlinked in: + +``` +# ln -sf go125 /skiff/base/usr/local/bin/go +# ln -sf gofmt125 /skiff/base/usr/local/bin/gofmt +``` + +Unmount devfs and snapshot once the image can actually build and run apps: + +``` +# umount /skiff/base/dev # zfs snapshot zroot/skiff/base@ready # zfs create -o mountpoint=none zroot/skiff/appbase # zfs create -o mountpoint=none zroot/skiff/appdata @@ -92,6 +138,32 @@ include /usr/local/etc/nginx/skiff.d/*.conf; Skiff only ever writes files under `skiff.d/` and reloads - it never touches `nginx.conf` itself. +### TLS (Let's Encrypt) + +Skiff issues certificates itself via `certbot certonly` (webroot validation, +never certbot's own `--nginx` plugin - Skiff owns `skiff.d/*.conf` and would +fight any other writer of it). Register the certbot account once: + +``` +# certbot register --agree-tos -m you@example.com +``` + +Renewal rides on FreeBSD's own `periodic(8)`, same as any other certbot +install - Skiff runs no daemon or scheduler of its own for this: + +``` +# sysrc -f /etc/periodic.conf weekly_certbot_enable="YES" +``` + +`certonly` mode doesn't reload nginx after a renewal the way certbot's +installer plugins do, so add that as a deploy hook once: + +``` +# mkdir -p /usr/local/etc/letsencrypt/renewal-hooks/deploy +# printf '#!/bin/sh\nnginx -s reload\n' > /usr/local/etc/letsencrypt/renewal-hooks/deploy/reload-nginx +# chmod +x /usr/local/etc/letsencrypt/renewal-hooks/deploy/reload-nginx +``` + ### Install skiff ``` @@ -104,7 +176,7 @@ $ GOOS=freebsd GOARCH=amd64 go build -o skiff ./cmd/skiff ### First deploy ``` -# skiff new-app --name myapp --ip 10.0.0.5 --port 8080 --server-name myapp.example.com +# skiff new-app --name myapp --ip 192.168.1.50 --port 8080 ``` Clones the base dataset, writes `/etc/jail.conf.d/myapp.conf` and the jail's @@ -112,6 +184,38 @@ own rc.d service, creates the bare repo at `/skiff/repos/myapp.git` with its `post-receive` hook installed, and starts the (empty) jail. Prints the remote URL to add locally. +`--ip` must actually route on the host's `--iface` interface - in practice, +an address in the same subnet as the host's own primary address on that +interface, not an arbitrary block like `10.0.0.0/8`. Classic (non-VNET) +jails share the host's routing table, so an out-of-subnet address can still +send packets - they just may never get a response, silently, if whatever +sits upstream (a NAT gateway, a virtualized bridge) only answers its own +subnet. This didn't matter before builds moved inside the jail; now every +deploy's build step needs real outbound network access from that exact +address (module downloads, `pkg install`, etc). + +### Domains & TLS + +`new-app` only provisions the jail - nothing is routed until you add a +domain: + +``` +# skiff domains add --app myapp --primary myapp.example.com +# skiff domains add --app myapp www.myapp.example.com +# skiff domains list --app myapp +# skiff domains remove --app myapp www.myapp.example.com +``` + +`add` routes the hostname over plain HTTP immediately, then requests or +expands a single certificate covering every domain currently registered for +the app (named after whichever is primary). If DNS isn't pointed at the +app's `--ip` yet, issuance fails and the command exits non-zero - the app +stays reachable over HTTP, and `add` is safe to re-run once DNS is fixed, +same as retrying a failed deploy. The first domain added becomes primary +automatically; every other domain 301-redirects to it, over HTTP and HTTPS +alike. `remove` only updates routing - it never calls out to certbot, so a +DNS or network problem can't make it fail. + ### Every deploy after ``` @@ -209,3 +313,6 @@ the Makefile handles. symlink flip and are safe to clean up by hand. - Single host only. Scaling to a second host is a `zfs send`/`receive` of an app's dataset plus the same jail.conf/nginx steps - not yet automated here. +- `skiff domains remove` doesn't shrink the shared certificate's SAN - a + removed hostname may still be listed on it until the next `add` re-syncs + the domain set. Harmless, since nothing routes to it anymore. diff --git a/USER_GUIDE.md b/USER_GUIDE.md index fe78bd0..63d6517 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -224,12 +224,23 @@ this in a hurry - it's more destructive than "undo." ``` # pkg install nginx git-lite ``` +- `certbot` - only needed if you'll route real domains via `skiff domains`: + ``` + # pkg install py312-certbot + ``` + (the package name tracks the default Python version, so this may drift) - Whatever your apps' Makefiles actually invoke to build - `go`, `python3`, `rustc`, `node`, etc. - baked into the base jail image (see [ZFS layout](#1-zfs-layout) below); builds run *inside* the jail via `jexec`, not on the host. Skiff itself doesn't care which; `make` is already in FreeBSD base (bmake), so it's never something you need - to install separately. + to install separately. Watch for versioned port names - FreeBSD's + `go125`, `python312`, etc. install as that literal binary, not a bare + `go`/`python3`; symlink a default into the base image's + `/usr/local/bin` if a Makefile invokes the unversioned name. The build + step also needs real outbound network access from inside the jail + (module/package downloads) - see the `--ip` note under [Provisioning a + new app](#provisioning-a-new-app). - `jail_enable="YES"` in `/etc/rc.conf`: ``` # sysrc jail_enable=YES @@ -246,6 +257,42 @@ this in a hurry - it's more destructive than "undo." # zfs create zroot/skiff # zfs create -o mountpoint=/skiff/base zroot/skiff/base # bsdinstall jail /skiff/base +``` + +`bsdinstall jail` only lays down the bare base system - it ships `pkg(8)`'s +bootstrap stub but never actually runs it, so a jail cloned from this alone +has no working `pkg` of its own (files can still be `pkg -c`-installed into +it from the host, but nothing *inside* the jail can manage them, and +`jexec pkg ...` fails with "the package management tool is not yet +installed"). Bootstrap pkg into the image itself, before snapshotting: + +``` +# mount -t devfs devfs /skiff/base/dev +# env ASSUME_ALWAYS_YES=yes pkg -c /skiff/base bootstrap -f +``` + +(`-f` because the stub already counts as "pkg is present" otherwise; devfs +is needed for pkg's own network/TLS calls inside a bare chroot - `fetch(1)` +wants `/dev/null` and `/dev/random`.) Now install whatever your apps' +Makefiles actually need to build: + +``` +# pkg -c /skiff/base install -y go125 +``` + +Watch for versioned port names - FreeBSD installs `go125` as literally +`go125`, not `go`, so a Makefile invoking the bare command needs a default +symlinked in: + +``` +# ln -sf go125 /skiff/base/usr/local/bin/go +# ln -sf gofmt125 /skiff/base/usr/local/bin/gofmt +``` + +Unmount devfs and snapshot once the image can actually build and run apps: + +``` +# umount /skiff/base/dev # zfs snapshot zroot/skiff/base@ready # zfs create -o mountpoint=none zroot/skiff/appbase # zfs create -o mountpoint=none zroot/skiff/appdata @@ -337,7 +384,38 @@ include /usr/local/etc/nginx/skiff.d/*.conf; Skiff only ever writes files under `skiff.d/` and reloads - it never touches `nginx.conf` itself. -#### 5. Install skiff +#### 5. TLS (Let's Encrypt) + +Skiff issues certificates itself via `certbot certonly` (webroot +validation, never certbot's own `--nginx` plugin - Skiff owns +`skiff.d/*.conf` and regenerates it on every domain/deploy change, and +would fight any other writer of that file). Register the certbot account +once: + +``` +# certbot register --agree-tos -m you@example.com +``` + +Renewal rides on FreeBSD's own `periodic(8)`, the same as any other +certbot install - Skiff runs no daemon or scheduler of its own for this, +and `certbot renew` is host-wide by design, so one cron job covers every +app's certificates regardless of when each was issued: + +``` +# sysrc -f /etc/periodic.conf weekly_certbot_enable="YES" +``` + +`certonly` mode doesn't reload nginx after a renewal the way certbot's +installer plugins do, so add that as a deploy hook once - certbot runs +every script under this directory after any successful renewal: + +``` +# mkdir -p /usr/local/etc/letsencrypt/renewal-hooks/deploy +# printf '#!/bin/sh\nnginx -s reload\n' > /usr/local/etc/letsencrypt/renewal-hooks/deploy/reload-nginx +# chmod +x /usr/local/etc/letsencrypt/renewal-hooks/deploy/reload-nginx +``` + +#### 6. Install skiff Cross-compile from any machine with Go, then copy the binary over: @@ -350,7 +428,7 @@ $ scp skiff root@host:/tmp/skiff ### Provisioning a new app ``` -# skiff new-app --name myapp --ip 10.0.0.5 --port 8080 --server-name myapp.example.com +# skiff new-app --name myapp --ip 192.168.1.50 --port 8080 ``` This clones the base dataset, writes `/etc/jail.conf.d/myapp.conf` and @@ -374,6 +452,58 @@ from the base image, so mail sent from inside the jail (e.g. by cron or the app itself) is attributed to `myapp.skiff.local` instead of the base image's own hostname. +**`--ip` must actually route on the host's `--iface` interface** - in +practice, an address in the same subnet as the host's own primary +address on that interface, not an arbitrary block like `10.0.0.0/8`. +Classic (non-VNET) jails share the host's routing table, so an +out-of-subnet address can still send packets - they just may never get a +response, silently, if whatever sits upstream (a NAT gateway, a +virtualized bridge) only answers its own subnet. This didn't matter +before builds moved inside the jail (see [Requirements](#requirements)); +now every deploy's build step needs real outbound network access from +that exact address (module downloads, `pkg install`, etc), not just +inbound reachability for the app itself. + +### Domains and TLS + +`new-app` only provisions the jail - nothing is routed until you add a +domain: + +``` +# skiff domains add --app myapp --primary myapp.example.com +# skiff domains add --app myapp www.myapp.example.com +# skiff domains list --app myapp +# skiff domains remove --app myapp www.myapp.example.com +``` + +`add` writes the hostname to the app's `meta.json`, routes it over plain +HTTP immediately (so the ACME HTTP-01 challenge below has somewhere to +land), then requests or expands a single certificate covering every +domain currently registered for the app, named after whichever is +primary - matching how a real multi-domain vhost is normally issued (one +SAN cert), not a cert per hostname. + +If DNS isn't pointed at the app's `--ip` yet, certificate issuance fails +and the command exits non-zero - the app stays reachable over HTTP, and +`add` is safe to re-run once DNS is fixed, exactly like retrying a failed +deploy (see [If the health check +fails](#if-the-health-check-fails)). Nothing about a failed `add` is +destructive. + +The first domain added for an app becomes primary automatically; +`--primary` reassigns it explicitly. Every non-primary domain +301-redirects to the primary, over HTTP and HTTPS alike, once its own +hostname is covered by the certificate. + +`remove` only updates routing - unlike `add`, it never calls out to +certbot, so a DNS or network problem can't make it fail. It's refused if +you try to remove the current primary while other domains remain - +reassign primary first (`skiff domains add --primary `). + +Domain/TLS changes never restart the jail and never run a health check - +unlike a code deploy or `skiff config`, they don't touch the running app +at all, only nginx and the certificate. + ### Rollback ``` @@ -452,6 +582,10 @@ Carried over from the project's own README, plus what testing surfaced: steps - not automated. - **`skiff rollback -r` is destructive** (see above) - there is currently no confirmation prompt or dry-run. +- **`skiff domains remove` doesn't shrink the shared certificate's SAN.** + A removed hostname may still be listed on the app's certificate until + the next `add` re-syncs the domain set. Harmless, since nothing routes + to it anymore. --- @@ -468,6 +602,9 @@ Carried over from the project's own README, plus what testing surfaced: | `git push` says `Everything up-to-date` but you wanted to retry a failed deploy | The git ref already advanced despite the deploy failing - nothing new to push | New commit (`git commit --allow-empty`), or on the host: `skiff deploy --app --new --ref refs/heads/main` | | `build: \`make\` did not produce ./app` | Your Makefile's default target ran (or was skipped as "up to date") without ultimately leaving a file named `app` in the repo root | Run `make -B` locally in a clean checkout and confirm `./app` exists after; check the target actually named `app` (or a dependency of the default target) writes there | | Health check fails and you're not sure why | - | `jexec sockstat -4 -l` to check for a listener; `jexec ps aux` to see if the process is even running; try `jexec /app/current/app` directly to see startup errors that daemon(8) would otherwise swallow | +| Build fails inside the jail with a missing command (`go: not found`, etc.) even though it's installed on the host | The base jail image itself is missing it, or FreeBSD installed it under a versioned name (`go125`, not `go`) | `jexec which `; if it's there under a versioned name, symlink a default into the base image's `/usr/local/bin`, re-snapshot, and `skiff rebuild --app --base-snapshot ` | +| Build fails inside the jail on any network call (module download, `pkg install`), or `pkg`/`fetch`/`go mod` just time out | The app's `--ip` doesn't actually route from this host - classic jails share the host's routing table, but an out-of-subnet address (e.g. `10.0.0.5` on a host whose real subnet is `192.168.1.0/24`) can send packets that silently never get a response behind some NAT/bridged setups | `jexec fetch -T5 -o /dev/null https://` to confirm; if it hangs, re-provision with an `--ip` in the same subnet as the host's own address on `--iface` (see [Provisioning a new app](#provisioning-a-new-app)) | +| `skiff domains add` fails with a certbot/ACME error | Most likely DNS isn't pointed at the app's `--ip` yet; less commonly, the app's `--ip` doesn't route outbound from this host (see above) | Fix DNS, then re-run `skiff domains add` with the same hostname - safe to retry, the app stays on plain HTTP in the meantime and nothing is left half-configured | --- @@ -475,7 +612,7 @@ Carried over from the project's own README, plus what testing surfaced: ``` skiff new-app --name --ip [--port 8080] [--health-path /healthz] - [--server-name ] [--base-snapshot ready] [--iface vtnet0] + [--base-snapshot ready] [--iface vtnet0] skiff deploy --app --new [--ref refs/heads/main] [--old ] (normally invoked by the post-receive hook, not by hand) @@ -489,4 +626,8 @@ skiff rebuild --all [--base-snapshot ready] [--iface vtnet0] skiff config set --app KEY=VALUE [KEY=VALUE...] skiff config unset --app KEY [KEY...] skiff config list --app + +skiff domains add --app [--primary] +skiff domains remove --app +skiff domains list --app ``` diff --git a/cmd/skiff/main.go b/cmd/skiff/main.go index 3286da7..cc48c67 100644 --- a/cmd/skiff/main.go +++ b/cmd/skiff/main.go @@ -14,6 +14,7 @@ import ( "skiff/internal/appmeta" "skiff/internal/deploy" + "skiff/internal/domains" "skiff/internal/envfile" "skiff/internal/jailmgr" "skiff/internal/layout" @@ -26,7 +27,7 @@ import ( func main() { if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, "usage: skiff [flags]") + fmt.Fprintln(os.Stderr, "usage: skiff [flags]") os.Exit(1) } @@ -42,6 +43,8 @@ func main() { err = cmdRebuild(os.Args[2:]) case "config": err = cmdConfig(os.Args[2:]) + case "domains": + err = cmdDomains(os.Args[2:]) default: fmt.Fprintf(os.Stderr, "unknown subcommand %q\n", os.Args[1]) os.Exit(1) @@ -86,7 +89,6 @@ func cmdNewApp(args []string) error { ip := fs.String("ip", "", "IPv4 address to assign the jail (operator-chosen)") port := fs.Int("port", layout.DefaultHealthPort, "port the app listens on inside the jail") healthPath := fs.String("health-path", layout.DefaultHealthPath, "health check path") - serverName := fs.String("server-name", "", "nginx server_name (defaults to )") baseSnapshot := fs.String("base-snapshot", "ready", "snapshot on the base dataset to clone from") iface := fs.String("iface", layout.DefaultJailIface, "host interface the jail's IP is aliased onto") fs.Parse(args) @@ -94,9 +96,6 @@ func cmdNewApp(args []string) error { if *name == "" || *ip == "" { return fmt.Errorf("--name and --ip are required") } - if *serverName == "" { - *serverName = *name - } ctx := context.Background() out := os.Stdout @@ -136,7 +135,6 @@ func cmdNewApp(args []string) error { IP: *ip, Port: *port, HealthPath: *healthPath, - ServerName: *serverName, }); err != nil { return fmt.Errorf("write app metadata: %w", err) } @@ -158,7 +156,7 @@ func cmdNewApp(args []string) error { return fmt.Errorf("start jail: %w", err) } - fmt.Fprintf(out, "\n%s is ready. From the app's repo:\n git remote add skiff :%s\n git push skiff main\n", *name, cfg.RepoPath(*name)) + fmt.Fprintf(out, "\n%s is ready. From the app's repo:\n git remote add skiff :%s\n git push skiff main\n\nNo domains are routed yet - once DNS points at %s:\n skiff domains add --app %s \n", *name, cfg.RepoPath(*name), *ip, *name) return nil } @@ -396,3 +394,107 @@ func applyConfigChange(app string, mutate func(map[string]string) error) error { fmt.Fprintln(out, "==> config updated") return nil } + +func cmdDomains(args []string) error { + if len(args) < 1 { + return fmt.Errorf("usage: skiff domains --app [args...]") + } + switch args[0] { + case "add": + return cmdDomainsAdd(args[1:]) + case "remove": + return cmdDomainsRemove(args[1:]) + case "list": + return cmdDomainsList(args[1:]) + default: + return fmt.Errorf("unknown domains subcommand %q", args[0]) + } +} + +// Unlike cmdConfigSet, these skip deploy.GuardedApply - see internal/domains. +func cmdDomainsAdd(args []string) error { + fs := flag.NewFlagSet("domains add", flag.ExitOnError) + app := fs.String("app", "", "app name") + primary := fs.Bool("primary", false, "make this the app's primary domain - every other domain redirects to it") + fs.Parse(args) + rest := fs.Args() + + if *app == "" || len(rest) != 1 { + return fmt.Errorf("usage: skiff domains add --app [--primary] ") + } + + cfg := layout.Default() + if _, err := os.Stat(cfg.MetaPath(*app)); err != nil { + return fmt.Errorf("app %q not found: %w", *app, err) + } + + unlock, err := lock.Acquire(cfg.LockPath(*app)) + if err != nil { + return fmt.Errorf("acquire app lock: %w", err) + } + defer unlock.Release() + + return domains.Add(context.Background(), os.Stdout, cfg, *app, rest[0], *primary) +} + +func cmdDomainsRemove(args []string) error { + fs := flag.NewFlagSet("domains remove", flag.ExitOnError) + app := fs.String("app", "", "app name") + fs.Parse(args) + rest := fs.Args() + + if *app == "" || len(rest) != 1 { + return fmt.Errorf("usage: skiff domains remove --app ") + } + + cfg := layout.Default() + if _, err := os.Stat(cfg.MetaPath(*app)); err != nil { + return fmt.Errorf("app %q not found: %w", *app, err) + } + + unlock, err := lock.Acquire(cfg.LockPath(*app)) + if err != nil { + return fmt.Errorf("acquire app lock: %w", err) + } + defer unlock.Release() + + return domains.Remove(context.Background(), os.Stdout, cfg, *app, rest[0]) +} + +func cmdDomainsList(args []string) error { + fs := flag.NewFlagSet("domains list", flag.ExitOnError) + app := fs.String("app", "", "app name") + fs.Parse(args) + + if *app == "" { + return fmt.Errorf("--app is required") + } + + cfg := layout.Default() + meta, err := appmeta.Read(cfg.MetaPath(*app)) + if err != nil { + return fmt.Errorf("app %q not found: %w", *app, err) + } + if len(meta.Domains) == 0 { + fmt.Println("no domains configured") + return nil + } + + tlsLive := make(map[string]bool, len(meta.TLSDomains)) + for _, h := range meta.TLSDomains { + tlsLive[h] = true + } + + fmt.Printf("%-30s %-8s %s\n", "HOSTNAME", "PRIMARY", "TLS") + for _, d := range meta.Domains { + primaryCol, tlsCol := "no", "pending" + if d.Primary { + primaryCol = "yes" + } + if tlsLive[d.Hostname] { + tlsCol = "issued" + } + fmt.Printf("%-30s %-8s %s\n", d.Hostname, primaryCol, tlsCol) + } + return nil +} diff --git a/internal/appmeta/appmeta.go b/internal/appmeta/appmeta.go index 61ceb18..72c8afc 100644 --- a/internal/appmeta/appmeta.go +++ b/internal/appmeta/appmeta.go @@ -1,7 +1,7 @@ // Package appmeta stores the small amount of per-app networking config -// Skiff itself needs (IP, port, health check) that isn't already implied by -// jail.conf/rc.conf - a flat file living with the app's own data, not a -// separate control-plane database. +// Skiff itself needs (IP, port, health check, domains) that isn't already +// implied by jail.conf/rc.conf - a flat file living with the app's own +// data, not a separate control-plane database. package appmeta import ( @@ -9,11 +9,30 @@ import ( "os" ) +// Domain is one hostname routed to the app; every non-Primary one redirects +// to whichever is Primary. +type Domain struct { + Hostname string `json:"hostname"` + Primary bool `json:"primary"` +} + type Meta struct { - IP string `json:"ip"` - Port int `json:"port"` - HealthPath string `json:"health_path"` - ServerName string `json:"server_name"` + IP string `json:"ip"` + Port int `json:"port"` + HealthPath string `json:"health_path"` + Domains []Domain `json:"domains,omitempty"` + // Hostnames the app's cert actually covers as of the last sync - may + // lag Domains after an add (not yet issued) or a remove (not pruned). + TLSDomains []string `json:"tls_domains,omitempty"` +} + +func (m Meta) PrimaryDomain() (Domain, bool) { + for _, d := range m.Domains { + if d.Primary { + return d, true + } + } + return Domain{}, false } func Write(path string, m Meta) error { diff --git a/internal/certbot/certbot.go b/internal/certbot/certbot.go new file mode 100644 index 0000000..3ce5b41 --- /dev/null +++ b/internal/certbot/certbot.go @@ -0,0 +1,26 @@ +// Package certbot issues Let's Encrypt certificates via HTTP-01 webroot +// validation. Deliberately not the --nginx installer plugin - Skiff, not +// certbot, owns and regenerates skiff.d/*.conf. +package certbot + +import ( + "context" + "io" + + "skiff/internal/sh" +) + +// Sync issues or expands the certificate named certName to cover exactly +// domains, via challenge files served from webroot. Idempotent. Assumes +// `certbot register` has already been run once on the host. +func Sync(ctx context.Context, out io.Writer, webroot, certName string, domains []string) error { + args := []string{ + "certonly", "--webroot", "-w", webroot, + "--cert-name", certName, "--expand", + "--non-interactive", "--agree-tos", "--keep-until-expiring", + } + for _, d := range domains { + args = append(args, "-d", d) + } + return sh.Run(ctx, out, "", "certbot", args...) +} diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index 8d32613..736b2e1 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -98,7 +98,7 @@ func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, sha string) if err != nil { return fmt.Errorf("read app metadata: %w", err) } - if err := nginx.EnsureRoute(ctx, out, cfg, app, meta.IP, meta.Port, meta.ServerName); err != nil { + if err := nginx.EnsureRoute(ctx, out, cfg, app, meta); err != nil { return fmt.Errorf("nginx: %w", err) } diff --git a/internal/domains/domains.go b/internal/domains/domains.go new file mode 100644 index 0000000..5bf264f --- /dev/null +++ b/internal/domains/domains.go @@ -0,0 +1,128 @@ +// Package domains manages an app's public hostnames and TLS. Unlike other +// app config, a domain/cert change never touches the jail, so it skips +// Skiff's usual snapshot-and-restart safety net entirely. +package domains + +import ( + "context" + "fmt" + "io" + "os" + + "skiff/internal/appmeta" + "skiff/internal/certbot" + "skiff/internal/layout" + "skiff/internal/nginx" +) + +// Add registers hostname for app and syncs routing/TLS. Safe to retry: a +// DNS-not-ready cert failure just leaves the app on plain HTTP. +func Add(ctx context.Context, out io.Writer, cfg layout.Config, app, hostname string, primary bool) error { + meta, err := appmeta.Read(cfg.MetaPath(app)) + if err != nil { + return fmt.Errorf("read app metadata: %w", err) + } + + makePrimary := primary || len(meta.Domains) == 0 + found := false + for i := range meta.Domains { + if meta.Domains[i].Hostname == hostname { + found = true + break + } + } + if makePrimary { + for i := range meta.Domains { + meta.Domains[i].Primary = meta.Domains[i].Hostname == hostname + } + } + if !found { + meta.Domains = append(meta.Domains, appmeta.Domain{Hostname: hostname, Primary: makePrimary}) + } + + if err := appmeta.Write(cfg.MetaPath(app), meta); err != nil { + return fmt.Errorf("write app metadata: %w", err) + } + + if err := syncCert(ctx, out, cfg, app, meta); err != nil { + return err + } + fmt.Fprintf(out, "==> %s added for %s\n", hostname, app) + return nil +} + +// Remove deregisters hostname and updates routing only - unlike Add, it +// never calls certbot, so it can't fail or block on it. Removing the +// primary while others remain is refused - reassign primary first with +// `skiff domains add --primary`. +func Remove(ctx context.Context, out io.Writer, cfg layout.Config, app, hostname string) error { + meta, err := appmeta.Read(cfg.MetaPath(app)) + if err != nil { + return fmt.Errorf("read app metadata: %w", err) + } + + idx := -1 + for i, d := range meta.Domains { + if d.Hostname == hostname { + idx = i + break + } + } + if idx == -1 { + return fmt.Errorf("%s is not registered for %s", hostname, app) + } + if meta.Domains[idx].Primary && len(meta.Domains) > 1 { + return fmt.Errorf("%s is %s's primary domain - reassign primary to another domain first (skiff domains add --primary )", hostname, app) + } + meta.Domains = append(meta.Domains[:idx], meta.Domains[idx+1:]...) + + if err := appmeta.Write(cfg.MetaPath(app), meta); err != nil { + return fmt.Errorf("write app metadata: %w", err) + } + + if len(meta.Domains) == 0 { + if err := nginx.Remove(ctx, out, cfg, app); err != nil { + return err + } + } else if err := nginx.EnsureRoute(ctx, out, cfg, app, meta); err != nil { + return fmt.Errorf("nginx: %w", err) + } + + fmt.Fprintf(out, "==> %s removed from %s\n", hostname, app) + return nil +} + +// syncCert routes over plain HTTP first, so a newly added hostname is +// reachable for the HTTP-01 challenge, then requests/expands the cert and +// re-routes with TLS enabled for whatever it now covers. +func syncCert(ctx context.Context, out io.Writer, cfg layout.Config, app string, meta appmeta.Meta) error { + primary, ok := meta.PrimaryDomain() + if !ok { + return fmt.Errorf("%s has domains but none marked primary", app) + } + + if err := nginx.EnsureRoute(ctx, out, cfg, app, meta); err != nil { + return fmt.Errorf("nginx: %w", err) + } + + if err := os.MkdirAll(cfg.AcmeWebrootDir, 0755); err != nil { + return fmt.Errorf("create acme webroot: %w", err) + } + + hostnames := make([]string, len(meta.Domains)) + for i, d := range meta.Domains { + hostnames[i] = d.Hostname + } + + fmt.Fprintln(out, "==> requesting certificate") + if err := certbot.Sync(ctx, out, cfg.AcmeWebrootDir, primary.Hostname, hostnames); err != nil { + return fmt.Errorf("certificate not issued (%w) - fix DNS and re-run this command", err) + } + + meta.TLSDomains = hostnames + if err := appmeta.Write(cfg.MetaPath(app), meta); err != nil { + return fmt.Errorf("write app metadata: %w", err) + } + + return nginx.EnsureRoute(ctx, out, cfg, app, meta) +} diff --git a/internal/layout/layout.go b/internal/layout/layout.go index 5ac6c26..f350efb 100644 --- a/internal/layout/layout.go +++ b/internal/layout/layout.go @@ -24,6 +24,9 @@ type Config struct { JailConfDir string // /etc/jail.conf.d NginxConfDir string // nginx include dir regenerated on every deploy LocksDir string // per-app flock files live here, outside any per-app dataset + + AcmeWebrootDir string // static dir nginx serves ACME HTTP-01 challenge files from, shared by every app + LetsEncryptDir string // certbot's own state dir } func Default() Config { @@ -35,6 +38,9 @@ func Default() Config { JailConfDir: "/etc/jail.conf.d", NginxConfDir: "/usr/local/etc/nginx/skiff.d", LocksDir: "/skiff/locks", + + AcmeWebrootDir: "/skiff/acme-challenge", + LetsEncryptDir: "/usr/local/etc/letsencrypt", } } @@ -78,6 +84,13 @@ func (c Config) NginxConfPath(app string) string { return filepath.Join(c.NginxConfDir, app+".conf") } +// LetsEncryptLiveDir is where certbot writes the live cert/key for a +// certificate named certName (Skiff always names one after its app's +// primary domain). +func (c Config) LetsEncryptLiveDir(certName string) string { + return filepath.Join(c.LetsEncryptDir, "live", certName) +} + func (c Config) PostReceiveHookPath(app string) string { return filepath.Join(c.RepoPath(app), "hooks", "post-receive") } diff --git a/internal/nginx/nginx.go b/internal/nginx/nginx.go index abdd4c9..4a85435 100644 --- a/internal/nginx/nginx.go +++ b/internal/nginx/nginx.go @@ -9,19 +9,44 @@ import ( "io" "os" + "skiff/internal/appmeta" "skiff/internal/layout" "skiff/internal/sh" "skiff/internal/tmpl" ) -// EnsureRoute writes app's vhost file and reloads nginx, but only if the -// rendered config actually changed - most deploys don't touch routing. -func EnsureRoute(ctx context.Context, out io.Writer, cfg layout.Config, app, ip string, port int, serverName string) error { +// EnsureRoute (re)writes app's vhost file from meta and reloads nginx, but +// only if the rendered config changed. A hostname gets HTTPS only once it's +// in both meta.Domains and meta.TLSDomains. No domains yet is a no-op. +func EnsureRoute(ctx context.Context, out io.Writer, cfg layout.Config, app string, meta appmeta.Meta) error { + if len(meta.Domains) == 0 { + fmt.Fprintf(out, "==> no domains configured for %s, skipping nginx route\n", app) + return nil + } + primary, ok := meta.PrimaryDomain() + if !ok { + return fmt.Errorf("%s has domains but none marked primary", app) + } + + live := make(map[string]bool, len(meta.TLSDomains)) + for _, h := range meta.TLSDomains { + live[h] = true + } + + doms := make([]tmpl.NginxDomain, len(meta.Domains)) + for i, d := range meta.Domains { + doms[i] = tmpl.NginxDomain{Hostname: d.Hostname, Primary: d.Primary, CertLive: live[d.Hostname]} + } + rendered, err := tmpl.NginxApp(tmpl.NginxAppData{ - Name: app, - IP: ip, - Port: port, - ServerName: serverName, + Name: app, + IP: meta.IP, + Port: meta.Port, + Webroot: cfg.AcmeWebrootDir, + PrimaryHostname: primary.Hostname, + PrimaryCertLive: live[primary.Hostname], + PrimaryLiveDir: cfg.LetsEncryptLiveDir(primary.Hostname), + Domains: doms, }) if err != nil { return err @@ -40,6 +65,20 @@ func EnsureRoute(ctx context.Context, out io.Writer, cfg layout.Config, app, ip return Reload(ctx, out) } +// Remove deletes app's vhost file entirely and reloads - used once its last +// domain is removed, since there's nothing left to route. +func Remove(ctx context.Context, out io.Writer, cfg layout.Config, app string) error { + path := cfg.NginxConfPath(app) + if err := os.Remove(path); err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("remove nginx conf for %s: %w", app, err) + } + fmt.Fprintf(out, "==> nginx route removed for %s, reloading\n", app) + return Reload(ctx, out) +} + func Reload(ctx context.Context, out io.Writer) error { return sh.Run(ctx, out, "", "nginx", "-s", "reload") } diff --git a/internal/tmpl/nginx-app.conf.tmpl b/internal/tmpl/nginx-app.conf.tmpl index b55a332..66ce0d5 100644 --- a/internal/tmpl/nginx-app.conf.tmpl +++ b/internal/tmpl/nginx-app.conf.tmpl @@ -2,14 +2,48 @@ upstream {{.Name}}_upstream { server {{.IP}}:{{.Port}}; } +{{range .Domains}} server { listen 80; - server_name {{.ServerName}}; + server_name {{.Hostname}}; + + location ^~ /.well-known/acme-challenge/ { + root {{$.Webroot}}; + } + + location / { +{{if .CertLive}} + return 301 https://{{$.PrimaryHostname}}$request_uri; +{{else if .Primary}} + proxy_pass http://{{$.Name}}_upstream; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +{{else if $.PrimaryCertLive}} + return 301 https://{{$.PrimaryHostname}}$request_uri; +{{else}} + return 301 http://{{$.PrimaryHostname}}$request_uri; +{{end}} + } +} +{{if .CertLive}} +server { + listen 443 ssl; + server_name {{.Hostname}}; + + ssl_certificate {{$.PrimaryLiveDir}}/fullchain.pem; + ssl_certificate_key {{$.PrimaryLiveDir}}/privkey.pem; location / { - proxy_pass http://{{.Name}}_upstream; +{{if .Primary}} + proxy_pass http://{{$.Name}}_upstream; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; +{{else}} + return 301 https://{{$.PrimaryHostname}}$request_uri; +{{end}} } } +{{end}} +{{end}} diff --git a/internal/tmpl/tmpl.go b/internal/tmpl/tmpl.go index 12baa70..0213919 100644 --- a/internal/tmpl/tmpl.go +++ b/internal/tmpl/tmpl.go @@ -46,11 +46,24 @@ type RCData struct{ Name string } func RCScript(d RCData) (string, error) { return Render(rcScript, d) } +// NginxDomain is one hostname in an app's vhost. CertLive gates HTTPS. +type NginxDomain struct { + Hostname string + Primary bool + CertLive bool +} + type NginxAppData struct { - Name string - IP string - Port int - ServerName string + Name string + IP string + Port int + Webroot string // ACME HTTP-01 challenge webroot, shared across apps + + PrimaryHostname string + PrimaryCertLive bool + PrimaryLiveDir string // certbot's live dir for the app's shared cert + + Domains []NginxDomain } func NginxApp(d NginxAppData) (string, error) { return Render(nginxApp, d) } -- cgit v1.3