// Package deploy orchestrates one push: checkout, build, snapshot, release // phase, cutover, health check, and - on a failed health check - a full // dataset rollback. It is the entire "control plane": one function, called // synchronously from the repo's post-receive hook. package deploy import ( "context" "fmt" "io" "os" "path/filepath" "skiff/internal/appmeta" "skiff/internal/health" "skiff/internal/jailmgr" "skiff/internal/layout" "skiff/internal/nginx" "skiff/internal/release" "skiff/internal/sh" "skiff/internal/zfsutil" ) func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, sha string) error { releaseDir := cfg.ReleaseDir(app, sha) fmt.Fprintf(out, "==> checking out %s\n", sha) // A previous rollback can revert the app's dataset (and so delete a // release dir) without the bare repo - on a separate dataset - ever // knowing; prune first or `worktree add` fails on the stale entry. if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "prune"); err != nil { fmt.Fprintf(out, "==> warning: worktree prune failed: %v\n", err) } if _, err := os.Stat(releaseDir); err == nil { // A retry of this exact sha whose previous attempt got far enough // to cut over, then failed its health check: the dataset rollback // reverts `current` but - by design - leaves this release dir (and // its worktree registration) on disk, since it existed at // snapshot time. Clear it so `add` below doesn't collide. fmt.Fprintln(out, "==> removing leftover checkout from a previous attempt") if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "remove", "--force", releaseDir); err != nil { fmt.Fprintf(out, "==> warning: worktree remove failed (%v), removing directory directly\n", err) if err := os.RemoveAll(releaseDir); err != nil { return fmt.Errorf("remove leftover checkout: %w", err) } } } if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "add", releaseDir, sha); err != nil { return fmt.Errorf("checkout: %w", err) } cutover := false defer func() { if cutover { return } // Nothing ever went live under this checkout - leaving it behind // blocks a retry of the same sha with "already exists" on the next // `worktree add`. if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "remove", "--force", releaseDir); err != nil { fmt.Fprintf(out, "==> warning: failed to clean up worktree %s: %v\n", releaseDir, err) } }() fmt.Fprintln(out, "==> building") if err := sh.Run(ctx, out, releaseDir, "make", "-B"); err != nil { return fmt.Errorf("build: %w", err) } if _, err := os.Stat(filepath.Join(releaseDir, "app")); err != nil { return fmt.Errorf("build: `make` did not produce ./app: %w", err) } if err := GuardedApply(ctx, out, cfg, app, "pre-"+sha, "deploy", func() error { hasRelease, err := release.Declared(ctx, releaseDir) if err != nil { return fmt.Errorf("check for release target: %w", err) } if hasRelease { if err := release.Run(ctx, out, app, cfg.ReleaseDirInJail(sha)); err != nil { return fmt.Errorf("release phase failed, deploy aborted before cutover: %w", err) } } else { fmt.Fprintln(out, "==> no release target declared, skipping") } fmt.Fprintln(out, "==> cutting over") if err := swapCurrent(cfg.CurrentRelease(app), cfg.ReleaseDirInJail(sha)); err != nil { return fmt.Errorf("cutover: %w", err) } cutover = true return nil }); err != nil { return err } fmt.Fprintln(out, "==> healthy, updating routing") meta, err := appmeta.Read(cfg.MetaPath(app)) if err != nil { return fmt.Errorf("read app metadata: %w", err) } if err := nginx.EnsureRoute(ctx, out, cfg, app, meta.IP, meta.Port, meta.ServerName); err != nil { return fmt.Errorf("nginx: %w", err) } fmt.Fprintf(out, "==> deployed %s at %s\n", app, sha) return nil } // GuardedApply snapshots app's dataset under tag, runs apply, restarts the // jail, and waits for it to report healthy - rolling the dataset back to // the snapshot (and restarting again) if it doesn't. Shared by code // deploys and config changes: either one is exactly as much of an outage // as the other if it goes wrong, so both are guarded the same way. label // is used only in the final error message (e.g. "deploy", "config change"). func GuardedApply(ctx context.Context, out io.Writer, cfg layout.Config, app, tag, label string, apply func() error) error { dataset := cfg.AppDataset(app) existing, err := zfsutil.Snapshots(ctx, dataset) if err != nil { return fmt.Errorf("list snapshots: %w", err) } for _, s := range existing { if s != tag { continue } // A previous attempt under this same tag (e.g. a retried deploy of // the same sha, rolled back to this exact snapshot) left it // behind - retaking it now just recaptures the state a rollback // already reverted the dataset to. fmt.Fprintf(out, "==> replacing stale snapshot %s from a previous attempt\n", tag) if err := zfsutil.DestroySnapshot(ctx, out, dataset, tag); err != nil { return fmt.Errorf("clean up stale snapshot: %w", err) } break } fmt.Fprintln(out, "==> snapshotting current state") if err := zfsutil.Snapshot(ctx, out, dataset, tag); err != nil { return fmt.Errorf("snapshot: %w", err) } if err := apply(); err != nil { // Nothing has been restarted yet, so this snapshot was never needed // as a rollback target - left behind, it collides with the same // tag on the next retry. if destroyErr := zfsutil.DestroySnapshot(ctx, out, dataset, tag); destroyErr != nil { fmt.Fprintf(out, "==> warning: failed to clean up snapshot %s: %v\n", tag, destroyErr) } return err } if err := jailmgr.Restart(ctx, out, app); err != nil { return fmt.Errorf("restart: %w", err) } meta, err := appmeta.Read(cfg.MetaPath(app)) if err != nil { return fmt.Errorf("read app metadata: %w", err) } if err := health.WaitHealthy(ctx, meta); err != nil { fmt.Fprintf(out, "==> health check failed (%v), rolling back to %s\n", err, tag) if rbErr := zfsutil.Rollback(ctx, out, dataset, tag); rbErr != nil { return fmt.Errorf("health check failed AND rollback failed: %v / %w", err, rbErr) } if rbErr := jailmgr.Restart(ctx, out, app); rbErr != nil { return fmt.Errorf("health check failed, rolled back, but restart failed: %v / %w", err, rbErr) } return fmt.Errorf("%s failed health check and was rolled back: %w", label, err) } return nil } // swapCurrent atomically repoints the `current` symlink at newTarget by // building the replacement next to it and renaming over it - never leaves // the symlink briefly missing. func swapCurrent(currentPath, newTarget string) error { tmp := currentPath + ".next" os.Remove(tmp) if err := os.Symlink(newTarget, tmp); err != nil { return err } return os.Rename(tmp, currentPath) }