// Package deploy orchestrates one push: checkout, build, snapshot, release // phase, cutover, health check, and - on a failed health check - a full // dataset rollback. It is the entire "control plane": one function, called // synchronously from the repo's post-receive hook. package deploy import ( "context" "fmt" "io" "net/http" "os" "path/filepath" "time" "skiff/internal/appmeta" "skiff/internal/jailmgr" "skiff/internal/layout" "skiff/internal/nginx" "skiff/internal/release" "skiff/internal/sh" "skiff/internal/zfsutil" ) func Run(ctx context.Context, out io.Writer, cfg layout.Config, app, sha string) error { dataset := cfg.AppDataset(app) snapshotTag := "pre-" + sha releaseDir := cfg.ReleaseDir(app, sha) fmt.Fprintf(out, "==> checking out %s\n", sha) // A previous rollback can revert the app's dataset (and so delete a // release dir) without the bare repo - on a separate dataset - ever // knowing; prune first or `worktree add` fails on the stale entry. if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "prune"); err != nil { fmt.Fprintf(out, "==> warning: worktree prune failed: %v\n", err) } if err := sh.Run(ctx, out, "", "git", "-C", cfg.RepoPath(app), "worktree", "add", releaseDir, sha); err != nil { return fmt.Errorf("checkout: %w", err) } fmt.Fprintln(out, "==> building") if err := sh.Run(ctx, out, releaseDir, "make", "-B"); err != nil { return fmt.Errorf("build: %w", err) } if _, err := os.Stat(filepath.Join(releaseDir, "app")); err != nil { return fmt.Errorf("build: `make` did not produce ./app: %w", err) } fmt.Fprintln(out, "==> snapshotting current state") if err := zfsutil.Snapshot(ctx, out, dataset, snapshotTag); err != nil { return fmt.Errorf("snapshot: %w", err) } hasRelease, err := release.Declared(ctx, releaseDir) if err != nil { return fmt.Errorf("check for release target: %w", err) } if hasRelease { if err := release.Run(ctx, out, app, cfg.ReleaseDirInJail(sha)); err != nil { return fmt.Errorf("release phase failed, deploy aborted before cutover: %w", err) } } else { fmt.Fprintln(out, "==> no release target declared, skipping") } fmt.Fprintln(out, "==> cutting over") if err := swapCurrent(cfg.CurrentRelease(app), cfg.ReleaseDirInJail(sha)); err != nil { return fmt.Errorf("cutover: %w", err) } if err := jailmgr.Restart(ctx, out, app); err != nil { return fmt.Errorf("restart: %w", err) } meta, err := appmeta.Read(cfg.MetaPath(app)) if err != nil { return fmt.Errorf("read app metadata: %w", err) } if err := waitHealthy(ctx, meta); err != nil { fmt.Fprintf(out, "==> health check failed (%v), rolling back to pre-%s\n", err, sha) if rbErr := zfsutil.Rollback(ctx, out, dataset, snapshotTag); rbErr != nil { return fmt.Errorf("health check failed AND rollback failed: %v / %w", err, rbErr) } if rbErr := jailmgr.Restart(ctx, out, app); rbErr != nil { return fmt.Errorf("health check failed, rolled back, but restart failed: %v / %w", err, rbErr) } return fmt.Errorf("deploy failed health check and was rolled back: %w", err) } fmt.Fprintln(out, "==> healthy, updating routing") if err := nginx.EnsureRoute(ctx, out, cfg, app, meta.IP, meta.Port, meta.ServerName); err != nil { return fmt.Errorf("nginx: %w", err) } fmt.Fprintf(out, "==> deployed %s at %s\n", app, sha) return nil } // swapCurrent atomically repoints the `current` symlink at newTarget by // building the replacement next to it and renaming over it - never leaves // the symlink briefly missing. func swapCurrent(currentPath, newTarget string) error { tmp := currentPath + ".next" os.Remove(tmp) if err := os.Symlink(newTarget, tmp); err != nil { return err } return os.Rename(tmp, currentPath) } func waitHealthy(ctx context.Context, meta appmeta.Meta) error { url := fmt.Sprintf("http://%s:%d%s", meta.IP, meta.Port, meta.HealthPath) deadline := time.Now().Add(layout.HealthTimeoutSecs * time.Second) client := &http.Client{Timeout: 2 * time.Second} var lastErr error for time.Now().Before(deadline) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return err } resp, err := client.Do(req) if err == nil { resp.Body.Close() if resp.StatusCode < 300 { return nil } lastErr = fmt.Errorf("status %d", resp.StatusCode) } else { lastErr = err } time.Sleep(500 * time.Millisecond) } return fmt.Errorf("never became healthy at %s: %w", url, lastErr) }